diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index db47e49e5..459a66922 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -183,7 +183,7 @@ "packages/app/src/context/models.tsx": "8904525767b36ed5ba93e85a0fddb9bf737a8d066dfff084380ce1a67182f535", "packages/app/src/context/platform.tsx": "edd9b1773feb5c084b922a57b6914f6a473eed558b44f9de0eeb1c4d4e2b49da", "packages/app/src/context/server-sdk.test.ts": "833920190c25089a5c3ed9849a62b021d2e0e2e4fb76196af8ddc9e6aa9817ba", - "packages/app/src/context/server-sdk.tsx": "36cfd55e634dbb974d4d6785281fedc80f4ddd539090f859fa7156f959ab500d", + "packages/app/src/context/server-sdk.tsx": "bfa88452f7d42ffd0453425995af5294bfc34dade76cee3fbd627bb96051725a", "packages/app/src/context/server-session-child-active.test.ts": "de8c467f67f88b6cce351c1805ec90728cf238eb1ef32f39ffdbbf3de4dcec22", "packages/app/src/context/server-session-orphan-part.test.ts": "1d635374c0bfff25a01825061aabac7334e5cdaedca2dab9f3d9467ff543caea", "packages/app/src/context/server-session.test.ts": "30cd24bb1345376fd1a165d65677dee5c36d586efa7969d3b4844967040fda59", @@ -200,14 +200,17 @@ "packages/app/src/context/session-working.ts": "aa3f70d326c481524920502aae5fc0cdc4dc2c5067dd94969242629154358e9c", "packages/app/src/context/settings.tsx": "19e078e81620ffa21a92797335009b84d4311485a37c5ede2f76906b734292ad", "packages/app/src/context/split.tsx": "891aeb290b369b327a63ce548dcaab1daca08e41fab430648787be7f84e53ec0", - "packages/app/src/context/sse-gap-frame.test.ts": "cf93c169015d66cebbf25752bc5a99296af80a7253bde8b6e565d2c0e50410f3", + "packages/app/src/context/sse-gap-frame.test.ts": "4b76900f9e28680ea1aaed8b391518ce08d0f04e7538a336b584834b913fa62b", "packages/app/src/context/sse-gap-frame.ts": "2f64fde540a305579762c5276721d4bbfbf7c862131aaf4462e8a4e8f32c5960", "packages/app/src/context/sse-origin.test.ts": "71cd3aa996a51d7e99236a49cede4769e520e20e53f57c6920fe441f2abb0c7d", "packages/app/src/context/sse-origin.ts": "da9a231907dc50d87cf01758177bee0d4a0b0165d8adb5e152db8eca29dfd9f5", + "packages/app/src/context/sse-reader.test.ts": "6429e846ef4ee2bccc25cdd7ee1915d787b557fb9355d6fbf2c99d15846337ac", + "packages/app/src/context/stream-cursor.test.ts": "c9715bb28f576cdc00683b0bdc27bdc2a56a6f212969cc090f99b3dc8988e6a7", + "packages/app/src/context/stream-cursor.ts": "2711f85581cd86bb69b50a441b6a04c532553f61fa5e00cd1b9daa12193d3697", "packages/app/src/context/stream-gap.test.ts": "b7203445564f44fd1d966ef684a11cb9b5ec567d9ab54f1b7617726fdf44d3f5", "packages/app/src/context/stream-gap.ts": "2fd8eba30a6112cebf38e33c9418ab2b7d9c9d466f76729d828d5ab52d56c5ec", - "packages/app/src/context/stream-liveness.test.ts": "63a06ca7fef3e38c0c9abfa23df3d9043b1a804f3c527400d08ed9eb32c466b0", - "packages/app/src/context/stream-liveness.ts": "94fe6fe8a776eb0de6903546f653c89f4577cf7628e7f051afa83280e391d16f", + "packages/app/src/context/stream-liveness.test.ts": "2759eb2233f8ff6ced02957f971e756fe525575b2c8aefcbd7749470dec960f2", + "packages/app/src/context/stream-liveness.ts": "875fe871312ad15f6376b933659c152f2241b8101e70de97ed410a720cbbafaa", "packages/app/src/context/tab-migration.ts": "569e69cd3425917c7e195f91c3a838abea68b616267ea7511f3ccf01d94ac01e", "packages/app/src/context/tabs.tsx": "06d6ddb6d50727a4e0a01ac1ca4a70b06bf63a4e5f881526aead54ccde62472b", "packages/app/src/context/vault-panel.ts": "f91cb5fb49a9f0d46bd2a1b131f9a218b30147e2ad3617d2a2dc4f1e14599a53", @@ -233,6 +236,7 @@ "packages/app/src/i18n/uk.ts": "c1f2fb529563fa211e3e322aca0d9417a34909fd056ae80b65c3861b0d6caee4", "packages/app/src/i18n/zh.ts": "50755c05ef47e910c3de7c1471f66b7cd21cea93c005480636ec085a11d5e09b", "packages/app/src/i18n/zht.ts": "c65962a29d6d772d1333ce85a7b3a6f8f3246fbef634ec148cc6531253bbeae9", + "packages/app/src/pages/directory-layout.tsx": "e580e6197408358ddd44eaa76cddd0828e87db255a9068971998f787851369d6", "packages/app/src/pages/empty-workspace-landing.tsx": "ac9410f5b223b30e2d82a6d222992fed2226b6afc1213ff0c8305a230cda7cb2", "packages/app/src/pages/error.tsx": "db9ba0847cfb205100ad6133cce30c6ea2ce983802af44947e6d2b746c2a6ec5", "packages/app/src/pages/hold-debug-badge-gate.test.ts": "661f59e172a5b074a6807896f46c18338faa59b6d9c0912fd824df2dae4a5730", @@ -265,6 +269,8 @@ "packages/app/src/utils/amicode-workspace-projects.test.ts": "8b5c32172e2039b3d9c52f2257023263fc846132e7c185e4a63ae014bc2e2bfb", "packages/app/src/utils/amicode-workspace-projects.ts": "cbe091c463177375c93cb43676c202bd7bf06baf36fdec779f7d641558ff6bee", "packages/app/src/utils/chrome-dropdown.ts": "f8b878c5df4e1afd96b0751c73daaad0a8127411030724cb8f9a5089ebb29eac", + "packages/app/src/utils/diffs.test.ts": "38a5b5dbe23014660204fe08e30599dad16c93665eeaf555284915d0de83a569", + "packages/app/src/utils/diffs.ts": "313ec1085e24b5b8d2844cf42ea2caa93cda46081a6aa5691de5bce8a6686744", "packages/app/src/utils/global-clipboard.test.ts": "18ac0ea486d36eed00597f2718f03bffce70a6e6a8f77c79aefe0795790d8771", "packages/app/src/utils/global-clipboard.ts": "b26373289d7402c88e8038575703b11fa5e4ef5a7f50ee05bceb45488022ab26", "packages/app/src/utils/pane-bridge.ts": "a8a8a40dc60a367086f3c4e4c561a289fb85d8b2086a8d62c2b66102de652ce4", @@ -316,6 +322,9 @@ "packages/llm/src/protocols/bedrock-converse.ts": "834a415fa8ddf9ad1224133f8352d0a926694f19f0954201d182111c171ec03b", "packages/llm/test/provider/bedrock-converse.test.ts": "e8e14d60e8502e928f170f8ac5b406c44bf16d2dd34996ddc2fb7b9261decf85", "packages/opencode/src/agent/agent.ts": "512372a4c62997e9565266638ada19b9dc0a3685b0c0483fd28fede2abdb5c9b", + "packages/opencode/src/bus/global-replay.ts": "3ce244c2dd0fd75cc1a9907f65d758833fae5da7165860189ca80fcd73e7ef60", + "packages/opencode/src/bus/global.ts": "65b3b380936fc34b2c5ef9f3ab419a6c8e6d4cfc696302a266188f26ccdc320a", + "packages/opencode/src/control-plane/workspace.ts": "cc56fb9a6dc6e0fe88e14237ff8aa29a46ce12971ea69a9d80982e0c58128c13", "packages/opencode/src/effect/runner.ts": "b660b5b22df810b16293c23c6d832e5e8dcfb12b9a8a858c875d2c051c855b81", "packages/opencode/src/mcp/browser.ts": "ecdda42cbd55a97b8d14050aa15c6e6450a1d63b0a641c8c8e5b81e50f957c87", "packages/opencode/src/mcp/index.ts": "da45165e6791a5b30c8962c7fd0ea391bb8e1931cc2357c6b90e9fd090cdeb0d", @@ -353,6 +362,8 @@ "packages/opencode/src/tool/edit.ts": "38aa4ce0751ff99bbb0807bcda52f8b9a1d32b6e342df4cc2aedab1f03d342b1", "packages/opencode/src/tool/question.txt": "49bbecb33a825620c2ed14ec9b61faa294359c3673c33277e32f57c1070d2806", "packages/opencode/src/tool/registry.ts": "0097ab8a7904559d0354d165245caff6bf0e887c99ce267be77fa4ee4b66ec3c", + "packages/opencode/src/tool/shell-output-throttle.ts": "00933a5c28bf4f3026541b1ca226678778177146b5b0b56bb5f189d837be9067", + "packages/opencode/src/tool/shell.ts": "09808bb3865574853d162bfd7e1492a618c230828b95ec3989e9e0afa4acb97c", "packages/opencode/src/tool/task.ts": "ddeb07a1cf009dcafc1d1d1c86b9d507676a8991db33a4632c8f29ae0e70e654", "packages/opencode/src/tool/truncate.ts": "92c59d83c4a660116d6ecc48e900aeb16bde93962bfae9f520193fda18273379", "packages/opencode/src/tool/widget-author.ts": "912be8636b162adca8796a501e927019c2449393c14bb5f3ff478b02083b73a9", @@ -360,6 +371,7 @@ "packages/opencode/src/tool/write.ts": "094661a3bdd3fab2cd439c0b6cb942b6a2653d3e9451420523b33acbc762d1a7", "packages/opencode/src/worktree/index.ts": "a86f542b579ef40f2940d36e2c8011ce5592d4811cd5ed437bafe614789fb87e", "packages/opencode/test/acp/service-session.test.ts": "5810c4a0c935f06389c5f01df791e4ae82a4c32268407baf32dc4502d487f282", + "packages/opencode/test/control-plane/workspace-sync-optin.test.ts": "dd77386e26d2d5b357f22e19ae384f35befe08421556f84215a5ac6678d78ac8", "packages/opencode/test/effect/runner-queue.test.ts": "9ce3074dc4a86f620381e2c7ada08b9034b500a62b5da1a7d63896f2519085ca", "packages/opencode/test/effect/runner.test.ts": "6be02ad6a7d173642100a9e17b323410bad7ece842a00018d2170b91d4894c5a", "packages/opencode/test/fixture/fixture.ts": "357bdaa06c6233a91069db20cc5f1cbb2d6eafa3bc7ba3e89b019859d195d6ef", @@ -387,9 +399,11 @@ "packages/opencode/test/server/amicode-widgets-src.test.ts": "fa95521084f95e0c421e7f7f13bf61de7e6b5cc9f42aa9fade486f5c1b53a65d", "packages/opencode/test/server/amicode-widgets.test.ts": "14fc6ccff4e83a5b005a724936e9f0d7ce411fab74109f319e8bc748665f7ab8", "packages/opencode/test/server/auth-accept-set.test.ts": "8fe698812ed9a809d21ace1a9751bd3f1976ab1e58ad5bda65dbfff58df8e3bc", + "packages/opencode/test/server/global-event-replay.test.ts": "a2ccc81b21eb99cb3eeeecd3973738c5ac3a0ecaba59de0516b6d0ab4fcb726c", "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", + "packages/opencode/test/server/httpapi-global-event.test.ts": "0f08bc78a1ba2e3069e7fb292202db88fad532a8603653e4e8593e4dd95ab066", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -398,6 +412,7 @@ "packages/opencode/test/server/session-compact-errors.test.ts": "2ae2044d66db1ba41a5e294afedea5271f11d5bf384685a06ea05ac515b16fb2", "packages/opencode/test/server/session-diff-missing-patch.test.ts": "70526ca7fb86741843fd167f8f2f9c6f6d7e5e415a20a5448d17d24229bdd65b", "packages/opencode/test/server/session-diff-scoped.test.ts": "20ef80b92ad872573b371540575f028be8f1ad3c9f34d72f814b621e14e7a59e", + "packages/opencode/test/server/summary-diff-wire.test.ts": "3b1653608a5c292074a46ea6a9cc0f680e03a3b65bc10c6b499250a5f761c170", "packages/opencode/test/server/workspace-routing.test.ts": "2574731ac379c540f450edc15f8750ac5360e8c33b4a0dad7c61e8eb686137bb", "packages/opencode/test/session/adversarial-e2e.test.ts": "1b471a69223d040c5d5f4f2ee99c2daa6df56bdf11679b65d902c15bedac3e80", "packages/opencode/test/session/business-record.test.ts": "7857094479408bcfd6ea8d9f5b425c6384bcf8490f930674bc1c203e82782d75", @@ -418,6 +433,7 @@ "packages/opencode/test/tool/edit.test.ts": "ec1ed35002b1af4e2ff508aeac456c3edbb2e1dde3f6df4abd957eda9ef6d056", "packages/opencode/test/tool/question.test.ts": "75760ba6ed98f2fdfd75945c3b492342dfe91026f99cde86db19cf52444260c2", "packages/opencode/test/tool/registry-schema.test.ts": "053cb9732299ea6554db007bad873d8d61cd3769aa9514d66c70c3bf0d405b90", + "packages/opencode/test/tool/shell-output-throttle.test.ts": "9fed6f4ea8f7c62941280ab4f973848ea4184274830a153fbea83aae2c028576", "packages/opencode/test/tool/task.test.ts": "fb0276cc33add808c669a4f43649d189d8f62c75974f3092a5f05e45cf176486", "packages/opencode/test/tool/truncation.test.ts": "582ef7199b3f294b1c1588fadda8a1c0ea43ca9af0424a97f48747ade1654ad1", "packages/opencode/test/tool/write.test.ts": "820d36fde2314abbd939ce43ca5bcc4cfa0139d12a4c669716249060d9a955ff", @@ -453,11 +469,14 @@ "packages/session-ui/src/components/session-review.css": "ec8c5b359dd4b7ff80d7fad8906359678adade9ce62c9d74aedc000b646db341", "packages/session-ui/src/components/session-review.tsx": "d2015c9f7419279f08e84523439a0b73e83bd5d06587ad56b85ea53b66ecaee4", "packages/session-ui/src/components/session-turn.css": "792915f3373134eadf3cd51e1d4bfa88ec347e6b037b6caf60b639ec5e74fbd8", - "packages/session-ui/src/components/session-turn.tsx": "9cbc0d7a35f42e32dad53ace7b3c7d92623fa2d3108edb49e0b7bc868f5c1966", + "packages/session-ui/src/components/session-turn.tsx": "42f247b0434a9f52d88d9fb96adff59afe6edeab84dd912b62eb153253b6696f", + "packages/session-ui/src/components/summary-patch.test.ts": "27ee37939eabcfd00dc3a7eb3da95d2179dd222d4193e9f71f7a840f6fd8579e", + "packages/session-ui/src/components/summary-patch.ts": "ab086ea1bc741e15b94f6e9aedc4d934cac17597f4aa1392b146994020006170", "packages/session-ui/src/components/tool-count-label.test.ts": "c62bbe2c662f6740794d25bd7c33fb3b9b46394a1b41d01d6568a8a2ee2b4e75", "packages/session-ui/src/components/tool-count-label.tsx": "06c96479710314d443d42d4c29a29b46d58a55ee84a844a5b6865b49a0513340", "packages/session-ui/src/components/tool-error-card.tsx": "e27b89836b7c709120ac15fbffd2200038944058ea3506e61e5897ffb3537503", "packages/session-ui/src/components/tool-status-title.tsx": "ae4561b58f55985d2294a352802e723b18943ac9a6551d97f1c4a6cc95368c8a", + "packages/session-ui/src/context/data.tsx": "7b840b5bf6929445d21da2cbd8e8006316c6950a09d559ea77e8c397e728cd99", "packages/session-ui/src/pierre/worker.ts": "7f210f931777956567eaeea15f2f5478befc7874a64a4b685504c59c66503fee", "packages/session-ui/src/util/clipboard.ts": "c37ba53914714ed04135e5a254e8ea90186f097d7470fa0873a83de82f3fef2d", "packages/ui/src/amicode/amico-presence.css": "b3f345b607e318dcfbefcdc677fe21cc7d387b9946b0f9d625b0a842fe7e31b9", @@ -917,6 +936,7 @@ "packages/session-ui/src/v2/components/prompt-input/attachments.ts": "ff66078f2edc9a67a4df0d7dfd76a49be5eab27ac332b303f3412721ef7268ea", "packages/session-ui/src/v2/components/prompt-input/index.tsx": "194b1c6d0c99000579df78b47c7ff72fe34f62a92ecd21b2a9377133d6d27bb9", "packages/opencode/src/server/routes/instance/httpapi/server.ts": "a94becd6a5224fb5fc2778c1091733eca334b308253234948bcdcafdc709b021", + "packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts": "9da41a900811e8526f3d6796341b787f7e3c64bdc9cc3a547977f05780c1f79f", "packages/opencode/src/server/routes/instance/httpapi/groups/experimental.ts": "944a0aee215a917e867fd9482f3c2bd40f49082bd3174c8a7d967ca40e34ec15", "packages/opencode/src/server/routes/instance/httpapi/groups/file.ts": "a96fdc55ddb95ca7c481c86f67e37ed88d4cf2a190a482765ab55ce4409f3a59", "packages/opencode/src/server/routes/instance/httpapi/groups/mcp.ts": "048efa7b74d43c4ad4a75d0f8421c808760c8c0dcd593f0051a87044e40487d6", @@ -924,6 +944,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "d0b4e3431b20d52a1e94fc7742cf073a0ca74398cdb7150a49a07bd32d0a563b", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", @@ -942,11 +963,11 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T14:32:31.142Z", + "extracted_at": "2026-10-13T01:45:32.169Z", "per_package": { "packages/app": { - "A": 237, - "M": 158, + "A": 240, + "M": 161, "D": 0 }, "packages/core": { @@ -960,8 +981,8 @@ "D": 0 }, "packages/opencode": { - "A": 75, - "M": 89, + "A": 82, + "M": 93, "D": 0 }, "packages/schema": { @@ -971,7 +992,7 @@ }, "packages/sdk": { "A": 0, - "M": 3, + "M": 4, "D": 0 }, "packages/server": { @@ -980,8 +1001,8 @@ "D": 0 }, "packages/session-ui": { - "A": 42, - "M": 31, + "A": 44, + "M": 32, "D": 0 }, "packages/ui": { @@ -991,7 +1012,7 @@ } }, "counts": { - "overlay_total": 920, + "overlay_total": 941, "deletions": 7, "server_coupled": 0 }, @@ -1192,6 +1213,9 @@ "packages/app/src/context/sse-gap-frame.ts": "A", "packages/app/src/context/sse-origin.test.ts": "A", "packages/app/src/context/sse-origin.ts": "A", + "packages/app/src/context/sse-reader.test.ts": "A", + "packages/app/src/context/stream-cursor.test.ts": "A", + "packages/app/src/context/stream-cursor.ts": "A", "packages/app/src/context/stream-gap.test.ts": "A", "packages/app/src/context/stream-gap.ts": "A", "packages/app/src/context/stream-liveness.test.ts": "A", @@ -1221,6 +1245,7 @@ "packages/app/src/i18n/uk.ts": "M", "packages/app/src/i18n/zh.ts": "M", "packages/app/src/i18n/zht.ts": "M", + "packages/app/src/pages/directory-layout.tsx": "M", "packages/app/src/pages/empty-workspace-landing.tsx": "A", "packages/app/src/pages/error.tsx": "M", "packages/app/src/pages/hold-debug-badge-gate.test.ts": "A", @@ -1253,6 +1278,8 @@ "packages/app/src/utils/amicode-workspace-projects.test.ts": "A", "packages/app/src/utils/amicode-workspace-projects.ts": "A", "packages/app/src/utils/chrome-dropdown.ts": "A", + "packages/app/src/utils/diffs.test.ts": "M", + "packages/app/src/utils/diffs.ts": "M", "packages/app/src/utils/global-clipboard.test.ts": "A", "packages/app/src/utils/global-clipboard.ts": "A", "packages/app/src/utils/pane-bridge.ts": "A", @@ -1304,6 +1331,9 @@ "packages/llm/src/protocols/bedrock-converse.ts": "M", "packages/llm/test/provider/bedrock-converse.test.ts": "M", "packages/opencode/src/agent/agent.ts": "M", + "packages/opencode/src/bus/global-replay.ts": "A", + "packages/opencode/src/bus/global.ts": "M", + "packages/opencode/src/control-plane/workspace.ts": "M", "packages/opencode/src/effect/runner.ts": "M", "packages/opencode/src/mcp/browser.ts": "M", "packages/opencode/src/mcp/index.ts": "M", @@ -1341,6 +1371,8 @@ "packages/opencode/src/tool/edit.ts": "M", "packages/opencode/src/tool/question.txt": "M", "packages/opencode/src/tool/registry.ts": "M", + "packages/opencode/src/tool/shell-output-throttle.ts": "A", + "packages/opencode/src/tool/shell.ts": "M", "packages/opencode/src/tool/task.ts": "M", "packages/opencode/src/tool/truncate.ts": "M", "packages/opencode/src/tool/widget-author.ts": "A", @@ -1348,6 +1380,7 @@ "packages/opencode/src/tool/write.ts": "M", "packages/opencode/src/worktree/index.ts": "M", "packages/opencode/test/acp/service-session.test.ts": "M", + "packages/opencode/test/control-plane/workspace-sync-optin.test.ts": "A", "packages/opencode/test/effect/runner-queue.test.ts": "A", "packages/opencode/test/effect/runner.test.ts": "M", "packages/opencode/test/fixture/fixture.ts": "M", @@ -1375,9 +1408,11 @@ "packages/opencode/test/server/amicode-widgets-src.test.ts": "A", "packages/opencode/test/server/amicode-widgets.test.ts": "A", "packages/opencode/test/server/auth-accept-set.test.ts": "A", + "packages/opencode/test/server/global-event-replay.test.ts": "A", "packages/opencode/test/server/httpapi-compression.test.ts": "M", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "A", "packages/opencode/test/server/httpapi-file.test.ts": "M", + "packages/opencode/test/server/httpapi-global-event.test.ts": "A", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "M", "packages/opencode/test/server/httpapi-provider.test.ts": "M", "packages/opencode/test/server/httpapi-session.test.ts": "M", @@ -1386,6 +1421,7 @@ "packages/opencode/test/server/session-compact-errors.test.ts": "A", "packages/opencode/test/server/session-diff-missing-patch.test.ts": "M", "packages/opencode/test/server/session-diff-scoped.test.ts": "A", + "packages/opencode/test/server/summary-diff-wire.test.ts": "A", "packages/opencode/test/server/workspace-routing.test.ts": "M", "packages/opencode/test/session/adversarial-e2e.test.ts": "A", "packages/opencode/test/session/business-record.test.ts": "A", @@ -1406,6 +1442,7 @@ "packages/opencode/test/tool/edit.test.ts": "M", "packages/opencode/test/tool/question.test.ts": "M", "packages/opencode/test/tool/registry-schema.test.ts": "A", + "packages/opencode/test/tool/shell-output-throttle.test.ts": "A", "packages/opencode/test/tool/task.test.ts": "M", "packages/opencode/test/tool/truncation.test.ts": "M", "packages/opencode/test/tool/write.test.ts": "M", @@ -1442,10 +1479,13 @@ "packages/session-ui/src/components/session-review.tsx": "M", "packages/session-ui/src/components/session-turn.css": "M", "packages/session-ui/src/components/session-turn.tsx": "M", + "packages/session-ui/src/components/summary-patch.test.ts": "A", + "packages/session-ui/src/components/summary-patch.ts": "A", "packages/session-ui/src/components/tool-count-label.test.ts": "A", "packages/session-ui/src/components/tool-count-label.tsx": "M", "packages/session-ui/src/components/tool-error-card.tsx": "M", "packages/session-ui/src/components/tool-status-title.tsx": "M", + "packages/session-ui/src/context/data.tsx": "M", "packages/session-ui/src/pierre/worker.ts": "M", "packages/session-ui/src/util/clipboard.ts": "A", "packages/ui/src/amicode/amico-presence.css": "A", @@ -1905,6 +1945,7 @@ "packages/session-ui/src/v2/components/prompt-input/attachments.ts": "M", "packages/session-ui/src/v2/components/prompt-input/index.tsx": "M", "packages/opencode/src/server/routes/instance/httpapi/server.ts": "M", + "packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/groups/experimental.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/groups/file.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/groups/mcp.ts": "M", @@ -1912,6 +1953,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "M", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "M", diff --git a/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx b/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx index 65559c790..bb89bdf59 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx +++ b/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx @@ -14,8 +14,8 @@ import { createRefCountMap } from "@/utils/refcount" import { useGlobal } from "./global" import { ServerScope } from "@/utils/server-scope" import { attachOwnerHeaderIfArmed } from "@/components/remote-create-arm" -import { parseCursorNamespaces, resolveEventOrigin } from "./sse-origin" -import { shouldReconnectIdleStream } from "./stream-liveness" +import { createStreamEndResync, shouldReconnectIdleStream } from "./stream-liveness" +import { applyStreamCursor, createStreamCursor } from "./stream-cursor" import { detectServerProtocol, type ServerProtocol } from "@/utils/server-protocol" import { createCompatibleApi, type CompatibleApi } from "@/utils/server-compat" @@ -357,55 +357,43 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS } })() - // #1264 lossless reconnect: the frontdoor buffers recent events and - // replays the gap when the stream is re-opened with ?lastEventID=. - // Track the last payload id seen (the server ids every event; id-less - // events just don't advance the cursor — replay may over-deliver, which - // the reconcile-based reducers tolerate by design). + // #1264 lossless reconnect: the engine (#1776) sequences its global stream + // and replays the gap when the stream is re-opened with ?lastEventID=. + // #1777: the cursor is the SSE `id:` of the last applied frame — the engine's + // `:`, or the fan-in's composite of every origin's position — + // never the JSON payload id (the engine rejects that as malformed, so every + // reconnect used to be a reset). It also checks contiguity per origin: a + // hole, an engine reset, or an unannounced restart forces a resync, and a + // replayed frame already applied is dropped (stream-cursor.ts). // #1264: the cursor must survive RELOADS — module state died with every // document, so a drop right after a reload reconnected cursor-less and // the gap's events were lost (the harness caught it: the sse-gap flow // failed only in the post-reload context). sessionStorage is per-tab, // cheap, and exactly the lifetime the cursor wants. const CURSOR_KEY = "amicode.sse.lastEventID" - let lastEventID: string | undefined + let storedCursor: string | undefined try { - lastEventID = sessionStorage.getItem(CURSOR_KEY) ?? undefined + storedCursor = sessionStorage.getItem(CURSOR_KEY) ?? undefined } catch { /* private mode etc. — degrade to module state */ } - const trackEventID = (payload: unknown) => { - const id = (payload as { id?: unknown } | undefined)?.id - if (typeof id === "string" && id) { - lastEventID = id - try { - sessionStorage.setItem(CURSOR_KEY, id) - } catch { - /* best-effort */ - } + const cursor = createStreamCursor(storedCursor) + const persistCursor = () => { + const current = cursor.current() + if (current === undefined || current === storedCursor) return + storedCursor = current + try { + sessionStorage.setItem(CURSOR_KEY, current) + } catch { + /* best-effort */ } } - // #1539: track the parsed composite cursor to detect event origin (local vs - // remote peer). Initialized from the persisted cursor so the first event - // after reconnect diffs correctly. - let previousParsedCursor = parseCursorNamespaces(lastEventID) const sseFetch: FetchLike = (input, init) => { const base = eventFetch ?? globalThis.fetch - try { - if (lastEventID) { - const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url - const u = new URL(url, server.http.url) - if (u.pathname === "/event" || u.pathname === "/global/event") { - u.searchParams.set("lastEventID", lastEventID) - const modified = u.toString() - if (typeof input === "string" || input instanceof URL) return base(modified, init) - return base(new Request(modified, input), init) - } - } - } catch { - /* fall through unmodified */ - } - return base(input as Parameters[0], init as Parameters[1]) + // #1777: the ONE carrier is `?lastEventID=` (the SDK's own Last-Event-ID + // header is stripped so two cursors can never disagree). + const applied = applyStreamCursor(input, init, server.http.url, cursor.current()) + return base(applied.input as Parameters[0], applied.init as Parameters[1]) } const eventApi = createApiForServer({ server: server.http, fetch: sseFetch as typeof fetch }) const eventSdk = createSdkForServer({ @@ -475,6 +463,28 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS // nominally connected, so the disconnect→connect edge never fires and the rail // would otherwise never self-heal a stranded view). const [resyncCount, setResyncCount] = createSignal(0) + // #1617 / #1777: a FULL resync of the viewed state — the forced-resync signal + // (the transcript and the entity rail refetch, #1646) plus a synthetic GLOBAL + // `amicode.sync.gap` event, on which server-sync refetches the bootstrap + // unconditionally (past the #1289 `server.connected` debounce, which keys on + // a different type). Driven by a relay's gap frame, and by the cursor's + // verdicts: an engine reset, a sequence hole, an unannounced restart. + const forceResync = (namespaces: string[]) => { + setResyncCount((n) => n + 1) + if ( + enqueueServerEvent(queue, { + directory: "global", + payload: { type: SYNC_GAP_TYPE, properties: { namespaces } } as unknown as ServerEvent, + }) + ) + schedule() + } + // #1773: an unexpected NORMAL end of the global stream is a loss (the relays + // never end a listener cleanly except on shutdown). The loop arms this latch + // on such an end; the reconnected stream's first frame advances resyncCount, + // so the viewed transcript and the entity rail refetch once the stream is live + // again. A user stop, a page hide, or the client's own abort arm nothing. + const streamEndResync = createStreamEndResync() // #1584 client-side liveness watchdog. The v1 event iterator neither throws // nor completes when the socket goes silently half-open (sleep/wake, Wi-Fi @@ -545,9 +555,21 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS error, }) } + // #1777: the SSE id of the frame about to be yielded. The SDK reader + // calls onSseEvent with each parsed frame right before it yields that + // frame's data, so this is the id of the event the loop reads next. + let frameSseId: string | undefined const events = kind === "v1" - ? (await eventSdk.global.event({ signal: currentAttempt.signal, onSseError })).stream + ? ( + await eventSdk.global.event({ + signal: currentAttempt.signal, + onSseError, + onSseEvent: (frame: { id?: string }) => { + frameSseId = frame.id + }, + }) + ).stream : eventApi.event.subscribe({ signal: currentAttempt.signal }) if (!streamGeneration.accepts(active)) return setStreamStatus("connected") @@ -560,8 +582,11 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS if (!streamGeneration.accepts(active)) return streamErrorLogged = false lastFrameAt = Date.now() + // #1773: the first frame after an unexpected end — the stream is live + // again, so force the resync now (the refetch cannot miss the events + // between the end and this re-subscription). + if (streamEndResync.frame()) setResyncCount((n) => n + 1) const legacy = "payload" in event - if (legacy && event.payload.type === "sync") continue const directory = legacy ? (event.directory ?? "global") : (event.location?.directory ?? "global") const payload = legacy ? (event.payload as Event) : adaptServerEvent(event) // #1522 (ADR 0033 decision A): the SSE fan-in aggregator emits @@ -602,26 +627,31 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS ? (payload as Record) : ((payload as { properties?: Record }).properties ?? {}) const gap = parseSyncGapFrame({ type: SYNC_GAP_TYPE, ...gapData }) + // #1777: the named origins resync, so their next positions are new + // baselines (not a second "hole" against the lost stretch). + cursor.gap(gap?.namespaces ?? []) // Bump the forced-resync signal so the entity rail self-heals even // though the socket never crossed a disconnect→connect edge. - setResyncCount((n) => n + 1) - if (enqueueServerEvent(queue, { - directory: "global", - payload: { type: SYNC_GAP_TYPE, properties: { namespaces: gap?.namespaces ?? [] } } as unknown as ServerEvent, - })) - schedule() + forceResync(gap?.namespaces ?? []) continue } - trackEventID(legacy ? event.payload : (event as { id?: string })) - // #1539: resolve origin from the composite cursor diff. Fleet-of-one - // (bare scalar id) returns undefined immediately — zero overhead. - const sseId: string | undefined = legacy - ? (event.payload as { id?: string }).id - : (event as { id?: string }).id - const resolved = resolveEventOrigin(sseId, previousParsedCursor) - previousParsedCursor = resolved.cursor - const taggedPayload: ServerEvent = resolved.origin !== undefined - ? { ...payload, origin: resolved.origin } + // #1777: the cursor and the contiguity check, on the frame's SSE id + // (v1; the v2 stream at /api/event carries no resume cursor). A hole, + // a reset or an unannounced engine restart owes a full resync; a + // replayed frame already applied is dropped, never applied twice. + const verdict = cursor.frame({ + id: kind === "v1" ? frameSseId : undefined, + type: legacy ? event.payload.type : payload.type, + }) + if (verdict.resync) forceResync(verdict.resync.namespaces) + if (!verdict.apply) continue + persistCursor() + // a sync copy still moves the cursor (it holds a seq), but is no event + if (legacy && event.payload.type === "sync") continue + // #1539: the origin is the composite cursor's moving namespace. + // Fleet-of-one (a bare engine id) is always local. + const taggedPayload: ServerEvent = verdict.origin !== undefined + ? { ...payload, origin: verdict.origin } : payload if (enqueueServerEvent(queue, { directory, payload: taggedPayload })) schedule() @@ -629,6 +659,13 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS yielded = Date.now() await wait(0) } + // #1773: the iterator COMPLETED — a clean end of the global stream. If + // the client neither aborted this attempt nor stopped the loop, it is + // an unexpected end: a loss. Arm the resync for the reconnected stream. + streamEndResync.ended({ + attemptAborted: currentAttempt.signal.aborted, + stopped: abort.signal.aborted || !started || !streamGeneration.accepts(active), + }) } catch (error) { if (streamGeneration.accepts(active) && !isStreamClosed(error, currentAttempt.signal)) setStreamStatus("disconnected") @@ -658,8 +695,8 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS } const clearGlobalStreamCursor = () => { - lastEventID = undefined - previousParsedCursor = {} + cursor.clear() + storedCursor = undefined try { sessionStorage.removeItem(CURSOR_KEY) } catch { diff --git a/packages/app-bundle/overlay/packages/app/src/context/sse-gap-frame.test.ts b/packages/app-bundle/overlay/packages/app/src/context/sse-gap-frame.test.ts index b4ea7025c..f7b7e585c 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/sse-gap-frame.test.ts +++ b/packages/app-bundle/overlay/packages/app/src/context/sse-gap-frame.test.ts @@ -60,3 +60,37 @@ describe("#1617 — gap interception is wired at both client seams", () => { expect(syncSource).toMatch(/server\.connected"\s*&&\s*Date\.now\(\)\s*-\s*lastConnectedQueueAt/) }) }) + +// #1773 — an unexpected NORMAL end of the global stream is a gap too (defense in +// depth: kept even once replay exists). The relay chain now error-closes every +// loss, but a clean end the client did not ask for still means events may have +// been dropped. The decision (`createStreamEndResync`, stream-liveness.ts) is +// unit-tested there; these pins assert the loop actually wires it — falsifiable: +// deleting either call, or reporting the end without the abort/stop state, reds. +describe("#1773 — an unexpected end of the global stream advances the forced-resync signal", () => { + const sdkSource = readFileSync(join(import.meta.dir, "server-sdk.tsx"), "utf8") + const loop = sdkSource.indexOf("for await (const event of events)") + const loopCatch = sdkSource.indexOf("} catch (error) {", loop) + + test("the loop's NORMAL completion (the iterator ended without throwing) reports the end", () => { + expect(loop).toBeGreaterThan(0) + const body = sdkSource.slice(loop, loopCatch) + // after the for-await body's last statement and before the catch: the normal-end path + const afterLoop = body.slice(body.lastIndexOf("await wait(0)")) + expect(afterLoop).toContain("streamEndResync.ended(") + }) + + test("a user stop / page hide / the client's own abort are NOT a loss: the end carries both states", () => { + const body = sdkSource.slice(loop, loopCatch) + expect(body).toMatch(/attemptAborted:\s*currentAttempt\.signal\.aborted/) + // stopped = the root abort (teardown), the started latch (stop()/pagehide), the generation fence (reset) + expect(body).toMatch(/stopped:\s*abort\.signal\.aborted\s*\|\|\s*!started\s*\|\|\s*!streamGeneration\.accepts\(active\)/) + }) + + test("the reconnected stream's first frame advances the forced-resync signal (the transcript + entity rail refetch)", () => { + const body = sdkSource.slice(loop, loopCatch) + expect(body).toMatch(/if \(streamEndResync\.frame\(\)\) setResyncCount\(\(n\) => n \+ 1\)/) + // the signal the transcript (pages/session.tsx) and the entity rail read + expect(sdkSource).toMatch(/resyncCount,\s*\n/) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts new file mode 100644 index 000000000..72113cd37 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts @@ -0,0 +1,263 @@ +import { appendFileSync } from "node:fs" +import { describe, expect, test } from "bun:test" +import { createSdkForServer } from "@/utils/server" + +// #1779 — the app's global-stream READER: the SDK's SSE reader +// (`@opencode-ai/sdk/v2` core/serverSentEvents.gen.ts) that server-sdk.tsx reads +// the global stream through, driven exactly as the app drives it — +// `createSdkForServer(...).global.event({ onSseEvent })` — with a fetch that +// answers with chosen byte chunks, so a test can place every chunk boundary. +// (Each body chunk is one read: TextDecoderStream decodes chunk by chunk and +// holds back a code point split across two.) + +const SERVER = { url: "http://127.0.0.1:4096" } + +/** What one parsed frame reports through `onSseEvent` (the app reads `id`). */ +type Frame = { data: unknown; event?: string; id?: string; retry?: number } + +/** One response body: its chunks, one body read each. With `failWhen`, the body + * errors (a dropped link) once that promise settles — never earlier, so no + * decoded chunk still queued in the pipe is discarded by the error. */ +type Delivery = { chunks: Uint8Array[]; failWhen?: Promise } + +/** A fetch that answers the n-th request with the n-th delivery. Every request + * is recorded. */ +function streamFetch(deliveries: Delivery[], requests: Request[] = []) { + return async (input: RequestInfo | URL, init?: RequestInit) => { + const request = input instanceof Request ? input : new Request(input, init) + const delivery = deliveries[Math.min(requests.length, deliveries.length - 1)] + requests.push(request) + let next = 0 + const body = new ReadableStream({ + pull(controller) { + if (next < delivery.chunks.length) controller.enqueue(delivery.chunks[next++]) + else if (delivery.failWhen) return delivery.failWhen.then(() => controller.error(new Error("link dropped"))) + else controller.close() + }, + }) + return new Response(body, { headers: { "content-type": "text/event-stream" } }) + } +} + +/** Read the global stream to its end: every frame `onSseEvent` reported, and + * every event the stream yielded, in order. `retryDelayMs` sets the SDK's + * reconnect delay (its default is 3000 ms, reported as each frame's `retry`). */ +async function readStream( + deliveries: Delivery[], + requests: Request[] = [], + opts: { onFrame?: (frame: Frame) => void; retryDelayMs?: number } = {}, +): Promise<{ frames: Frame[]; events: unknown[] }> { + const sdk = createSdkForServer({ server: SERVER, fetch: streamFetch(deliveries, requests) as typeof fetch }) + const frames: Frame[] = [] + const result = await sdk.global.event({ + onSseEvent: (frame: Frame) => { + frames.push({ ...frame }) + opts.onFrame?.(frame) + }, + ...(opts.retryDelayMs !== undefined ? { sseDefaultRetryDelay: opts.retryDelayMs } : {}), + }) + const events: unknown[] = [] + for await (const event of result.stream) events.push(event) + return { frames, events } +} + +const encoder = new TextEncoder() +const bytes = (text: string) => encoder.encode(text) + +// ── #1771's chunked large-frame fixture ───────────────────────────────────── +// Ported verbatim from the extension's engine-shaped fixtures +// (test/fixtures/engine_stream/engine_frames.ts `largeSummaryFrame` / +// `chunkBytes`) — the app suite cannot import them: a `message.updated` summary +// frame whose `info.summary.diffs[].patch` text brings it to ≥ 10 MB of UTF-8, +// with non-ASCII text so a 64 KB byte boundary can fall inside a code point. + +const TEN_MB = 10 * 1024 * 1024 +const CHUNK_64K = 64 * 1024 +const EPOCH = "bG9jYWwx" + +function largeSummaryFrame(seq: number, targetBytes = TEN_MB): { frame: string; event: unknown } { + const lines: string[] = [] + for (let i = 0; i < 1400; i++) lines.push(`+ const row${i} = "résumé → ${"x".repeat(60)}";`) + const patch = `@@ -1,0 +1,1400 @@\n${lines.join("\n")}\n` + const perDiff = encoder.encode(JSON.stringify(patch)).length + 96 + const count = Math.max(1, Math.ceil(targetBytes / perDiff)) + const diffs = Array.from({ length: count }, (_, i) => ({ + file: `src/module_${i}.ts`, + patch, + additions: 1400, + deletions: 0, + status: "modified", + })) + const event = { + directory: "/Users/jj/harmoniqs/amicode", + project: "prj_0fixture", + payload: { + id: "evt_summary", + type: "message.updated", + properties: { info: { id: "msg_summary", sessionID: "ses_large", role: "user", summary: { diffs } } }, + }, + } + return { frame: `id: ${EPOCH}:${seq}\ndata: ${JSON.stringify(event)}\n\n`, event } +} + +/** Split text's UTF-8 BYTES into `size` pieces — how the network delivers it. */ +function chunkBytes(text: string, size = CHUNK_64K): Uint8Array[] { + const all = encoder.encode(text) + const out: Uint8Array[] = [] + for (let off = 0; off < all.length; off += size) out.push(all.subarray(off, off + size)) + return out +} + +const median = (xs: number[]) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)] + +/** #1779 AC1's assertion, made robust: a warm-up, then INTERLEAVED trials (both + * deliveries see the same machine noise), the median of each, and a generous + * factor. A quadratic reader misses it by an order of magnitude. + * `SPLIT_RATIO_LOG` (a file path) records each run's ratio for a flake report. */ +async function readTimeRatio(whole: Uint8Array[], chunked: Uint8Array[]) { + const timed = async (chunks: Uint8Array[]) => { + const t0 = performance.now() + await readStream([{ chunks }]) + return performance.now() - t0 + } + for (let i = 0; i < 2; i++) { + await timed(whole) + await timed(chunked) + } + const one: number[] = [] + const many: number[] = [] + for (let i = 0; i < 7; i++) { + one.push(await timed(whole)) + many.push(await timed(chunked)) + } + const wholeMs = median(one) + const chunkedMs = median(many) + return { ratio: chunkedMs / wholeMs, wholeMs, chunkedMs } +} +const SPLIT_RATIO_FACTOR = 3 + +describe("#1779 AC1 — the app reader splits a 10 MB frame in 64 KB chunks within a small constant factor of one chunk", () => { + test( + "chunked ÷ single-chunk read time (medians) stays ≤ 3×, and both deliver the one event with its SSE id", + async () => { + const big = largeSummaryFrame(1) + const whole = [bytes(big.frame)] + const chunked = chunkBytes(big.frame) + expect(whole[0].length).toBeGreaterThanOrEqual(TEN_MB) + expect(chunked.length).toBeGreaterThan(160) + for (const chunks of [whole, chunked]) { + const { frames, events } = await readStream([{ chunks }]) + expect(frames.map((f) => f.id)).toEqual([`${EPOCH}:1`]) // #1777's cursor source + expect(events).toHaveLength(1) + expect(Bun.deepEquals(events[0], big.event)).toBe(true) // no 10 MB diff on failure + } + const { ratio, wholeMs, chunkedMs } = await readTimeRatio(whole, chunked) + if (process.env.SPLIT_RATIO_LOG) + appendFileSync(process.env.SPLIT_RATIO_LOG, `app ${ratio.toFixed(3)} ${wholeMs.toFixed(2)} ${chunkedMs.toFixed(2)}\n`) + expect(ratio).toBeLessThanOrEqual(SPLIT_RATIO_FACTOR) + }, + 120_000, + ) +}) + +/** Every way to cut `text`'s bytes into two chunks, and one byte at a time. */ +function everySplit(text: string): Uint8Array[][] { + const all = bytes(text) + const deliveries: Uint8Array[][] = [] + for (let k = 1; k < all.length; k++) deliveries.push([all.subarray(0, k), all.subarray(k)]) + deliveries.push(Array.from({ length: all.length }, (_, i) => all.subarray(i, i + 1))) + return deliveries +} + +describe("#1779 AC2 — the app reader finds frame boundaries split across chunks, CRLF line endings included", () => { + const stream = [ + `id: ${EPOCH}:0\ndata: {"payload":{"type":"server.connected"}}\n\n`, // LF, the engine's framing + `id: ${EPOCH}:1\r\ndata: {"n":1,"text":"résumé → done"}\r\n\r\n`, // CRLF, non-ASCII + `data: first line\r\ndata: second line\r\n\r\n`, // CRLF, multi-line data + `event: ping\rdata: {"n":3}\r\r`, // CR line endings + `data: {"n":4}\n\n`, + ].join("") + const expected: Frame[] = [ + { data: { payload: { type: "server.connected" } }, event: undefined, id: `${EPOCH}:0`, retry: 3000 }, + { data: { n: 1, text: "résumé → done" }, event: undefined, id: `${EPOCH}:1`, retry: 3000 }, + { data: "first line\nsecond line", event: undefined, id: `${EPOCH}:1`, retry: 3000 }, + { data: { n: 3 }, event: "ping", id: `${EPOCH}:1`, retry: 3000 }, + { data: { n: 4 }, event: undefined, id: `${EPOCH}:1`, retry: 3000 }, + ] + + test("in one chunk: each frame once, with its id", async () => { + const { frames, events } = await readStream([{ chunks: [bytes(stream)] }]) + expect(frames).toEqual(expected) + expect(events).toEqual(expected.map((f) => f.data)) + }) + + test("cut at EVERY byte — a delimiter, a CRLF, a code point split between two chunks — and byte by byte: the same frames, no extra, none merged", async () => { + for (const chunks of everySplit(stream)) { + const { frames, events } = await readStream([{ chunks }]) + const cuts = chunks.map((c) => c.length) + expect({ cuts, frames }).toEqual({ cuts, frames: expected }) + expect({ cuts, events }).toEqual({ cuts, events: expected.map((f) => f.data) }) + } + }) + + test("a stream that ENDS on a CR line ending still delivers its last frame (a held \\r is a line end at the end)", async () => { + const crOnly = `data: {"n":1}\r\rdata: {"n":2}\r\r` + for (const chunks of [[bytes(crOnly)], ...everySplit(crOnly)]) { + const { events } = await readStream([{ chunks }]) + expect({ cuts: chunks.map((c) => c.length), events }).toEqual({ cuts: chunks.map((c) => c.length), events: [{ n: 1 }, { n: 2 }] }) + } + }) +}) + +describe("#1779 AC3 — the app reader's framing is unchanged: multi-line data, event names, ids, retry fields", () => { + // Pinned against the generated reader as it was before #1779, in one chunk and + // cut at every byte (the parse of each frame must not depend on where it was cut). + const stream = [ + `: a comment line is ignored\nevent: message.updated\nid: ${EPOCH}:1\ndata: {"a":\ndata: 1}\n\n`, // multi-line data joins with \n + `data:no-space\nretry: 5000\n\n`, // a field without a space; a retry sets the delay + `data: padded\nretry: soon\n\n`, // the reader strips all leading whitespace; a bad retry is ignored + `: keep-alive\n\n`, // no data: reported, never yielded + `id: ${EPOCH}:2\nevent: x\nevent: y\ndata: {"b":2}\n\n`, // last event name wins + `data: {"c":3}\n\n`, // no id line: the last id stands (SSE last-event-ID) + `\ndata: {"d":4}\n\n`, // a stray blank line before a frame joins that frame (no empty dispatch) + `id:\ndata: {"e":5}\n\n`, // an empty id resets it + `data: {"lost":true}\n`, // incomplete at the end of the stream: discarded + ].join("") + const expected: Frame[] = [ + { data: { a: 1 }, event: "message.updated", id: `${EPOCH}:1`, retry: 3000 }, + { data: "no-space", event: undefined, id: `${EPOCH}:1`, retry: 5000 }, + { data: "padded", event: undefined, id: `${EPOCH}:1`, retry: 5000 }, + { data: undefined, event: undefined, id: `${EPOCH}:1`, retry: 5000 }, + { data: { b: 2 }, event: "y", id: `${EPOCH}:2`, retry: 5000 }, + { data: { c: 3 }, event: undefined, id: `${EPOCH}:2`, retry: 5000 }, + { data: { d: 4 }, event: undefined, id: `${EPOCH}:2`, retry: 5000 }, + { data: { e: 5 }, event: undefined, id: "", retry: 5000 }, + ] + const yielded = [{ a: 1 }, "no-space", "padded", { b: 2 }, { c: 3 }, { d: 4 }, { e: 5 }] + + test("in one chunk, and cut at every byte: the same frames, fields and events", async () => { + for (const chunks of [[bytes(stream)], ...everySplit(stream)]) { + const { frames, events } = await readStream([{ chunks }]) + const cuts = chunks.map((c) => c.length) + expect({ cuts, frames }).toEqual({ cuts, frames: expected }) + expect({ cuts, events }).toEqual({ cuts, events: yielded }) + } + }) + + test("ids: after a dropped link the SDK's own retry carries the last id it parsed (the Last-Event-ID header #1777's sseFetch strips)", async () => { + const requests: Request[] = [] + const first = `id: ${EPOCH}:6\ndata: {"n":6}\n\nid: ${EPOCH}:7\ndata: {"n":7}\n\n` + let parsedLast!: () => void + const dropLink = new Promise((resolve) => (parsedLast = resolve)) + const { frames, events } = await readStream( + [{ chunks: chunkBytes(first, 7), failWhen: dropLink }, { chunks: [bytes(`id: ${EPOCH}:8\ndata: {"n":8}\n\n`)] }], + requests, + { onFrame: (frame) => frame.id === `${EPOCH}:7` && parsedLast(), retryDelayMs: 0 }, + ) + expect(requests).toHaveLength(2) + expect(requests[0].headers.get("last-event-id")).toBeNull() + expect(requests[1].headers.get("last-event-id")).toBe(`${EPOCH}:7`) + expect(frames.map((f) => f.id)).toEqual([`${EPOCH}:6`, `${EPOCH}:7`, `${EPOCH}:8`]) + expect(events).toEqual([{ n: 6 }, { n: 7 }, { n: 8 }]) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.test.ts b/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.test.ts new file mode 100644 index 000000000..ddb19fa4b --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, test } from "bun:test" +import { applyStreamCursor, createStreamCursor } from "./stream-cursor" + +// #1777 — the client's resume cursor and its contiguity check, extracted as a +// ZERO-DEPENDENCY pure module (like stream-liveness.ts) so it runs in place. +// server-sdk.tsx's global stream loop feeds it every data frame's SSE `id:` (the +// SDK reader's `onSseEvent`) and its payload type, and acts on the verdict. +// +// Wire facts (#1776, #1511): an engine stamps every global-stream frame +// `id: :` — bus events take strictly increasing seqs, and the +// per-connection control frames (server.connected / server.heartbeat / +// server.cursor.unservable) carry the CURRENT seq without consuming one. A +// fleet-of-one fan-in passes those ids through untouched; a fanned-in stream's +// id is the composite `local=;=` of every origin's position. + +const E = "bG9jYWwx" // the local engine's epoch +const P = "c3R1ZGlv" // a peer engine's epoch +const id = (epoch: string, seq: number) => `${epoch}:${seq}` +const composite = (local: number, studio: number) => `local=${id(E, local)};studio=${id(P, studio)}` +const event = (frameId: string | undefined, type = "message.part.updated") => ({ id: frameId, type }) + +describe("#1777 AC4 — the cursor is the SSE id, and it survives a reconnect and a reload", () => { + test("fleet-of-one: the cursor is the engine's bare id, verbatim", () => { + const cursor = createStreamCursor() + expect(cursor.current()).toBeUndefined() + cursor.frame(event(id(E, 0), "server.connected")) + cursor.frame(event(id(E, 1))) + expect(cursor.current()).toBe(id(E, 1)) + }) + + test("fanned in: the cursor is the composite id, verbatim", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(1, 7))) + cursor.frame(event(composite(2, 7))) + expect(cursor.current()).toBe(composite(2, 7)) + }) + + test("never the JSON payload id: a frame without an SSE id leaves the cursor where it was", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 4))) + expect(cursor.frame(event(undefined))).toEqual({ apply: true, origin: undefined }) + expect(cursor.current()).toBe(id(E, 4)) + }) + + test("after a reload: the stored cursor is resumed from, and frames are checked against it", () => { + const cursor = createStreamCursor(id(E, 9)) + expect(cursor.current()).toBe(id(E, 9)) + expect(cursor.frame(event(id(E, 9), "server.connected")).resync).toBeUndefined() // the engine resumed at it + expect(cursor.frame(event(id(E, 11))).resync).toEqual({ reason: "hole", namespaces: ["local"] }) // 10 never came + }) + + test("the ONE carrier: `?lastEventID=` on the global stream, and never a conflicting Last-Event-ID header", () => { + const base = "http://127.0.0.1:4096" + const req = new Request(`${base}/global/event`, { headers: { "Last-Event-ID": "stale:1", authorization: "Basic x" } }) + const out = applyStreamCursor(req, undefined, base, composite(3, 8)) + const sent = out.input as Request + expect(new URL(sent.url).searchParams.get("lastEventID")).toBe(composite(3, 8)) + expect(sent.headers.get("last-event-id")).toBeNull() + expect(sent.headers.get("authorization")).toBe("Basic x") + + // a string input, a stale header in init: the same single carrier + const str = applyStreamCursor(`${base}/global/event?lastEventID=old`, { headers: { "Last-Event-ID": "old" } }, base, id(E, 2)) + expect(new URL(str.input as string).searchParams.get("lastEventID")).toBe(id(E, 2)) + expect(new Headers(str.init?.headers).get("last-event-id")).toBeNull() + + // no cursor (a fresh or reset stream): no cursor at all + const fresh = applyStreamCursor(`${base}/global/event?lastEventID=old`, undefined, base, undefined) + expect(new URL(fresh.input as string).searchParams.has("lastEventID")).toBe(false) + + // any other request is untouched + const other = `${base}/session` + expect(applyStreamCursor(other, undefined, base, id(E, 2)).input).toBe(other) + }) +}) + +describe("#1777 AC4 — a sequence hole within one origin triggers a resync of that origin", () => { + test("fleet-of-one: a bus event that skips a seq is a hole — applied, and a resync is owed", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 1))) + expect(cursor.frame(event(id(E, 3)))).toEqual({ apply: true, origin: undefined, resync: { reason: "hole", namespaces: ["local"] } }) + expect(cursor.frame(event(id(E, 4))).resync).toBeUndefined() // contiguous again + }) + + test("an idle stream: a heartbeat carrying a seq the client never saw is a hole", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 5))) + expect(cursor.frame(event(id(E, 5), "server.heartbeat")).resync).toBeUndefined() // at the position: fine + expect(cursor.frame(event(id(E, 6), "server.heartbeat")).resync).toEqual({ reason: "hole", namespaces: ["local"] }) + }) + + test("fanned in: contiguity is per origin — a hole in the peer's seq resyncs the peer, the local arm's frames never count against it", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(5, 42))) + expect(cursor.frame(event(composite(6, 42))).resync).toBeUndefined() // local advanced by one + expect(cursor.frame(event(composite(7, 42))).resync).toBeUndefined() + expect(cursor.frame(event(composite(7, 43)))).toEqual({ apply: true, origin: "studio" }) // the peer, contiguous + expect(cursor.frame(event(composite(7, 45)))).toEqual({ apply: true, origin: "studio", resync: { reason: "hole", namespaces: ["studio"] } }) + }) + + test("per epoch: an engine restart nothing announced is a resync, and the new epoch is the new baseline", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 30))) + expect(cursor.frame(event(id("bG9jYWwy", 1))).resync).toEqual({ reason: "hole", namespaces: ["local"] }) + expect(cursor.frame(event(id("bG9jYWwy", 2))).resync).toBeUndefined() + }) + + test("a first sighting is a baseline, and ids that are not an engine's (an older relay) are never checked", () => { + const cursor = createStreamCursor() + expect(cursor.frame(event(id(E, 50))).resync).toBeUndefined() + const relay = createStreamCursor() + relay.frame(event("17")) + expect(relay.frame(event("40"))).toEqual({ apply: true, origin: undefined }) + }) +}) + +describe("#1777 AC2 — an upstream reset forces exactly one full resync", () => { + test("fleet-of-one: the engine's reset frame is a resync of `local`, and its new position is the baseline", () => { + const cursor = createStreamCursor(id(E, 12)) // a cursor from before the engine restarted + const reset = cursor.frame(event(id("bG9jYWwy", 3), "server.cursor.unservable")) + expect(reset).toEqual({ apply: true, origin: undefined, resync: { reason: "reset", namespaces: ["local"] } }) + expect(cursor.current()).toBe(id("bG9jYWwy", 3)) + expect(cursor.frame(event(id("bG9jYWwy", 3), "server.connected")).resync).toBeUndefined() + expect(cursor.frame(event(id("bG9jYWwy", 4))).resync).toBeUndefined() + }) + + test("a reset BEHIND the client's cursor (the engine's `ahead` reason, same epoch) is still the new baseline", () => { + const cursor = createStreamCursor(id(E, 40)) // a cursor past what the engine has + expect(cursor.frame(event(id(E, 7), "server.cursor.unservable")).resync).toEqual({ reason: "reset", namespaces: ["local"] }) + expect(cursor.current()).toBe(id(E, 7)) + expect(cursor.frame(event(id(E, 8)))).toEqual({ apply: true, origin: undefined }) // not mistaken for a duplicate + }) + + test("fanned in: after a gap names the peer, its next position (a new epoch) is a new baseline — one resync, not two", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(5, 42))) + cursor.gap(["studio"]) // the fan-in turned the peer's reset into a gap; the loop resyncs on it + const next = `local=${id(E, 5)};studio=${id("c3R1ZGlvMg", 7)}` + expect(cursor.frame(event(next, "server.connected"))).toEqual({ apply: true, origin: "studio" }) + expect(cursor.frame(event(`local=${id(E, 5)};studio=${id("c3R1ZGlvMg", 8)}`)).resync).toBeUndefined() + }) + + test("a gap naming no namespace re-bases every origin", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(5, 42))) + cursor.gap([]) + expect(cursor.frame(event(composite(9, 42))).resync).toBeUndefined() + expect(cursor.frame(event(composite(9, 50))).resync).toBeUndefined() + expect(cursor.frame(event(composite(9, 52))).resync).toEqual({ reason: "hole", namespaces: ["studio"] }) // checked again + }) + + test("clear() forgets the cursor (an attachment change opens the stream cursorless)", () => { + const cursor = createStreamCursor(id(E, 3)) + cursor.clear() + expect(cursor.current()).toBeUndefined() + expect(cursor.frame(event(id("bG9jYWwy", 90))).resync).toBeUndefined() + }) +}) + +describe("#1777 AC5 — replayed frames are never applied twice", () => { + test("fleet-of-one: a bus event at or behind the applied seq is dropped; the cursor never moves back", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 5))) + expect(cursor.frame(event(id(E, 5))).apply).toBe(false) // the last frame again + expect(cursor.frame(event(id(E, 3))).apply).toBe(false) // an older one + expect(cursor.current()).toBe(id(E, 5)) + expect(cursor.frame(event(id(E, 6)))).toEqual({ apply: true, origin: undefined }) + }) + + test("fanned in: a peer frame behind the peer's applied seq is dropped", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(5, 42))) + expect(cursor.frame(event(composite(5, 41))).apply).toBe(false) + expect(cursor.current()).toBe(composite(5, 42)) + expect(cursor.frame(event(composite(5, 43))).apply).toBe(true) + }) + + test("a connection resumed from an older cursor: its handshake still applies, but the replay of what was applied is dropped", () => { + const cursor = createStreamCursor() + for (const s of [1, 2, 3]) cursor.frame(event(id(E, s))) + expect(cursor.frame(event(id(E, 1), "server.connected")).apply).toBe(true) // control: delivered, cursor kept + expect(cursor.current()).toBe(id(E, 3)) + expect(cursor.frame(event(id(E, 2))).apply).toBe(false) + expect(cursor.frame(event(id(E, 3))).apply).toBe(false) + expect(cursor.frame(event(id(E, 4)))).toEqual({ apply: true, origin: undefined }) + }) + + test("the control frames at the position are never mistaken for duplicates", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 5))) + for (const type of ["server.connected", "server.heartbeat"]) expect(cursor.frame(event(id(E, 5), type)).apply).toBe(true) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.ts b/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.ts new file mode 100644 index 000000000..6cb2edb20 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.ts @@ -0,0 +1,176 @@ +// #1777 — the client's resume cursor for the global event stream, and the +// contiguity check that makes every unreplayable loss loud. A ZERO-DEPENDENCY +// pure module (like stream-liveness.ts — no solid-js, no @opencode-ai/*) so its +// test runs in place; server-sdk.tsx's stream loop feeds it each data frame's +// SSE id and payload type, and acts on the verdict. +// +// The wire (#1776, #1511): an engine stamps every global-stream frame +// `id: :`. Bus events take strictly increasing seqs; the +// per-connection control frames (`server.connected`, `server.heartbeat`, the +// `server.cursor.unservable` reset) consume none and carry the CURRENT seq — so +// a heartbeat exposes a hole even on an idle stream. A fleet-of-one fan-in +// passes the engine's ids through untouched; a fanned-in stream's id is the +// composite `local=;=` of every origin's position, and each frame +// moves only its own origin's. +// +// The CURSOR is that SSE id — never the JSON payload id the client tracked +// before, which the engine rejects as malformed (every reconnect was a reset). +// Contiguity is checked per origin, per epoch: +// - the next bus event of an origin must be seq + 1; anything later is a HOLE +// (a resync of that origin is owed), anything at or behind it a DUPLICATE +// (a replayed frame already applied — dropped, never applied twice); +// - a control frame may sit at the position; past it is a hole; +// - a new epoch nothing announced is a hole (an engine restart the relay +// chain lost); a reset, or a gap the relay announced, re-bases the origin. +import { resolveEventOrigin } from "./sse-origin" + +/** The reserved namespace of the always-present local source (#1511 §D4). */ +export const LOCAL_ORIGIN = "local" + +const CONTROL_TYPES = new Set(["server.connected", "server.heartbeat", "server.cursor.unservable"]) +const RESET_TYPE = "server.cursor.unservable" + +type Position = { epoch: string; seq: number } + +function enginePosition(value: string | undefined): Position | undefined { + if (value === undefined) return undefined + const m = /^([A-Za-z0-9_-]+):(\d+)$/.exec(value) + return m ? { epoch: m[1], seq: Number(m[2]) } : undefined +} + +/** A stream id as per-origin positions: the composite `=;…`, or a bare + * id (no `=`, the fleet-of-one / direct-engine form) as the local origin's. */ +export function originPositions(id: string | undefined): Map { + const out = new Map() + const trimmed = id?.trim() + if (!trimmed) return out + if (!trimmed.includes("=")) { + out.set(LOCAL_ORIGIN, trimmed) + return out + } + for (const entry of trimmed.split(";")) { + const eq = entry.indexOf("=") + if (eq <= 0) continue + out.set(entry.slice(0, eq), entry.slice(eq + 1)) + } + return out +} + +export type StreamResync = { reason: "hole" | "reset"; namespaces: string[] } + +/** What the stream loop does with one data frame. */ +export type StreamCursorVerdict = { + /** false: a duplicate of a frame already applied — drop it. */ + apply: boolean + /** The origin that sent it: a peer's machine id, or undefined for local. */ + origin: string | undefined + /** Present when frames of these origins were lost: resync them. */ + resync?: StreamResync +} + +export function createStreamCursor(initial?: string) { + let raw = initial?.trim() || undefined + let applied = originPositions(raw) + // origins whose next position is a new baseline (after an announced gap) + const rebase = new Set() + + const frame = (input: { id: string | undefined; type: string | undefined }): StreamCursorVerdict => { + const id = input.id?.trim() + if (!id) return { apply: true, origin: undefined } // no SSE id (an older engine): nothing to check + const next = originPositions(id) + const bare = !id.includes("=") + const control = input.type !== undefined && CONTROL_TYPES.has(input.type) + const reset = input.type === RESET_TYPE + const changed = [...next.keys()].filter((ns) => next.get(ns) !== applied.get(ns)) + // #1539: the origin is the namespace the composite moved (fleet-of-one: local) + const origin = resolveEventOrigin(id, Object.fromEntries(applied)).origin + + const holes: string[] = [] + let behind = false + for (const ns of changed) { + const prev = enginePosition(applied.get(ns)) + const cur = enginePosition(next.get(ns)) + if (reset || rebase.has(ns) || prev === undefined || cur === undefined) continue // a baseline + if (cur.epoch !== prev.epoch) { + holes.push(ns) + continue + } + if (cur.seq <= prev.seq) { + behind = true + continue + } + if (cur.seq > prev.seq + (control ? 0 : 1)) holes.push(ns) + } + + // A frame behind what was applied: a replayed bus event is a duplicate + // (dropped); a control frame (a connection resumed from an older cursor) + // still applies, but never moves the cursor back. + if (behind) return control ? { apply: true, origin } : { apply: false, origin } + // Fleet-of-one: the last bus event again (same engine id) is a duplicate. + if (changed.length === 0 && bare && !control && enginePosition(id)) return { apply: false, origin } + + raw = id + if (bare) applied = next + else for (const [ns, value] of next) applied.set(ns, value) + for (const ns of changed) rebase.delete(ns) + + if (reset) return { apply: true, origin, resync: { reason: "reset", namespaces: changed.length > 0 ? changed : [...next.keys()] } } + if (holes.length > 0) return { apply: true, origin, resync: { reason: "hole", namespaces: holes } } + return { apply: true, origin } + } + + return { + /** The cursor to resume from: the last applied frame's SSE id, verbatim. */ + current: (): string | undefined => raw, + frame, + /** The relay announced a gap for these origins (none named = every origin); + * the loop resyncs them, so their next positions are new baselines. */ + gap(namespaces: string[]): void { + for (const ns of namespaces.length > 0 ? namespaces : [...applied.keys()]) rebase.add(ns) + }, + /** Forget the cursor: the next connection opens without one. */ + clear(): void { + raw = undefined + applied = new Map() + rebase.clear() + }, + } +} + +export type StreamCursor = ReturnType + +const STREAM_PATHS = new Set(["/event", "/global/event"]) + +const withoutLastEventIdHeader = (init: RequestInit | undefined): RequestInit | undefined => { + if (!init?.headers) return init + const headers = new Headers(init.headers) + headers.delete("last-event-id") + return { ...init, headers } +} + +/** The ONE cursor carrier on the event stream request: `?lastEventID=` + * — the parameter the fan-in, the proxies and the engine all read. The SDK + * reader would add its own `Last-Event-ID` header on an internal retry, which + * the engine prefers; it is removed, so two cursors can never disagree. No + * cursor: none is sent. Any other request passes through untouched. */ +export function applyStreamCursor( + input: RequestInfo | URL, + init: RequestInit | undefined, + base: string, + cursor: string | undefined, +): { input: RequestInfo | URL; init: RequestInit | undefined } { + let url: URL + try { + url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, base) + } catch { + return { input, init } + } + if (!STREAM_PATHS.has(url.pathname)) return { input, init } + if (cursor) url.searchParams.set("lastEventID", cursor) + else url.searchParams.delete("lastEventID") + if (typeof input === "string") return { input: url.toString(), init: withoutLastEventIdHeader(init) } + if (input instanceof URL) return { input: url, init: withoutLastEventIdHeader(init) } + const request = new Request(url.toString(), input) + request.headers.delete("last-event-id") + return { input: request, init: withoutLastEventIdHeader(init) } +} diff --git a/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.test.ts b/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.test.ts index b2364a6f5..89984dbed 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.test.ts +++ b/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test" -import { shouldReconnectIdleStream } from "./stream-liveness" +import { createStreamEndResync, isUnexpectedStreamEnd, shouldReconnectIdleStream } from "./stream-liveness" // #1584 — the client-side SSE liveness watchdog's DECISION, extracted as a // zero-dependency pure fn so it runs in-place (`bun test --conditions=solid`) @@ -37,3 +37,58 @@ describe("shouldReconnectIdleStream (#1584) — the liveness watchdog's decision ).toBe(false) }) }) + +// #1773 — an unexpected NORMAL end of the global stream is a loss. The global +// stream never ends on purpose except on shutdown, so a clean end the client did +// not ask for means events were dropped somewhere on the relay chain (a severed +// fan-in listener, a relay that ended instead of erroring). The loop used to +// reconnect 250 ms later without refetching anything. Now it forces a resync — +// but NOT for a user-initiated stop, a page hide, or the client's own abort. +describe("isUnexpectedStreamEnd (#1773) — which normal ends are a loss", () => { + test("a completed stream the client neither aborted nor stopped → unexpected (a loss)", () => { + expect(isUnexpectedStreamEnd({ attemptAborted: false, stopped: false })).toBe(true) + }) + + test("a user-initiated stop / page hide (the loop is stopped) → expected, no resync", () => { + expect(isUnexpectedStreamEnd({ attemptAborted: true, stopped: true })).toBe(false) + expect(isUnexpectedStreamEnd({ attemptAborted: false, stopped: true })).toBe(false) + }) + + test("the client's own abort (watchdog, error handling, reset) → expected, no resync here", () => { + expect(isUnexpectedStreamEnd({ attemptAborted: true, stopped: false })).toBe(false) + }) +}) + +describe("createStreamEndResync (#1773) — an unexpected end resyncs once the stream is back", () => { + test("after an unexpected end, the reconnected stream's FIRST frame fires the resync — exactly once", () => { + const resync = createStreamEndResync() + expect(resync.frame()).toBe(false) // a healthy stream never resyncs + resync.ended({ attemptAborted: false, stopped: false }) + expect(resync.pending()).toBe(true) + // the refetch waits for the re-subscription (the first frame on the new + // stream), so events between the end and the reconnect cannot slip past it + expect(resync.frame()).toBe(true) + expect(resync.frame()).toBe(false) + expect(resync.pending()).toBe(false) + }) + + test("a user stop or a page hide arms nothing — the next stream resyncs nothing", () => { + const resync = createStreamEndResync() + resync.ended({ attemptAborted: true, stopped: true }) // stop() / pagehide + expect(resync.pending()).toBe(false) + expect(resync.frame()).toBe(false) + }) + + test("the client's own abort arms nothing (its own path already heals: watchdog bump, disconnect edge)", () => { + const resync = createStreamEndResync() + resync.ended({ attemptAborted: true, stopped: false }) + expect(resync.frame()).toBe(false) + }) + + test("several unexpected ends before the stream is back still resync once", () => { + const resync = createStreamEndResync() + resync.ended({ attemptAborted: false, stopped: false }) + resync.ended({ attemptAborted: false, stopped: false }) + expect([resync.frame(), resync.frame()]).toEqual([true, false]) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.ts b/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.ts index 9fbd435ed..04fa5aa18 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.ts +++ b/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.ts @@ -29,3 +29,40 @@ export function shouldReconnectIdleStream(input: { }): boolean { return input.status === "connected" && input.now - input.lastFrameAt > input.staleMs } + +// #1773 — an unexpected NORMAL end of the global stream is a loss. The global +// stream never ends on purpose except on shutdown, so a clean end the client did +// not ask for means events were dropped somewhere on the relay chain (a severed +// fan-in listener, a relay that ended instead of erroring). The loop used to +// reconnect 250 ms later without refetching anything: the viewed transcript and +// the entity rail stayed silently behind. Kept even once replay exists (defense +// in depth). + +/** Is this completion of the stream iterator a LOSS (a resync is owed)? Only + * when the client neither aborted the attempt (its watchdog, its error + * handling, a reset — those paths heal on their own) nor stopped the loop (a + * user-initiated stop, a page hide, teardown). */ +export function isUnexpectedStreamEnd(input: { attemptAborted: boolean; stopped: boolean }): boolean { + return !input.attemptAborted && !input.stopped +} + +/** The stream loop's end-of-stream resync latch. `ended()` is told every normal + * completion; an unexpected one ARMS a resync, which fires on the reconnected + * stream's FIRST frame (`frame()` returns true exactly once) — after the + * re-subscription is live, so a refetch cannot run ahead of the reconnect and + * miss the events in between. */ +export function createStreamEndResync() { + let armed = false + return { + ended(input: { attemptAborted: boolean; stopped: boolean }): void { + if (isUnexpectedStreamEnd(input)) armed = true + }, + /** A frame arrived: true when it is the first since an unexpected end. */ + frame(): boolean { + if (!armed) return false + armed = false + return true + }, + pending: (): boolean => armed, + } +} diff --git a/packages/app-bundle/overlay/packages/app/src/pages/directory-layout.tsx b/packages/app-bundle/overlay/packages/app/src/pages/directory-layout.tsx new file mode 100644 index 000000000..7120fd4cc --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/pages/directory-layout.tsx @@ -0,0 +1,135 @@ +import { DataProvider } from "@opencode-ai/session-ui/context" +import { showToast } from "@/utils/toast" +import { base64Encode } from "@opencode-ai/core/util/encode" +import { useLocation, useNavigate, useParams } from "@solidjs/router" +import { type Accessor, createEffect, createMemo, createResource, onCleanup, type ParentProps, Show } from "solid-js" +import { useLanguage } from "@/context/language" +import { LocalProvider } from "@/context/local" +import { SDKProvider, useSDK } from "@/context/sdk" +import { useSync } from "@/context/sync" +import { decode64 } from "@/utils/base64" +import { Schema } from "effect" +import type { ServerConnection } from "@/context/server" +import { sessionHref } from "@/utils/session-route" +import { useServerSync } from "@/context/server-sync" + +export function DirectoryDataProvider( + props: ParentProps<{ + directory: string | Accessor + draftID?: string + server?: Accessor + }>, +) { + const location = useLocation() + const navigate = useNavigate() + const params = useParams() + const sdk = useSDK() + const sync = useSync() + const serverSync = useServerSync() + const directory = () => (typeof props.directory === "function" ? props.directory() : props.directory) + const slug = createMemo(() => base64Encode(directory())) + const href = (sessionID: string) => { + const server = props.server?.() + if (server) return sessionHref(server, sessionID) + return `/${slug()}/session/${sessionID}` + } + // #1778 (amicode overlay): live summaries carry no patch bodies; the per-turn + // diff view fetches a turn's stored diffs (patches included) when a file is + // expanded — the stored message is the source. + const loadSummaryDiffs = ({ sessionID, messageID }: { sessionID: string; messageID: string }) => + sdk() + .client.session.message({ sessionID, messageID }) + .then((response) => { + const info = response.data?.info + return info?.role === "user" ? info.summary?.diffs : undefined + }) + + createEffect(() => { + // A draft lives at /new-session?draftId=… and has no directory segment to normalize. + if (props.draftID || props.server?.()) return + const next = sync().data.path.directory + if (!next || next === directory()) return + const path = location.pathname.slice(slug().length + 1) + navigate(`/${base64Encode(next)}${path}${location.search}${location.hash}`, { replace: true }) + }) + + createResource( + () => params.id, + (id) => + sync() + .session.sync(id) + .catch(() => {}), + ) + + createEffect(() => { + const sessionID = params.id + if (!sessionID) return + serverSync().session.pin(sessionID) + onCleanup(() => serverSync().session.unpin(sessionID)) + }) + + return ( + + {(directory) => ( + navigate(href(sessionID))} + onSessionHref={href} + onLoadSummaryDiffs={loadSummaryDiffs} + > + {props.children} + + )} + + ) +} + +export const ProjectDirString = Schema.String.pipe(Schema.brand("ProjectDirString")) +export type ProjectDirString = Schema.Schema.Type + +export function decodeDirectory(dir: string): ProjectDirString | undefined { + const decoded = decode64(dir) + if (!decoded) return + return ProjectDirString.make(decoded) +} + +export default function Layout(props: ParentProps) { + const params = useParams() + const language = useLanguage() + const navigate = useNavigate() + let invalid = "" + + const resolved = createMemo(() => { + if (!params.dir) return "" + return decodeDirectory(params.dir) ?? "" + }) + + createEffect(() => { + const dir = params.dir + if (!dir) return + if (resolved()) { + invalid = "" + return + } + if (invalid === dir) return + invalid = dir + showToast({ + variant: "error", + title: language.t("common.requestFailed"), + description: language.t("directory.error.invalidUrl"), + }) + navigate("/", { replace: true }) + }) + + return ( + + {(resolved) => ( + + {props.children} + + )} + + ) +} diff --git a/packages/app-bundle/overlay/packages/app/src/utils/diffs.test.ts b/packages/app-bundle/overlay/packages/app/src/utils/diffs.test.ts new file mode 100644 index 000000000..bfa354df6 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/utils/diffs.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, test } from "bun:test" +import { readFileSync } from "node:fs" +import { resolve } from "node:path" +import type { SnapshotFileDiff } from "@opencode-ai/sdk/v2" +import type { FileDiffInfo } from "@opencode-ai/client/promise" +import type { Message } from "@opencode-ai/sdk/v2/client" +import { diffs, message } from "./diffs" + +const item = { + file: "src/app.ts", + patch: "@@ -1 +1 @@\n-old\n+new\n", + additions: 1, + deletions: 1, + status: "modified", +} satisfies FileDiffInfo & SnapshotFileDiff + +describe("diffs", () => { + test("keeps valid arrays", () => { + expect(diffs([item])).toEqual([item]) + }) + + test("wraps a single diff object", () => { + expect(diffs(item)).toEqual([item]) + }) + + test("reads keyed diff objects", () => { + expect(diffs({ a: item })).toEqual([item]) + }) + + test("drops invalid entries", () => { + expect( + diffs([ + item, + { file: "src/bad.ts", patch: 42, additions: 1, deletions: 1 }, + { file: "src/bad.ts", additions: "1", deletions: 1 }, + { patch: item.patch, additions: 1, deletions: 1 }, + ]), + ).toEqual([item]) + }) + + // #1778 (amicode overlay): a live summary's diffs carry no patch body — the + // engine strips it on the wire, and the per-turn diff view fetches it on + // demand. A patch-less diff (file, status, counts) is valid, not dropped: + // before, the whole live file list vanished. + test("keeps a patch-less diff — the live wire form", () => { + const { patch: _patch, ...wire } = item + expect(diffs([item, wire, { file: "src/new.ts", additions: 4, deletions: 0 }])).toEqual([ + item, + wire, + { file: "src/new.ts", additions: 4, deletions: 0 }, + ]) + }) +}) + +describe("message", () => { + test("normalizes user summaries with object diffs", () => { + const input = { + id: "msg_1", + sessionID: "ses_1", + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: "openai", modelID: "gpt-5" }, + summary: { + title: "Edit", + diffs: { a: item }, + }, + } as unknown as Message + + expect(message(input)).toMatchObject({ + summary: { + title: "Edit", + diffs: [item], + }, + }) + }) + + test("drops invalid user summaries", () => { + const input = { + id: "msg_1", + sessionID: "ses_1", + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: "openai", modelID: "gpt-5" }, + summary: true, + } as unknown as Message + + expect(message(input)).toMatchObject({ summary: undefined }) + }) + + test("a live message.updated summary keeps its file list and counts without patch bodies (#1778)", () => { + const wire = [ + { file: "src/app.ts", additions: 1, deletions: 1, status: "modified" }, + { file: "src/new.ts", additions: 4, deletions: 0, status: "added" }, + ] + const input = { + id: "msg_1", + sessionID: "ses_1", + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: "openai", modelID: "gpt-5" }, + summary: { title: "Edit", diffs: wire }, + } as unknown as Message + + // unchanged: the normalizer returns the very same message + expect(message(input)).toBe(input) + }) +}) + +describe("directory data provider — on-demand summary patches (#1778)", () => { + const source = readFileSync(resolve(__dirname, "../pages/directory-layout.tsx"), "utf8") + + test("supplies the per-turn diff view's patch source from the stored message", () => { + expect(source).toContain("onLoadSummaryDiffs={loadSummaryDiffs}") + expect(source).toContain(".client.session.message({ sessionID, messageID })") + expect(source).toContain("summary?.diffs") + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/utils/diffs.ts b/packages/app-bundle/overlay/packages/app/src/utils/diffs.ts new file mode 100644 index 000000000..47415f871 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/utils/diffs.ts @@ -0,0 +1,53 @@ +import type { SnapshotFileDiff, VcsFileDiff } from "@opencode-ai/sdk/v2" +import type { FileDiffInfo } from "@opencode-ai/client/promise" +import type { Message } from "@opencode-ai/sdk/v2/client" + +type Diff = FileDiffInfo | SnapshotFileDiff | VcsFileDiff + +function diff(value: unknown): value is Diff { + if (!value || typeof value !== "object" || Array.isArray(value)) return false + if (!("file" in value) || typeof value.file !== "string") return false + // #1778 (amicode overlay): a live summary diff has no patch body (the engine + // strips it on the wire; it is fetched on demand) — absent is valid, a + // non-string patch is not. + if ("patch" in value && value.patch !== undefined && typeof value.patch !== "string") return false + if (!("additions" in value) || typeof value.additions !== "number") return false + if (!("deletions" in value) || typeof value.deletions !== "number") return false + if (!("status" in value) || value.status === undefined) return true + return value.status === "added" || value.status === "deleted" || value.status === "modified" +} + +function object(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value) +} + +export function diffs(value: unknown): Diff[] { + if (Array.isArray(value) && value.every(diff)) return value + if (Array.isArray(value)) return value.filter(diff) + if (diff(value)) return [value] + if (!object(value)) return [] + return Object.values(value).filter(diff) +} + +export function message(value: Message): Message { + if (value.role !== "user") return value + + const raw = value.summary as unknown + if (raw === undefined) return value + if (!object(raw)) return { ...value, summary: undefined } + + const title = typeof raw.title === "string" ? raw.title : undefined + const body = typeof raw.body === "string" ? raw.body : undefined + const next = diffs(raw.diffs) + + if (title === raw.title && body === raw.body && next === raw.diffs) return value + + return { + ...value, + summary: { + ...(title === undefined ? {} : { title }), + ...(body === undefined ? {} : { body }), + diffs: next, + }, + } +} diff --git a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts new file mode 100644 index 000000000..1eefa5834 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts @@ -0,0 +1,286 @@ +// The global event stream's replay buffer (#1776). One tap on the global bus +// serializes each event once, at emit, into a frame stamped with its +// `:` id; the buffer keeps recent frames (one buffer shared by +// every subscriber, bounded by bytes) and fans each frame out to the live +// subscriptions. A connection that resumes from a servable cursor gets the +// frames after it, then live frames; any other cursor gets an explicit reset. +import { GlobalBus, type GlobalEvent } from "./global" + +export type Frame = { + /** The SSE id: `:`. */ + readonly id: string + readonly seq: number + /** The serialized global event — the SSE `data`. */ + readonly data: string + /** UTF-8 size of `data`, charged against the byte bound. */ + readonly bytes: number + /** Push time (ms), checked against the age bound. */ + readonly time: number + /** Push order; unlike `seq` it is unique per frame (a sync copy shares its source's seq). */ + readonly ordinal: number + /** + * A sync copy sharing the sequence of the frame it mirrors (#1778). Sent only + * to subscribers that request sync copies; dropping it leaves no hole in a + * subscriber's sequence, because its source carries the same one. + */ + readonly syncCopy: boolean +} + +export type Bounds = { + /** Retained frame bytes; the oldest frames are evicted past it. */ + readonly maxBytes: number + /** Retained frame age in milliseconds; older frames are evicted. */ + readonly maxAgeMs: number +} + +/** About 32 MB and 5 minutes of history: covers link blips and window reloads. */ +export const DEFAULT_BOUNDS: Bounds = { maxBytes: 32 * 1024 * 1024, maxAgeMs: 5 * 60_000 } + +export const BOUNDS_ENV = { + maxBytes: "OPENCODE_GLOBAL_EVENT_REPLAY_MAX_BYTES", + maxAgeMs: "OPENCODE_GLOBAL_EVENT_REPLAY_MAX_AGE_MS", +} as const + +/** The buffer bounds from the environment; an unset or invalid value keeps its default. */ +export function boundsFromEnv(env: Record): Bounds { + const read = (key: string, fallback: number) => { + const raw = env[key]?.trim() + if (!raw || !/^\d+$/.test(raw)) return fallback + return Number(raw) + } + return { + maxBytes: read(BOUNDS_ENV.maxBytes, DEFAULT_BOUNDS.maxBytes), + maxAgeMs: read(BOUNDS_ENV.maxAgeMs, DEFAULT_BOUNDS.maxAgeMs), + } +} + +export function formatID(epoch: string, seq: number) { + return `${epoch}:${seq}` +} + +/** Parse an `:` cursor; `undefined` when malformed. */ +export function parseCursor(raw: string): { epoch: string; seq: number } | undefined { + const match = /^([A-Za-z0-9_-]+):(\d+)$/.exec(raw.trim()) + if (!match) return undefined + return { epoch: match[1], seq: Number(match[2]) } +} + +/** Why a cursor cannot be served. */ +export type ResetReason = "malformed" | "epoch" | "ahead" | "evicted" + +export type Subscription = { + /** The connection's starting position: the sequence its first frame follows. */ + readonly position: number + /** Set when the cursor could not be served: the connection must start with a reset frame. */ + readonly reset: ResetReason | undefined + /** Frames after the cursor, in order — delivered before any live frame. */ + readonly replay: readonly Frame[] + /** Begin delivering live frames (any that arrived since subscribing first, minus duplicates of `replay`). */ + start(deliver: (frame: Frame) => void): void + /** Unregister. Idempotent. */ + close(): void +} + +export class ReplayBuffer { + readonly #listeners = new Set<(frame: Frame) => void>() + // Retained frames are #frames[#head..]; evicted slots before #head are + // compacted away in batches so eviction stays O(1) amortized. + #frames: (Frame | undefined)[] = [] + #head = 0 + #bytes = 0 + #ordinal = 0 + #current: number + // The highest sequence whose frames are no longer all retained (evicted, or + // emitted before this buffer existed). A cursor below it cannot be served. + #floor: number + + readonly epoch: string + readonly bounds: Bounds + readonly #now: () => number + + constructor(input: { + epoch: string + bounds?: Bounds + /** The sequence the buffer starts at (events before it were never recorded). */ + start?: number + now?: () => number + }) { + this.epoch = input.epoch + this.bounds = input.bounds ?? DEFAULT_BOUNDS + this.#now = input.now ?? Date.now + this.#current = input.start ?? 0 + this.#floor = this.#current + } + + /** The current sequence: the last one pushed (or skipped). */ + get current() { + return this.#current + } + + /** Live subscriptions currently registered. */ + get subscribers() { + return this.#listeners.size + } + + push(seq: number, data: string, options: { syncCopy?: boolean } = {}): Frame { + const frame: Frame = { + id: formatID(this.epoch, seq), + seq, + data, + bytes: Buffer.byteLength(data), + time: this.#now(), + ordinal: ++this.#ordinal, + syncCopy: options.syncCopy === true, + } + this.#current = Math.max(this.#current, seq) + this.#frames.push(frame) + this.#bytes += frame.bytes + for (const listener of this.#listeners) listener(frame) + this.#evict() + return frame + } + + /** Advance past a sequence that produced no frame (the hole stays visible to clients). */ + skip(seq: number) { + this.#current = Math.max(this.#current, seq) + } + + /** + * Subscribe, optionally resuming after `cursor`. The live listener is + * registered BEFORE the buffer is read, and live frames already covered by + * the replay are dropped, so nothing published around the reconnect is + * missed or delivered twice. Sync copies reach the subscription only when it + * asks for them (`sync`), in its replay and its live frames alike. + */ + subscribe(cursor?: string, options: { sync?: boolean } = {}): Subscription { + const wanted = options.sync === true ? () => true : (frame: Frame) => !frame.syncCopy + let deliver: ((frame: Frame) => void) | undefined + let pending: Frame[] = [] + const listener = (frame: Frame) => { + if (!wanted(frame)) return + if (deliver) deliver(frame) + else pending.push(frame) + } + this.#listeners.add(listener) + + this.#evict() + const watermark = this.#ordinal + const resolved = this.#resolve(cursor) + const replay = resolved.reset + ? [] + : this.#retained().filter((frame) => frame.seq > resolved.position && wanted(frame)) + + return { + ...resolved, + replay, + start: (fn) => { + const queued = pending + pending = [] + for (const frame of queued) if (frame.ordinal > watermark) fn(frame) + deliver = fn + }, + close: () => { + this.#listeners.delete(listener) + pending = [] + }, + } + } + + #resolve(cursor: string | undefined): { position: number; reset: ResetReason | undefined } { + if (cursor === undefined) return { position: this.#current, reset: undefined } + const parsed = parseCursor(cursor) + if (!parsed) return { position: this.#current, reset: "malformed" } + const reset: ResetReason | undefined = + parsed.epoch !== this.epoch + ? "epoch" + : parsed.seq > this.#current + ? "ahead" + : parsed.seq < this.#floor + ? "evicted" + : undefined + return reset ? { position: this.#current, reset } : { position: parsed.seq, reset: undefined } + } + + #retained(): Frame[] { + return this.#frames.slice(this.#head) as Frame[] + } + + // Oldest-first, until retained bytes and the oldest frame's age are both within bounds. + #evict() { + const oldest = this.#now() - this.bounds.maxAgeMs + while (this.#head < this.#frames.length) { + const frame = this.#frames[this.#head]! + if (this.#bytes <= this.bounds.maxBytes && frame.time >= oldest) break + this.#frames[this.#head++] = undefined + this.#bytes -= frame.bytes + this.#floor = Math.max(this.#floor, frame.seq) + } + if (this.#head > 1024 && this.#head * 2 > this.#frames.length) { + this.#frames = this.#frames.slice(this.#head) + this.#head = 0 + } + } +} + +export const GlobalEventReplay = new ReplayBuffer({ + epoch: GlobalBus.epoch, + bounds: boundsFromEnv(process.env), + start: GlobalBus.seq, +}) + +/** + * The wire form of a global event (#1778). A per-turn summary (`info.summary. + * diffs` of a message or session) carries the full patch text of every changed + * file — measured up to 12.3 MB in one event. On the wire each diff keeps its + * file, status and counts and drops its patch body; the surfaces that render a + * patch fetch the stored message on demand. Only the serialized frame changes: + * the event object (what in-process subscribers and storage see) is never + * mutated, and a sync copy is sent whole — its subscriber replays it into a + * store of its own, which must keep the full record. + */ +export function wireEvent(event: GlobalEvent): GlobalEvent { + const payload = event.payload + if (!payload || typeof payload !== "object" || payload.type === "sync") return event + const info = payload.properties?.info + const diffs = info?.summary?.diffs + if (!Array.isArray(diffs) || !diffs.some((diff) => diff && typeof diff === "object" && "patch" in diff)) return event + return { + ...event, + payload: { + ...payload, + properties: { + ...payload.properties, + info: { + ...info, + summary: { + ...info.summary, + diffs: diffs.map((diff) => { + if (!diff || typeof diff !== "object" || !("patch" in diff)) return diff + const { patch: _patch, ...rest } = diff + return rest + }), + }, + }, + }, + }, + } +} + +GlobalBus.tap((event, seq) => { + // A sync copy whose sequence was already handed out mirrors its source. One + // that consumed a fresh sequence (no source framed nearby) is a frame of its + // own and reaches every subscriber, so filtering never opens a hole. + const syncCopy = event.payload?.type === "sync" && seq <= GlobalEventReplay.current + let data: string + try { + // serialized once: every subscriber, live or replayed, gets this same frame + data = JSON.stringify(wireEvent(event)) + } catch (error) { + // An unserializable event cannot be framed; its sequence becomes a hole the + // client detects (heartbeats carry the current sequence), never a dead stream. + console.error("global event not serializable", { seq, type: event.payload?.type, error }) + GlobalEventReplay.skip(seq) + return + } + GlobalEventReplay.push(seq, data, { syncCopy }) +}) diff --git a/packages/app-bundle/overlay/packages/opencode/src/bus/global.ts b/packages/app-bundle/overlay/packages/opencode/src/bus/global.ts new file mode 100644 index 000000000..2e6ee4e20 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/bus/global.ts @@ -0,0 +1,76 @@ +import { EventEmitter } from "events" +import { randomBytes } from "crypto" +import { Identifier } from "@/id/id" + +export type GlobalEvent = { + directory?: string + project?: string + workspace?: string + payload: any +} + +/** + * A pre-dispatch observer. Taps see every emitted event with its sequence + * BEFORE any ordinary listener runs, so they observe events in strict + * sequence order even when a listener emits from inside its handler (a nested + * emit is sequenced and tapped only after the outer event was). A tap must not + * throw and must not emit. + */ +export type GlobalBusTap = (event: GlobalEvent, seq: number) => void + +// How many recent source-event ids are remembered so a sync copy (which the +// event-v2 bridge emits right after its source) can share the source's +// sequence. The copy follows its source immediately; the window only has to +// absorb events a listener emits in between. +const MIRROR_WINDOW = 256 + +class GlobalBusEmitter extends EventEmitter<{ + event: [GlobalEvent] +}> { + /** Random per-process token; `:` is a global-stream frame id (#1776). */ + readonly epoch = randomBytes(6).toString("base64url") + #seq = 0 + #mirrors = new Map() + #taps = new Set() + + /** The current sequence: the one assigned to the last emitted event (0 before any). */ + get seq() { + return this.#seq + } + + /** Register a pre-dispatch tap; returns its unregister function. */ + tap(fn: GlobalBusTap): () => void { + this.#taps.add(fn) + return () => { + this.#taps.delete(fn) + } + } + + override emit(eventName: "event", event: GlobalEvent): boolean { + if (event.payload && typeof event.payload === "object" && !("id" in event.payload)) { + event.payload.id = event.payload.syncEvent?.id ?? Identifier.create("evt", "ascending") + } + const seq = this.#sequence(event.payload) + for (const tap of this.#taps) tap(event, seq) + return super.emit(eventName, event) + } + + // One strictly increasing counter, assigned once per event at emit and shared + // by every subscriber. A sync copy is not a new event: it carries the + // sequence of the event it mirrors, so filtering sync copies leaves no holes. + #sequence(payload: any): number { + if (payload?.type === "sync") { + const source = typeof payload.syncEvent?.id === "string" ? this.#mirrors.get(payload.syncEvent.id) : undefined + if (source !== undefined) return source + return ++this.#seq + } + const seq = ++this.#seq + if (typeof payload?.id === "string") { + this.#mirrors.set(payload.id, seq) + if (this.#mirrors.size > MIRROR_WINDOW) this.#mirrors.delete(this.#mirrors.keys().next().value!) + } + return seq + } +} + +export const GlobalBus = new GlobalBusEmitter() diff --git a/packages/app-bundle/overlay/packages/opencode/src/control-plane/workspace.ts b/packages/app-bundle/overlay/packages/opencode/src/control-plane/workspace.ts new file mode 100644 index 000000000..b89c5a188 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/control-plane/workspace.ts @@ -0,0 +1,970 @@ +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { httpClient } from "@opencode-ai/core/effect/app-node-platform" +import { Context, Effect, FiberMap, Iterable, Layer, Schema, Stream } from "effect" +import { serviceUse } from "@opencode-ai/core/effect/service-use" +import { FetchHttpClient, HttpBody, HttpClient, HttpClientError, HttpClientRequest } from "effect/unstable/http" +import { Database } from "@opencode-ai/core/database/database" +import { asc } from "drizzle-orm" +import { eq } from "drizzle-orm" +import { inArray } from "drizzle-orm" +import { Project } from "@/project/project" +import { GlobalBus } from "@/bus/global" +import { Auth } from "@/auth" +import { EventV2 } from "@opencode-ai/core/event" +import { EventV2Bridge } from "@/event-v2-bridge" +import { EventSequenceTable, EventTable } from "@opencode-ai/core/event/sql" +import { FSUtil } from "@opencode-ai/core/fs-util" +import { RuntimeFlags } from "@/effect/runtime-flags" +import { ProjectV2 } from "@opencode-ai/core/project" +import { Slug } from "@opencode-ai/core/util/slug" +import { WorkspaceTable } from "@opencode-ai/core/control-plane/workspace.sql" +import { getAdapter, registeredAdapters } from "./adapters" +import { type Target, type WorkspaceInfo, WorkspaceInfo as WorkspaceInfoSchema } from "./types" +import { WorkspaceV2 } from "@opencode-ai/core/workspace" +import { Session } from "@/session/session" +import { SessionPrompt } from "@/session/prompt" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionID } from "@/session/schema" +import { NotFoundError } from "@/storage/storage" +import { errorData } from "@/util/error" +import { waitEvent } from "./util" +import { WorkspaceRef } from "@/effect/instance-ref" +import { Vcs } from "@/project/vcs" +import { InstanceStore } from "@/project/instance-store" +import { WorkspaceAdapterRuntime } from "./workspace-adapter-runtime" +import { AppNodeBuilderV1 } from "@/effect/app-node-builder-v1" +import { WorkspaceEvent } from "@opencode-ai/schema/workspace-event" + +export const Info = Schema.Struct({ + ...WorkspaceInfoSchema.fields, + timeUsed: Schema.Number, +}).annotate({ identifier: "Workspace" }) +export type Info = WorkspaceInfo & { timeUsed: number } + +export const ConnectionStatus = WorkspaceEvent.ConnectionStatus +export type ConnectionStatus = WorkspaceEvent.ConnectionStatus + +export const Event = WorkspaceEvent + +function fromRow(row: typeof WorkspaceTable.$inferSelect): Info { + return { + id: row.id, + type: row.type, + branch: row.branch, + name: row.name, + directory: row.directory, + extra: row.extra, + projectID: row.project_id, + timeUsed: row.time_used, + } +} + +export const CreateInput = Schema.Struct({ + id: Schema.optional(WorkspaceV2.ID), + type: Info.fields.type, + branch: Info.fields.branch, + projectID: ProjectV2.ID, + extra: Schema.optional(Info.fields.extra), +}) +export type CreateInput = Schema.Schema.Type + +export const SessionWarpInput = Schema.Struct({ + workspaceID: Schema.NullOr(WorkspaceV2.ID), + sessionID: SessionID, + copyChanges: Schema.optional(Schema.Boolean), +}) +export type SessionWarpInput = Schema.Schema.Type + +export class SyncHttpError extends Schema.TaggedErrorClass()("WorkspaceSyncHttpError", { + message: Schema.String, + status: Schema.Number, + body: Schema.optional(Schema.String), +}) {} + +export class WorkspaceNotFoundError extends Schema.TaggedErrorClass()( + "WorkspaceNotFoundError", + { + message: Schema.String, + workspaceID: WorkspaceV2.ID, + }, +) {} + +export class SessionEventsNotFoundError extends Schema.TaggedErrorClass()( + "WorkspaceSessionEventsNotFoundError", + { + message: Schema.String, + sessionID: SessionID, + }, +) {} + +export class SessionWarpHttpError extends Schema.TaggedErrorClass()( + "WorkspaceSessionWarpHttpError", + { + message: Schema.String, + workspaceID: WorkspaceV2.ID, + sessionID: SessionID, + status: Schema.Number, + body: Schema.String, + }, +) {} + +export class SyncTimeoutError extends Schema.TaggedErrorClass()("WorkspaceSyncTimeoutError", { + message: Schema.String, + state: Schema.Record(Schema.String, Schema.Number), +}) {} + +export class SyncAbortedError extends Schema.TaggedErrorClass()("WorkspaceSyncAbortedError", { + message: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) {} + +type CreateError = Auth.AuthError +type SessionWarpError = + | WorkspaceNotFoundError + | SessionEventsNotFoundError + | SessionWarpHttpError + | Vcs.PatchApplyError + | HttpClientError.HttpClientError +type WaitForSyncError = SyncTimeoutError | SyncAbortedError +type SyncLoopError = SyncHttpError | HttpClientError.HttpClientError + +export interface Interface { + readonly create: (input: CreateInput) => Effect.Effect + readonly sessionWarp: (input: SessionWarpInput) => Effect.Effect + readonly list: (project: Project.Info) => Effect.Effect + readonly syncList: (project: Project.Info) => Effect.Effect + readonly get: (id: WorkspaceV2.ID) => Effect.Effect + readonly remove: (id: WorkspaceV2.ID) => Effect.Effect + readonly status: () => Effect.Effect + readonly isSyncing: (workspaceID: WorkspaceV2.ID) => Effect.Effect + readonly waitForSync: ( + workspaceID: WorkspaceV2.ID, + state: Record, + signal?: AbortSignal, + timeout?: number, + ) => Effect.Effect + readonly startWorkspaceSyncing: (projectID: ProjectV2.ID) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/Workspace") {} + +export const use = serviceUse(Service) + +const layer = Layer.effect( + Service, + Effect.gen(function* () { + const auth = yield* Auth.Service + const session = yield* Session.Service + const prompt = yield* SessionPrompt.Service + const http = yield* HttpClient.HttpClient + const events = yield* EventV2Bridge.Service + const vcs = yield* Vcs.Service + const flags = yield* RuntimeFlags.Service + const fs = yield* FSUtil.Service + const { db } = yield* Database.Service + const connections = new Map() + const syncFibers = yield* FiberMap.make() + + const setStatus = (id: WorkspaceV2.ID, status: ConnectionStatus["status"]) => { + const prev = connections.get(id) + if (prev?.status === status) return + const next = { workspaceID: id, status } + connections.set(id, next) + + GlobalBus.emit("event", { + directory: "global", + workspace: id, + payload: { + type: Event.Status.type, + properties: next, + }, + }) + } + + const connectSSE = Effect.fn("Workspace.connectSSE")(function* ( + url: URL | string, + headers: HeadersInit | undefined, + ) { + // Sync copies are opt-in per subscription (#1778, amicode overlay): this + // loop replays them into the local store, so it asks for them. + const request = new Headers(headers) + request.set("x-opencode-sync-copies", "true") + const response = yield* http.execute( + HttpClientRequest.get(route(url, "/global/event"), { + headers: request, + accept: "text/event-stream", + }), + ) + if (response.status < 200 || response.status >= 300) { + return yield* new SyncHttpError({ + message: `Workspace sync HTTP failure: ${response.status}`, + status: response.status, + }) + } + return response.stream + }) + + const parseSSE = Effect.fn("Workspace.parseSSE")(function* ( + stream: Stream.Stream, + onEvent: (event: unknown) => Effect.Effect, + ) { + yield* stream.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.mapAccum( + () => ({ data: [] as string[], id: undefined as string | undefined, retry: 1000 }), + (state, line) => { + if (line === "") { + if (!state.data.length) return [state, []] + return [{ ...state, data: [] }, [{ data: state.data.join("\n"), id: state.id, retry: state.retry }]] + } + + const index = line.indexOf(":") + const field = index === -1 ? line : line.slice(0, index) + const value = index === -1 ? "" : line.slice(index + (line[index + 1] === " " ? 2 : 1)) + + if (field === "data") return [{ ...state, data: [...state.data, value] }, []] + if (field === "id") return [{ ...state, id: value }, []] + if (field === "retry") { + const retry = Number.parseInt(value, 10) + return [Number.isNaN(retry) ? state : { ...state, retry }, []] + } + return [state, []] + }, + { + onHalt: (state) => + state.data.length ? [{ data: state.data.join("\n"), id: state.id, retry: state.retry }] : [], + }, + ), + Stream.map((event) => { + try { + return JSON.parse(event.data) as unknown + } catch { + return { + type: "sse.message", + properties: { + data: event.data, + id: event.id || undefined, + retry: event.retry, + }, + } + } + }), + Stream.runForEach(onEvent), + ) + }) + + const runInWorkspace = (input: { + workspaceID?: WorkspaceV2.ID + local: () => Effect.Effect + remote: (input: { + workspace: Info + target: Extract + }) => HttpClientRequest.HttpClientRequest + fallback: A + response?: "json" | "text" + }) => + Effect.gen(function* () { + if (!input.workspaceID) return yield* input.local() + + const workspace = yield* get(input.workspaceID) + if (!workspace) return input.fallback + + const target = yield* WorkspaceAdapterRuntime.target(workspace) + + if (target.type === "local") { + const store = yield* InstanceStore.Service + return yield* store.provide({ directory: target.directory }, input.local()) + } + + const response = yield* http.execute(input.remote({ workspace, target })).pipe( + Effect.catch((error) => + Effect.logWarning("workspace target request failed", { + workspaceID: workspace.id, + error: errorData(error), + }).pipe(Effect.as(undefined)), + ), + ) + if (!response) return input.fallback + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text.pipe(Effect.catch(() => Effect.succeed(""))) + yield* Effect.logWarning("workspace target request failed", { + workspaceID: workspace.id, + status: response.status, + body, + }) + return input.fallback + } + + const body = input.response === "text" ? response.text : response.json + return yield* body.pipe( + Effect.map((result) => result as A), + Effect.catch((error) => + Effect.logWarning("workspace target response decode failed", { + workspaceID: workspace.id, + error: errorData(error), + }).pipe(Effect.as(input.fallback)), + ), + ) + }) + + const syncHistory = Effect.fn("Workspace.syncHistory")(function* ( + space: Info, + url: URL | string, + headers: HeadersInit | undefined, + ) { + const sessionIDs = (yield* db + .select({ id: SessionTable.id }) + .from(SessionTable) + .where(eq(SessionTable.workspace_id, space.id)) + .all() + .pipe(Effect.orDie)).map((row) => row.id) + const state = sessionIDs.length + ? Object.fromEntries( + (yield* db + .select() + .from(EventSequenceTable) + .where(inArray(EventSequenceTable.aggregate_id, sessionIDs)) + .all() + .pipe(Effect.orDie)).map((row) => [row.aggregate_id, row.seq]), + ) + : {} + + const response = yield* http.execute( + HttpClientRequest.post(route(url, "/sync/history"), { + headers: new Headers(headers), + body: HttpBody.jsonUnsafe(state), + }), + ) + + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text + return yield* new SyncHttpError({ + message: `Workspace history HTTP failure: ${response.status} ${body}`, + status: response.status, + body, + }) + } + + const history = (yield* response.json) as HistoryEvent[] + + yield* Effect.forEach( + history, + (event) => + events + .replay( + { + id: EventV2.ID.make(event.id), + aggregateID: event.aggregate_id, + seq: event.seq, + type: event.type, + data: event.data, + }, + { publish: true, ownerID: space.id }, + ) + .pipe(Effect.provideService(WorkspaceRef, space.id)), + { discard: true }, + ) + }) + + const syncWorkspaceLoop = Effect.fn("Workspace.syncWorkspaceLoop")(function* (space: Info) { + const target = yield* WorkspaceAdapterRuntime.target(space) + + if (target.type === "local") return + + let attempt = 0 + + while (true) { + setStatus(space.id, "connecting") + + const stream = yield* connectSSE(target.url, target.headers).pipe( + Effect.tap(() => syncHistory(space, target.url, target.headers)), + Effect.catch((err) => + Effect.gen(function* () { + setStatus(space.id, "error") + yield* Effect.logWarning("failed to connect to global sync", { + workspace: space.name, + error: errorData(err), + }) + return null + }), + ), + ) + + if (stream) { + attempt = 0 + + setStatus(space.id, "connected") + + yield* parseSSE(stream, (evt) => + Effect.gen(function* () { + if (!evt || typeof evt !== "object" || !("payload" in evt)) return + const payload = evt.payload as { type?: string; syncEvent?: EventV2.SerializedEvent } + if (payload.type === "server.heartbeat") return + + if (payload.type === "sync" && payload.syncEvent) { + const failed = yield* events.replay(payload.syncEvent, { publish: true, ownerID: space.id }).pipe( + Effect.as(false), + Effect.catchCause((error) => + Effect.logWarning("failed to replay global event", error).pipe( + Effect.annotateLogs({ workspaceID: space.id }), + Effect.as(true), + ), + ), + ) + if (failed) return + } + + try { + const event = evt as { directory?: string; project?: string; payload: unknown } + GlobalBus.emit("event", { + directory: event.directory, + project: event.project, + workspace: space.id, + payload: event.payload, + }) + } catch (error) { + yield* Effect.logWarning("failed to emit global event", { + workspaceID: space.id, + error: errorData(error), + }) + } + }), + ) + + setStatus(space.id, "disconnected") + } + + // Back off reconnect attempts up to 2 minutes while the workspace + // stays unavailable. + yield* Effect.sleep(`${Math.min(120_000, 1_000 * 2 ** attempt)} millis`) + attempt += 1 + } + }) + + const startSync = Effect.fn("Workspace.startSync")(function* (space: Info) { + if (!flags.experimentalWorkspaces) return + + const target = yield* WorkspaceAdapterRuntime.target(space).pipe( + Effect.catch((error) => + Effect.gen(function* () { + setStatus(space.id, "error") + yield* Effect.logWarning("workspace target failed", { + workspaceID: space.id, + error: errorData(error), + }) + return null + }), + ), + ) + if (!target) return + + if (target.type === "local") { + setStatus(space.id, (yield* fs.existsSafe(target.directory)) ? "connected" : "error") + return + } + + const exists = yield* FiberMap.has(syncFibers, space.id) + if (exists && connections.get(space.id)?.status !== "error") return + + setStatus(space.id, "disconnected") + + yield* FiberMap.run( + syncFibers, + space.id, + // TODO: look into `tapError` to set the status but still + // allow the fiber to fail and automatically get removed + syncWorkspaceLoop(space).pipe( + Effect.catch((error) => + Effect.gen(function* () { + setStatus(space.id, "error") + yield* Effect.logWarning("workspace listener failed", { + workspaceID: space.id, + error: errorData(error), + }) + }), + ), + ), + ) + }) + + const stopSync = Effect.fn("Workspace.stopSync")(function* (id: WorkspaceV2.ID) { + yield* FiberMap.remove(syncFibers, id) + connections.delete(id) + }) + + const create = Effect.fn("Workspace.create")(function* (input: CreateInput) { + const id = WorkspaceV2.ID.ascending(input.id) + const adapter = getAdapter(input.projectID, input.type) + const config = yield* WorkspaceAdapterRuntime.configure(adapter, { + ...input, + id, + name: Slug.create(), + directory: null, + extra: input.extra ?? null, + }) + + const info: Info = { + id, + type: config.type, + branch: config.branch ?? null, + name: config.name ?? null, + directory: config.directory ?? null, + extra: config.extra ?? null, + projectID: input.projectID, + timeUsed: Date.now(), + } + + yield* db + .insert(WorkspaceTable) + .values({ + id: info.id, + type: info.type, + branch: info.branch, + name: info.name, + directory: info.directory, + extra: info.extra, + project_id: info.projectID, + time_used: info.timeUsed, + }) + .run() + .pipe(Effect.orDie) + + const env = { + OPENCODE_AUTH_CONTENT: JSON.stringify(yield* auth.all()), + OPENCODE_WORKSPACE_ID: config.id, + OPENCODE_EXPERIMENTAL_WORKSPACES: "true", + OTEL_EXPORTER_OTLP_HEADERS: process.env.OTEL_EXPORTER_OTLP_HEADERS, + OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_RESOURCE_ATTRIBUTES: process.env.OTEL_RESOURCE_ATTRIBUTES, + } + + yield* WorkspaceAdapterRuntime.create(adapter, config, env) + yield* Effect.all( + [ + waitEvent({ + timeout: TIMEOUT, + fn(event) { + if (event.workspace === info.id && event.payload.type === Event.Status.type) { + const { status } = event.payload.properties + return status === "error" || status === "connected" + } + return false + }, + }), + startSync(info), + ], + { concurrency: 2, discard: true }, + ) + + return info + }) + + const sessionWarp = Effect.fn("Workspace.sessionWarp")(function* (input: SessionWarpInput) { + return yield* Effect.gen(function* () { + const current = yield* db + .select({ workspaceID: SessionTable.workspace_id }) + .from(SessionTable) + .where(eq(SessionTable.id, input.sessionID)) + .get() + .pipe(Effect.orDie) + + if (current?.workspaceID) { + const previous = yield* get(current.workspaceID) + if (previous) { + const target = yield* WorkspaceAdapterRuntime.target(previous) + + if (target.type === "remote") { + yield* syncHistory(previous, target.url, target.headers).pipe( + Effect.catch((error) => + Effect.logWarning("session warp final source sync failed", { + workspaceID: previous.id, + sessionID: input.sessionID, + error: errorData(error), + }), + ), + ) + } else { + yield* prompt.cancel(input.sessionID) + } + + // "claim" this session so any future events coming from + // the old workspace are ignored + yield* events.claim(input.sessionID, input.workspaceID ?? previous.projectID) + } + } + + const sourcePatch = + input.copyChanges && current?.workspaceID + ? yield* runInWorkspace({ + workspaceID: current?.workspaceID ?? undefined, + local: () => vcs.diffRaw(), + remote: ({ target }) => + HttpClientRequest.get(route(target.url, "/vcs/diff/raw"), { + headers: new Headers(target.headers), + }), + fallback: "", + response: "text", + }).pipe(Effect.provide(AppNodeBuilderV1.build(InstanceStore.node))) + : "" + + if (sourcePatch) { + // Attempt to apply the file changes to the new workspace. + // We intentionally do first so if it fails we don't warp + // the session. + yield* runInWorkspace({ + workspaceID: input.workspaceID ?? undefined, + local: () => vcs.apply({ patch: sourcePatch }), + remote: ({ target }) => + HttpClientRequest.post(route(target.url, "/vcs/apply"), { + headers: new Headers(target.headers), + body: HttpBody.jsonUnsafe({ patch: sourcePatch }), + }), + fallback: { applied: false }, + }).pipe(Effect.provide(AppNodeBuilderV1.build(InstanceStore.node))) + } + + if (input.workspaceID === null) { + yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: undefined }) + + return + } + + const workspaceID = input.workspaceID + const space = yield* get(workspaceID) + if (!space) + return yield* new WorkspaceNotFoundError({ + message: `Workspace not found: ${workspaceID}`, + workspaceID, + }) + + const target = yield* WorkspaceAdapterRuntime.target(space) + + if (target.type === "local") { + yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: input.workspaceID }) + + return + } + + const rows = yield* db + .select({ + id: EventTable.id, + aggregateID: EventTable.aggregate_id, + seq: EventTable.seq, + type: EventTable.type, + data: EventTable.data, + }) + .from(EventTable) + .where(eq(EventTable.aggregate_id, input.sessionID)) + .orderBy(asc(EventTable.seq)) + .all() + .pipe(Effect.orDie) + if (rows.length === 0) + return yield* new SessionEventsNotFoundError({ + message: `No events found for session: ${input.sessionID}`, + sessionID: input.sessionID, + }) + + const batches = Iterable.chunksOf(rows, 10) + const total = Iterable.size(batches) + + yield* Effect.forEach( + batches, + (events, i) => + Effect.gen(function* () { + const response = yield* http.execute( + HttpClientRequest.post(route(target.url, "/sync/replay"), { + headers: new Headers(target.headers), + body: HttpBody.jsonUnsafe({ + directory: space.directory ?? "", + events, + }), + }), + ) + + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text + return yield* new SessionWarpHttpError({ + message: `Failed to warp session ${input.sessionID} into workspace ${workspaceID}: HTTP ${response.status} ${body}`, + workspaceID, + sessionID: input.sessionID, + status: response.status, + body, + }) + } + }), + { discard: true }, + ) + + const response = yield* http.execute( + HttpClientRequest.post(route(target.url, "/sync/steal"), { + headers: new Headers(target.headers), + body: HttpBody.jsonUnsafe({ sessionID: input.sessionID }), + }), + ) + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text + return yield* new SessionWarpHttpError({ + message: `Failed to steal session ${input.sessionID} into workspace ${workspaceID}: HTTP ${response.status} ${body}`, + workspaceID, + sessionID: input.sessionID, + status: response.status, + body, + }) + } + + yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: input.workspaceID }) + }) + }) + + const list = Effect.fn("Workspace.list")(function* (project: Project.Info) { + if (!flags.experimentalWorkspaces) return [] + return (yield* db + .select() + .from(WorkspaceTable) + .where(eq(WorkspaceTable.project_id, project.id)) + .all() + .pipe(Effect.orDie)) + .map(fromRow) + .sort((a, b) => a.id.localeCompare(b.id)) + }) + + const syncList = Effect.fn("Workspace.syncList")(function* (project: Project.Info) { + const names = new Set((yield* list(project)).map((workspace) => workspace.name)) + const discovered = yield* Effect.forEach( + registeredAdapters(project.id), + ([type, adapter]) => + WorkspaceAdapterRuntime.list(adapter).pipe( + Effect.catchCause((error) => + Effect.logWarning("workspace adapter list failed", { type, error }).pipe(Effect.as([])), + ), + ), + { concurrency: "unbounded" }, + ).pipe(Effect.map((items) => items.flat())) + + yield* Effect.forEach( + discovered, + (item) => + Effect.gen(function* () { + if (names.has(item.name)) return + names.add(item.name) + + const info: Info = { + id: WorkspaceV2.ID.ascending(), + type: item.type, + branch: item.branch, + name: item.name, + directory: item.directory, + extra: item.extra, + projectID: item.projectID, + timeUsed: Date.now(), + } + + yield* db + .insert(WorkspaceTable) + .values({ + id: info.id, + type: info.type, + branch: info.branch, + name: info.name, + directory: info.directory, + extra: info.extra, + project_id: info.projectID, + time_used: info.timeUsed, + }) + .run() + .pipe(Effect.orDie) + + yield* startSync(info) + }), + { concurrency: 1 }, + ) + }) + + const get = Effect.fn("Workspace.get")(function* (id: WorkspaceV2.ID) { + const row = yield* db.select().from(WorkspaceTable).where(eq(WorkspaceTable.id, id)).get().pipe(Effect.orDie) + if (!row) return + return fromRow(row) + }) + + const remove = Effect.fn("Workspace.remove")(function* (id: WorkspaceV2.ID) { + const sessions = yield* db + .select({ id: SessionTable.id, parentID: SessionTable.parent_id }) + .from(SessionTable) + .where(eq(SessionTable.workspace_id, id)) + .all() + .pipe(Effect.orDie) + const sessionIDs = new Set(sessions.map((sessionInfo) => sessionInfo.id)) + yield* Effect.forEach( + sessions.filter((sessionInfo) => !sessionInfo.parentID || !sessionIDs.has(sessionInfo.parentID)), + (sessionInfo) => + session.remove(sessionInfo.id).pipe(Effect.catchIf(NotFoundError.isInstance, () => Effect.void)), + { discard: true }, + ) + + const row = yield* db.select().from(WorkspaceTable).where(eq(WorkspaceTable.id, id)).get().pipe(Effect.orDie) + if (!row) return + + yield* stopSync(id) + + const info = fromRow(row) + yield* Effect.catchCause( + Effect.gen(function* () { + yield* WorkspaceAdapterRuntime.remove(info) + }), + () => Effect.logError("adapter not available when removing workspace", { type: row.type }), + ) + + yield* db.delete(WorkspaceTable).where(eq(WorkspaceTable.id, id)).run().pipe(Effect.orDie) + return info + }) + + const status = Effect.fn("Workspace.status")(function* () { + return [...connections.values()] + }) + + const isSyncing = Effect.fn("Workspace.isSyncing")(function* (workspaceID: WorkspaceV2.ID) { + const exists = yield* FiberMap.has(syncFibers, workspaceID) + return exists && connections.get(workspaceID)?.status !== "error" + }) + + const waitForSync = Effect.fn("Workspace.waitForSync")(function* ( + workspaceID: WorkspaceV2.ID, + state: Record, + signal?: AbortSignal, + timeout = TIMEOUT, + ) { + if (yield* synced(db, state)) return + + yield* Effect.catch( + waitUntilSynced({ db, workspaceID, state, signal, timeout }), + (): Effect.Effect => + signal?.aborted + ? Effect.fail( + new SyncAbortedError({ + message: signal.reason instanceof Error ? signal.reason.message : "Request aborted", + cause: signal.reason, + }), + ) + : Effect.fail( + new SyncTimeoutError({ + message: `Timed out waiting for sync fence: ${JSON.stringify(state)}`, + state, + }), + ), + ) + }) + + const startWorkspaceSyncing = Effect.fn("Workspace.startWorkspaceSyncing")(function* (projectID: ProjectV2.ID) { + if (!flags.experimentalWorkspaces) return + const rows = yield* db + .selectDistinct({ workspace: WorkspaceTable }) + .from(WorkspaceTable) + .where(eq(WorkspaceTable.project_id, projectID)) + .all() + .pipe(Effect.orDie) + + for (const { workspace } of rows) { + yield* startSync(fromRow(workspace)).pipe( + Effect.catch((error) => + Effect.sync(() => { + setStatus(workspace.id, "error") + }), + ), + Effect.forkDetach, + ) + } + }) + + return Service.of({ + create, + sessionWarp, + list, + syncList, + get, + remove, + status, + isSyncing, + waitForSync, + startWorkspaceSyncing, + }) + }), +) + +const TIMEOUT = 5000 + +type HistoryEvent = { + id: string + aggregate_id: string + seq: number + type: string + data: Record +} + +function waitUntilSynced(input: { + db: Database.Interface["db"] + workspaceID: WorkspaceV2.ID + state: Record + signal?: AbortSignal + timeout: number +}): Effect.Effect { + return Effect.suspend(() => + waitEvent({ + timeout: input.timeout, + signal: input.signal, + fn(event) { + return event.workspace === input.workspaceID || event.payload.type === "sync" + }, + }).pipe( + Effect.andThen(synced(input.db, input.state)), + Effect.flatMap((done): Effect.Effect => (done ? Effect.void : waitUntilSynced(input))), + ), + ) +} + +function synced(db: Database.Interface["db"], state: Record): Effect.Effect { + const ids = Object.keys(state) + if (ids.length === 0) return Effect.succeed(true) + + return db + .select({ + id: EventSequenceTable.aggregate_id, + seq: EventSequenceTable.seq, + }) + .from(EventSequenceTable) + .where(inArray(EventSequenceTable.aggregate_id, ids)) + .all() + .pipe( + Effect.orDie, + Effect.map((rows) => { + const done = Object.fromEntries(rows.map((row) => [row.id, row.seq])) as Record + return ids.every((id) => (done[id] ?? -1) >= state[id]) + }), + ) +} + +function route(url: string | URL, path: string) { + const next = new URL(url) + next.pathname = `${next.pathname.replace(/\/$/, "")}${path}` + next.search = "" + next.hash = "" + return next +} + +export const node = LayerNode.make({ + service: Service, + layer: layer, + deps: [ + Auth.node, + Session.node, + SessionPrompt.node, + httpClient, + EventV2Bridge.node, + Vcs.node, + RuntimeFlags.node, + FSUtil.node, + Database.node, + ], +}) + +export * as Workspace from "./workspace" diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts new file mode 100644 index 000000000..d905bd7f7 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts @@ -0,0 +1,233 @@ +import { Config } from "@/config/config" +import { GlobalBus } from "@/bus/global" +import { GlobalEventReplay, formatID, type Frame, type Subscription } from "@/bus/global-replay" +import { EffectBridge } from "@/effect/bridge" +import { EventV2 } from "@opencode-ai/core/event" +import { Installation } from "@/installation" +import { disposeAllInstancesAndEmitGlobalDisposed } from "@/server/global-lifecycle" +import { InstallationVersion } from "@opencode-ai/core/installation/version" +import { Clock, Deferred, Effect, Exit, Queue } from "effect" +import * as Stream from "effect/Stream" +import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import { HttpApiBuilder } from "effect/unstable/httpapi" +import * as Sse from "effect/unstable/encoding/Sse" +import type { Socket } from "node:net" +import { RootHttpApi } from "../api" +import { GlobalUpgradeInput } from "../groups/global" + +// The resume cursor: the standard `Last-Event-ID` header (what an SSE client +// sends once it has seen an `id:` line), else the `lastEventID` query +// parameter clients already send. +function cursorOf(request: HttpServerRequest.HttpServerRequest) { + const header = request.headers["last-event-id"]?.trim() + if (header) return header + return new URL(request.url, "http://localhost").searchParams.get("lastEventID")?.trim() || undefined +} + +// Sync copies (the event-v2 bridge's `type: "sync"` twin of every durable +// event) are opt-in per subscription (#1778): only a subscriber that replays +// them (the workspace control plane, control-plane/workspace.ts) asks — with +// the `x-opencode-sync-copies` header, else the `sync` query parameter; every +// other subscriber discards them, so they stay off its wire. +function syncRequested(request: HttpServerRequest.HttpServerRequest) { + const raw = + request.headers["x-opencode-sync-copies"] ?? new URL(request.url, "http://localhost").searchParams.get("sync") + const value = raw?.trim().toLowerCase() + return value === "true" || value === "1" +} + +function frameEvent(frame: Pick): Sse.Event { + return { _tag: "Event", event: "message", id: frame.id, data: frame.data } +} + +// A per-connection control frame (connected, heartbeat, reset). It never +// consumes a sequence — that would punch a hole in every other subscriber's +// stream — and carries the connection's current position as its id instead, +// so ids on one connection never decrease. +function controlEvent(seq: number, type: string, properties: Record = {}): Sse.Event { + return frameEvent({ + id: formatID(GlobalEventReplay.epoch, seq), + data: JSON.stringify({ payload: { id: EventV2.ID.create(), type, properties } }), + }) +} + +// The node:http adapter interrupts a request fiber only on the response's +// "close" event, which Bun's node:http shim (the engine's runtime) never emits +// when a client aborts a streaming response — it emits the request's +// "aborted"/"close" and the socket's "close", and writes to the dead socket keep +// returning true. Without this the stream, its bus subscription and its queue +// outlive the client forever. Watch the socket itself; the response owns it +// for the stream's lifetime. +function clientGone(request: HttpServerRequest.HttpServerRequest) { + return Effect.gen(function* () { + const gone = yield* Deferred.make() + const done = () => Deferred.doneUnsafe(gone, Exit.void) + const socket = (request.source as { socket?: Partial } | undefined)?.socket + if (!socket || typeof socket.once !== "function" || typeof socket.off !== "function") return gone + if (socket.destroyed) { + done() + return gone + } + socket.once("close", done) + yield* Effect.addFinalizer(() => Effect.sync(() => socket.off?.("close", done))) + return gone + }) +} + +function eventResponse() { + return Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest + const cursor = cursorOf(request) + const sync = syncRequested(request) + const gone = yield* clientGone(request) + // Registration is eager and synchronous: a bus event emitted while the + // response body is starting is queued, never lost. + const queue = yield* Queue.unbounded() + // Unregister the listener and drop the undelivered frames with the queue. + // Idempotent: it runs at the stream's end and again at the request scope's. + const release = (subscription: Subscription) => + Effect.andThen(Effect.sync(() => subscription.close()), Queue.shutdown(queue)) + const subscription = yield* Effect.acquireRelease( + Effect.sync(() => { + const subscription = GlobalEventReplay.subscribe(cursor, { sync }) + // An unservable cursor is never silent: a reset frame comes first, + // carrying the current epoch and sequence, and the client resyncs. + if (subscription.reset) + Queue.offerUnsafe( + queue, + controlEvent(subscription.position, "server.cursor.unservable", { + epoch: GlobalEventReplay.epoch, + seq: subscription.position, + }), + ) + Queue.offerUnsafe(queue, controlEvent(subscription.position, "server.connected")) + for (const frame of subscription.replay) Queue.offerUnsafe(queue, frameEvent(frame)) + subscription.start((frame) => Queue.offerUnsafe(queue, frameEvent(frame))) + return subscription + }), + release, + ) + yield* Effect.logInfo("global event connected").pipe( + Effect.annotateLogs({ + subscribers: GlobalEventReplay.subscribers, + resume: subscription.reset ? `reset:${subscription.reset}` : cursor ? "replay" : "live", + replayed: subscription.replay.length, + sync, + }), + ) + // The heartbeat enters the connection's own queue, behind every frame up to + // the current sequence, so a backlogged connection never sees a heartbeat + // claim a sequence it has not been sent yet (that would read as a hole). + const heartbeat = Stream.tick("10 seconds").pipe( + Stream.drop(1), + Stream.mapEffect(() => + Effect.map(Clock.currentTimeMillis, (time) => { + const seq = GlobalEventReplay.current + Queue.offerUnsafe(queue, controlEvent(seq, "server.heartbeat", { time, seq })) + }), + ), + Stream.drain, + ) + + return HttpServerResponse.stream( + Stream.fromQueue(queue).pipe( + Stream.merge(heartbeat, { haltStrategy: "left" }), + Stream.interruptWhen(Deferred.await(gone)), + Stream.pipeThroughChannel(Sse.encode()), + Stream.encodeText, + // Release when the stream ends (client gone, or server shutdown), then + // log the disconnect with what remains (read after the release ran). + Stream.ensuring( + Effect.suspend(() => { + const dropped = Queue.sizeUnsafe(queue) + return release(subscription).pipe( + Effect.andThen( + Effect.suspend(() => + Effect.logInfo("global event disconnected").pipe( + Effect.annotateLogs({ subscribers: GlobalEventReplay.subscribers, dropped }), + ), + ), + ), + ) + }), + ), + ), + { + contentType: "text/event-stream", + headers: { + "Cache-Control": "no-cache, no-transform", + "X-Accel-Buffering": "no", + "X-Content-Type-Options": "nosniff", + }, + }, + ) + }) +} + +export const globalHandlers = HttpApiBuilder.group(RootHttpApi, "global", (handlers) => + Effect.gen(function* () { + const config = yield* Config.Service + const installation = yield* Installation.Service + const bridge = yield* EffectBridge.make() + + const health = Effect.fn("GlobalHttpApi.health")(function* () { + return { healthy: true as const, version: InstallationVersion } + }) + + const event = Effect.fn("GlobalHttpApi.event")(function* () { + return yield* eventResponse() + }) + + const configGet = Effect.fn("GlobalHttpApi.configGet")(function* () { + return yield* config.getGlobal() + }) + + const configUpdate = Effect.fn("GlobalHttpApi.configUpdate")(function* (ctx) { + const result = yield* config.updateGlobal(ctx.payload) + if (result.changed) bridge.fork(disposeAllInstancesAndEmitGlobalDisposed({ swallowErrors: true })) + return result.info + }) + + const dispose = Effect.fn("GlobalHttpApi.dispose")(function* () { + yield* disposeAllInstancesAndEmitGlobalDisposed() + return true + }) + + const upgrade = Effect.fn("GlobalHttpApi.upgrade")(function* (ctx: { payload: typeof GlobalUpgradeInput.Type }) { + const method = yield* installation.method() + if (method === "unknown") { + return HttpServerResponse.jsonUnsafe( + { success: false as const, error: "Unknown installation method" }, + { status: 400 }, + ) + } + const target = ctx.payload.target + const result = yield* installation.upgrade(method, target).pipe( + Effect.as({ success: true as const, version: target }), + Effect.catch((err) => + Effect.succeed({ + success: false as const, + error: err instanceof Error ? err.message : String(err), + }), + ), + ) + if (!result.success) return HttpServerResponse.jsonUnsafe(result, { status: 500 }) + GlobalBus.emit("event", { + directory: "global", + payload: { + type: Installation.Event.Updated.type, + properties: { version: target }, + }, + }) + return HttpServerResponse.jsonUnsafe(result) + }) + + return handlers + .handle("health", health) + .handleRaw("event", event) + .handle("configGet", configGet) + .handle("configUpdate", configUpdate) + .handle("dispose", dispose) + .handle("upgrade", upgrade) + }), +) diff --git a/packages/app-bundle/overlay/packages/opencode/src/tool/shell-output-throttle.ts b/packages/app-bundle/overlay/packages/opencode/src/tool/shell-output-throttle.ts new file mode 100644 index 000000000..69cd1c6fe --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/tool/shell-output-throttle.ts @@ -0,0 +1,63 @@ +// #1778 — throttle a running tool's output preview. Each preview update is a +// `message.part.updated` on the live stream carrying up to 30 KB of output, and +// a chatty command produces one per output chunk. The first update goes out at +// once; after that at most one per interval, a trailing update carrying the +// latest value; `flush` delivers whatever is still unsent (on completion). +// Time is effect's Clock, so TestClock drives it deterministically. +import { Clock, Duration, Effect, Fiber } from "effect" +import type * as Scope from "effect/Scope" + +export const OUTPUT_THROTTLE_MS = 200 + +export const makeOutputThrottle = Effect.fnUntraced(function* ( + send: (output: string) => Effect.Effect, + interval: Duration.Input = Duration.millis(OUTPUT_THROTTLE_MS), +) { + const scope: Scope.Scope = yield* Effect.scope + const every = Duration.toMillis(Duration.fromInputUnsafe(interval)) + let latest: string | undefined + let sent: string | undefined + let sentAt = Number.NEGATIVE_INFINITY + let trailing: Fiber.Fiber | undefined + + // Sends the latest value unless it already went out. + const deliver = Effect.suspend(() => { + if (latest === undefined || latest === sent) return Effect.void + const value = latest + sent = value + return Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => { + sentAt = now + return send(value) + }), + ) + }) + + /** A new preview value: sent now if the interval has passed, else by the trailing update. */ + const offer = (value: string) => + Effect.gen(function* () { + latest = value + if (trailing) return + const wait = sentAt + every - (yield* Clock.currentTimeMillis) + if (wait <= 0) return yield* deliver + trailing = yield* Effect.sleep(Duration.millis(wait)).pipe( + Effect.andThen( + Effect.suspend(() => { + trailing = undefined + return deliver + }), + ), + Effect.forkIn(scope), + ) + }) + + /** Deliver anything unsent now (the guaranteed final update) and drop the pending trailing one. */ + const flush = Effect.gen(function* () { + const pending = trailing + trailing = undefined + if (pending) yield* Fiber.interrupt(pending) + yield* deliver + }) + + return { offer, flush } +}) diff --git a/packages/app-bundle/overlay/packages/opencode/src/tool/shell.ts b/packages/app-bundle/overlay/packages/opencode/src/tool/shell.ts new file mode 100644 index 000000000..4d80b0929 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/tool/shell.ts @@ -0,0 +1,646 @@ +import { Effect, Fiber, Stream } from "effect" +import os from "os" +import { createWriteStream } from "node:fs" +import * as Tool from "./tool" +import path from "path" +import { containsPath, type InstanceContext } from "../project/instance-context" +import { InstanceState } from "@/effect/instance-state" +import { lazy } from "@/util/lazy" +import { Language, type Node } from "web-tree-sitter" + +import { FSUtil } from "@opencode-ai/core/fs-util" +import { fileURLToPath } from "url" +import { Config } from "@/config/config" +import { RuntimeFlags } from "@/effect/runtime-flags" +import { Shell } from "@opencode-ai/core/shell" +import { ShellID } from "./shell/id" + +import * as Truncate from "./truncate" +import { Plugin } from "@/plugin" +import { ChildProcess } from "effect/unstable/process" +import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner" +import { ShellPrompt, type Parameters } from "./shell/prompt" +import { makeOutputThrottle } from "./shell-output-throttle" +import { BashArity } from "@/permission/arity" + +export { Parameters } from "./shell/prompt" + +const MAX_METADATA_LENGTH = 30_000 +const CWD = new Set(["cd", "chdir", "popd", "pushd", "push-location", "set-location"]) +const FILES = new Set([ + ...CWD, + "rm", + "cp", + "mv", + "mkdir", + "touch", + "chmod", + "chown", + "cat", + // Leave PowerShell aliases out for now. Common ones like cat/cp/mv/rm/mkdir + // already hit the entries above, and alias normalization should happen in one + // place later so we do not risk double-prompting. + "get-content", + "set-content", + "add-content", + "copy-item", + "move-item", + "remove-item", + "new-item", + "rename-item", +]) +const CMD_FILES = new Set([ + "copy", + "del", + "dir", + "erase", + "md", + "mkdir", + "move", + "rd", + "ren", + "rename", + "rmdir", + "type", +]) +const FLAGS = new Set(["-destination", "-literalpath", "-path"]) +const SWITCHES = new Set(["-confirm", "-debug", "-force", "-nonewline", "-recurse", "-verbose", "-whatif"]) + +type Part = { + type: string + text: string +} + +type Scan = { + dirs: Set + patterns: Set + always: Set +} + +type Chunk = { + text: string + size: number +} + +const resolveWasm = (asset: string) => { + if (asset.startsWith("file://")) return fileURLToPath(asset) + if (asset.startsWith("/") || /^[a-z]:/i.test(asset)) return asset + const url = new URL(asset, import.meta.url) + return fileURLToPath(url) +} + +function parts(node: Node) { + const out: Part[] = [] + for (let i = 0; i < node.childCount; i++) { + const child = node.child(i) + if (!child) continue + if (child.type === "command_elements") { + for (let j = 0; j < child.childCount; j++) { + const item = child.child(j) + if (!item || item.type === "command_argument_sep" || item.type === "redirection") continue + out.push({ type: item.type, text: item.text }) + } + continue + } + if ( + child.type !== "command_name" && + child.type !== "command_name_expr" && + child.type !== "word" && + child.type !== "string" && + child.type !== "raw_string" && + child.type !== "concatenation" + ) { + continue + } + out.push({ type: child.type, text: child.text }) + } + return out +} + +function source(node: Node) { + return (node.parent?.type === "redirected_statement" ? node.parent.text : node.text).trim() +} + +function commands(node: Node) { + return node.descendantsOfType("command").filter((child): child is Node => Boolean(child)) +} + +function unquote(text: string) { + if (text.length < 2) return text + const first = text[0] + const last = text[text.length - 1] + if ((first === '"' || first === "'") && first === last) return text.slice(1, -1) + return text +} + +function home(text: string) { + if (text === "~") return os.homedir() + if (text.startsWith("~/") || text.startsWith("~\\")) return path.join(os.homedir(), text.slice(2)) + return text +} + +function envValue(key: string) { + if (process.platform !== "win32") return process.env[key] + const name = Object.keys(process.env).find((item) => item.toLowerCase() === key.toLowerCase()) + return name ? process.env[name] : undefined +} + +function auto(key: string, cwd: string, shell: string) { + const name = key.toUpperCase() + if (name === "HOME") return os.homedir() + if (name === "PWD") return cwd + if (name === "PSHOME") return path.dirname(shell) +} + +function expand(text: string, cwd: string, shell: string) { + const out = unquote(text) + .replace(/\$\{env:([^}]+)\}/gi, (_, key: string) => envValue(key) || "") + .replace(/\$env:([A-Za-z_][A-Za-z0-9_]*)/gi, (_, key: string) => envValue(key) || "") + .replace(/\$(HOME|PWD|PSHOME)(?=$|[\\/])/gi, (_, key: string) => auto(key, cwd, shell) || "") + return home(out) +} + +function provider(text: string) { + const match = text.match(/^([A-Za-z]+)::(.*)$/) + if (match) { + if (match[1].toLowerCase() !== "filesystem") return + return match[2] + } + const prefix = text.match(/^([A-Za-z]+):(.*)$/) + if (!prefix) return text + if (prefix[1].length === 1) return text + return +} + +function dynamic(text: string, ps: boolean) { + if (text.startsWith("(") || text.startsWith("@(")) return true + if (text.includes("$(") || text.includes("${") || text.includes("`")) return true + if (ps) return /\$(?!env:)/i.test(text) + return text.includes("$") +} + +function prefix(text: string) { + const match = /[?*[]/.exec(text) + if (!match) return text + if (match.index === 0) return + return text.slice(0, match.index) +} + +function pathArgs(list: Part[], ps: boolean, cmd = false) { + if (!ps) { + return list + .slice(1) + .filter( + (item) => + !item.text.startsWith("-") && + !(cmd && item.text.startsWith("/")) && + !(list[0]?.text === "chmod" && item.text.startsWith("+")), + ) + .map((item) => item.text) + } + + const out: string[] = [] + let want = false + for (const item of list.slice(1)) { + if (want) { + out.push(item.text) + want = false + continue + } + if (item.type === "command_parameter") { + const flag = item.text.toLowerCase() + if (SWITCHES.has(flag)) continue + want = FLAGS.has(flag) + continue + } + out.push(item.text) + } + return out +} + +function preview(text: string) { + if (text.length <= MAX_METADATA_LENGTH) return text + return "...\n\n" + text.slice(-MAX_METADATA_LENGTH) +} + +function tail(text: string, maxLines: number, maxBytes: number) { + const lines = text.split("\n") + if (lines.length <= maxLines && Buffer.byteLength(text, "utf-8") <= maxBytes) { + return { + text, + cut: false, + } + } + + const out: string[] = [] + let bytes = 0 + for (let i = lines.length - 1; i >= 0 && out.length < maxLines; i--) { + const size = Buffer.byteLength(lines[i], "utf-8") + (out.length > 0 ? 1 : 0) + if (bytes + size > maxBytes) { + if (out.length === 0) { + const buf = Buffer.from(lines[i], "utf-8") + let start = buf.length - maxBytes + if (start < 0) start = 0 + while (start < buf.length && (buf[start] & 0xc0) === 0x80) start++ + out.unshift(buf.subarray(start).toString("utf-8")) + } + break + } + out.unshift(lines[i]) + bytes += size + } + return { + text: out.join("\n"), + cut: true, + } +} + +const parse = Effect.fn("ShellTool.parse")(function* (command: string, ps: boolean) { + const tree = yield* Effect.promise(() => parser().then((p) => (ps ? p.ps : p.bash).parse(command))) + if (!tree) throw new Error("Failed to parse command") + return tree +}) + +const ask = Effect.fn("ShellTool.ask")(function* (ctx: Tool.Context, scan: Scan, input: { command: string }) { + if (scan.dirs.size > 0) { + const directories = Array.from(scan.dirs) + const globs = directories.map((dir) => { + if (process.platform === "win32") return FSUtil.normalizePathPattern(path.join(dir, "*")) + return path.join(dir, "*") + }) + yield* ctx.ask({ + permission: "external_directory", + patterns: globs, + always: globs, + metadata: { + command: input.command, + directories, + patterns: globs, + }, + }) + } + + if (scan.patterns.size === 0) return + yield* ctx.ask({ + permission: ShellID.ToolID, + patterns: Array.from(scan.patterns), + always: Array.from(scan.always), + metadata: { + command: input.command, + }, + }) +}) + +function cmd(shell: string, command: string, cwd: string, env: NodeJS.ProcessEnv) { + if (process.platform === "win32" && Shell.ps(shell)) { + return ChildProcess.make(shell, ["-NoLogo", "-NoProfile", "-NonInteractive", "-Command", command], { + cwd, + env, + stdin: "ignore", + detached: false, + }) + } + + return ChildProcess.make(command, [], { + shell, + cwd, + env, + stdin: "ignore", + detached: process.platform !== "win32", + }) +} +const parser = lazy(async () => { + const { Parser } = await import("web-tree-sitter") + const { default: treeWasm } = await import("web-tree-sitter/tree-sitter.wasm" as string, { + with: { type: "wasm" }, + }) + const treePath = resolveWasm(treeWasm) + await Parser.init({ + locateFile() { + return treePath + }, + }) + const { default: bashWasm } = await import("tree-sitter-bash/tree-sitter-bash.wasm" as string, { + with: { type: "wasm" }, + }) + const { default: psWasm } = await import("tree-sitter-powershell/tree-sitter-powershell.wasm" as string, { + with: { type: "wasm" }, + }) + const bashPath = resolveWasm(bashWasm) + const psPath = resolveWasm(psWasm) + const [bashLanguage, psLanguage] = await Promise.all([Language.load(bashPath), Language.load(psPath)]) + const bash = new Parser() + bash.setLanguage(bashLanguage) + const ps = new Parser() + ps.setLanguage(psLanguage) + return { bash, ps } +}) + +export const ShellTool = Tool.define( + ShellID.ToolID, + Effect.gen(function* () { + const config = yield* Config.Service + const spawner = yield* ChildProcessSpawner + const fs = yield* FSUtil.Service + const trunc = yield* Truncate.Service + const plugin = yield* Plugin.Service + const flags = yield* RuntimeFlags.Service + const defaultTimeoutMs = flags.bashDefaultTimeoutMs ?? 2 * 60 * 1000 + + const cygpath = Effect.fn("ShellTool.cygpath")(function* (shell: string, text: string) { + const lines = yield* spawner + .lines(ChildProcess.make(shell, ["-lc", 'cygpath -w -- "$1"', "_", text])) + .pipe(Effect.catch(() => Effect.succeed([] as string[]))) + const file = lines[0]?.trim() + if (!file) return + return FSUtil.normalizePath(file) + }) + + const resolvePath = Effect.fn("ShellTool.resolvePath")(function* (text: string, root: string, shell: string) { + if (process.platform === "win32") { + if (Shell.posix(shell) && text.startsWith("/") && FSUtil.windowsPath(text) === text) { + const file = yield* cygpath(shell, text) + if (file) return file + } + return FSUtil.normalizePath(path.resolve(root, FSUtil.windowsPath(text))) + } + return path.resolve(root, text) + }) + + const argPath = Effect.fn("ShellTool.argPath")(function* (arg: string, cwd: string, ps: boolean, shell: string) { + const text = ps ? expand(arg, cwd, shell) : home(unquote(arg)) + const file = text && prefix(text) + if (!file || dynamic(file, ps)) return + const next = ps ? provider(file) : file + if (!next) return + return yield* resolvePath(next, cwd, shell) + }) + + const collect = Effect.fn("ShellTool.collect")(function* ( + root: Node, + cwd: string, + ps: boolean, + shell: string, + instance: InstanceContext, + ) { + const scan: Scan = { + dirs: new Set(), + patterns: new Set(), + always: new Set(), + } + const shellKind = ShellID.toKind(Shell.name(shell)) + + for (const node of commands(root)) { + const command = parts(node) + const tokens = command.map((item) => item.text) + const cmd = ps || shellKind === "cmd" ? tokens[0]?.toLowerCase() : tokens[0] + + if (cmd && (FILES.has(cmd) || (shellKind === "cmd" && CMD_FILES.has(cmd)))) { + for (const arg of pathArgs(command, ps, shellKind === "cmd")) { + const resolved = yield* argPath(arg, cwd, ps, shell) + yield* Effect.logInfo("resolved path", { arg, resolved }) + if (!resolved || containsPath(resolved, instance)) continue + const dir = (yield* fs.isDir(resolved)) ? resolved : path.dirname(resolved) + scan.dirs.add(dir) + } + } + + if (tokens.length && (!cmd || !CWD.has(cmd))) { + scan.patterns.add(source(node)) + scan.always.add(BashArity.prefix(tokens).join(" ") + " *") + } + } + + return scan + }) + + const shellEnv = Effect.fn("ShellTool.shellEnv")(function* (ctx: Tool.Context, cwd: string) { + const extra = yield* plugin.trigger( + "shell.env", + { cwd, sessionID: ctx.sessionID, callID: ctx.callID }, + { env: {} }, + ) + return { + ...process.env, + ...extra.env, + } + }) + + const run = Effect.fn("ShellTool.run")(function* ( + input: { + shell: string + command: string + cwd: string + env: NodeJS.ProcessEnv + timeout: number + }, + ctx: Tool.Context, + ) { + const limits = yield* trunc.limits() + const keep = limits.maxBytes * 2 + let full = "" + let last = "" + const list: Chunk[] = [] + let used = 0 + let file = "" + let sink: ReturnType | undefined + let cut = false + let expired = false + let aborted = false + + const closeSink = Effect.fnUntraced(function* () { + const stream = sink + if (!stream) return + sink = undefined + if (stream.destroyed || stream.closed) return + yield* Effect.promise( + () => + new Promise((resolve) => { + let settled = false + const done = () => { + if (settled) return + settled = true + stream.off("close", done) + stream.off("error", done) + stream.off("finish", done) + resolve() + } + stream.once("close", done) + stream.once("error", done) + stream.once("finish", done) + stream.end(done) + }), + ).pipe(Effect.catch(() => Effect.void)) + }) + + yield* ctx.metadata({ + metadata: { + output: "", + }, + }) + + // #1778 (amicode overlay): every preview update is a live-stream frame of + // up to 30 KB, and a chatty command makes one per output chunk. Throttle + // them (first at once, then <= 1 per 200 ms, trailing) and flush the final + // output once the command is done. + const code: number | null = yield* Effect.scoped( + Effect.gen(function* () { + yield* Effect.addFinalizer(closeSink) + const throttle = yield* makeOutputThrottle((output) => ctx.metadata({ metadata: { output } })) + const handle = yield* spawner.spawn(cmd(input.shell, input.command, input.cwd, input.env)) + + const reader = yield* Effect.forkScoped( + Stream.runForEach(Stream.decodeText(handle.all), (chunk) => { + const size = Buffer.byteLength(chunk, "utf-8") + list.push({ text: chunk, size }) + used += size + while (used > keep && list.length > 1) { + const item = list.shift() + if (!item) break + used -= item.size + cut = true + } + + last = preview(last + chunk) + + if (file) { + sink?.write(chunk) + } else { + full += chunk + if (Buffer.byteLength(full, "utf-8") > limits.maxBytes) { + return trunc.write(full).pipe( + Effect.andThen((next) => + Effect.sync(() => { + file = next + cut = true + sink = createWriteStream(next, { flags: "a" }) + full = "" + }), + ), + Effect.andThen(throttle.offer(last)), + ) + } + } + + return throttle.offer(last) + }), + ) + + const abort = Effect.callback((resume) => { + if (ctx.abort.aborted) return resume(Effect.void) + const handler = () => resume(Effect.void) + ctx.abort.addEventListener("abort", handler, { once: true }) + return Effect.sync(() => ctx.abort.removeEventListener("abort", handler)) + }) + + const timeout = Effect.sleep(`${input.timeout + 100} millis`) + + const exit = yield* Effect.raceAll([ + handle.exitCode.pipe(Effect.map((code) => ({ kind: "exit" as const, code }))), + abort.pipe(Effect.map(() => ({ kind: "abort" as const, code: null }))), + timeout.pipe(Effect.map(() => ({ kind: "timeout" as const, code: null }))), + ]) + + if (exit.kind === "abort") { + aborted = true + yield* handle.kill({ forceKillAfter: "3 seconds" }).pipe(Effect.orDie) + } + if (exit.kind === "timeout") { + expired = true + yield* handle.kill({ forceKillAfter: "3 seconds" }).pipe(Effect.orDie) + } + + // Stop the reader (as the scope's close would), then the final output + // always goes out: no update can race behind the flush. + yield* Fiber.interrupt(reader) + yield* throttle.flush + + return exit.kind === "exit" ? exit.code : null + }), + ).pipe(Effect.orDie) + + const meta: string[] = [] + if (expired) { + meta.push( + `shell tool terminated command after exceeding timeout ${input.timeout} ms. If this command is expected to take longer and is not waiting for interactive input, retry with a larger timeout value in milliseconds.`, + ) + } + if (aborted) meta.push("User aborted the command") + const raw = list.map((item) => item.text).join("") + const end = tail(raw, limits.maxLines, limits.maxBytes) + if (end.cut) cut = true + if (!file && end.cut) { + file = yield* trunc.write(raw) + } + + let output = end.text + if (!output) output = "(no output)" + + if (cut && file) { + output = `...output truncated...\n\nFull output saved to: ${file}\n\n` + output + } + + if (meta.length > 0) { + output += "\n\n\n" + meta.join("\n") + "\n" + } + return { + title: input.command, + metadata: { + output: last || preview(output), + exit: code, + truncated: cut, + ...(cut && file ? { outputPath: file } : {}), + }, + output, + } + }) + + return () => + Effect.gen(function* () { + const cfg = yield* config.get() + const shell = Shell.acceptable(cfg.shell) + const name = Shell.name(shell) + const limits = yield* trunc.limits() + const prompt = ShellPrompt.render(name, process.platform, limits, defaultTimeoutMs) + yield* Effect.logInfo("shell tool using shell", { shell }) + + return { + description: prompt.description, + parameters: prompt.parameters, + execute: (params: Parameters, ctx: Tool.Context) => + Effect.gen(function* () { + const instanceCtx = yield* InstanceState.context + const cwd = params.workdir + ? yield* resolvePath(params.workdir, instanceCtx.directory, shell) + : instanceCtx.directory + if (params.timeout !== undefined && params.timeout < 0) { + throw new Error(`Invalid timeout value: ${params.timeout}. Timeout must be a positive number.`) + } + const timeout = params.timeout ?? defaultTimeoutMs + const ps = Shell.ps(shell) + yield* Effect.scoped( + Effect.gen(function* () { + const tree = yield* Effect.acquireRelease(parse(params.command, ps), (tree) => + Effect.sync(() => tree.delete()), + ) + const scan = yield* collect(tree.rootNode, cwd, ps, shell, instanceCtx) + if (!containsPath(cwd, instanceCtx)) scan.dirs.add(cwd) + yield* ask(ctx, scan, params) + }), + ) + + return yield* run( + { + shell, + command: params.command, + cwd, + env: yield* shellEnv(ctx, cwd), + timeout, + }, + ctx, + ) + }), + } + }) + }), +) diff --git a/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts b/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts new file mode 100644 index 000000000..de2e8c254 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts @@ -0,0 +1,262 @@ +// #1778 — sync copies are opt-in per global-stream subscription. The workspace +// control plane is the one subscriber that replays them, so it must ask for +// them; the remote here behaves like an engine with the opt-in filter (engine- +// shaped frames, `id: :`, sync copies only for `?sync=true`). +// Harness trimmed from workspace.test.ts (upstream) — same layers and helpers. +import { afterEach, beforeEach, describe, expect, mock } from "bun:test" +import Http from "node:http" +import { NodeHttpServer } from "@effect/platform-node" +import { Effect, Layer } from "effect" +import { HttpServer, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import { eq } from "drizzle-orm" +import { GlobalBus, type GlobalEvent } from "@/bus/global" +import { Database } from "@opencode-ai/core/database/database" +import { ProjectV2 } from "@opencode-ai/core/project" +import { Session as SessionNs } from "@/session/session" +import { SessionID } from "@/session/schema" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { EventSequenceTable } from "@opencode-ai/core/event/sql" +import { resetDatabase } from "../fixture/db" +import { disposeAllInstances, provideTmpdirInstance, requireInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" +import { registerAdapter } from "../../src/control-plane/adapters" +import { WorkspaceV2 } from "@opencode-ai/core/workspace" +import { WorkspaceTable } from "@opencode-ai/core/control-plane/workspace.sql" +import type { WorkspaceAdapter } from "../../src/control-plane/types" +import * as Workspace from "../../src/control-plane/workspace" +import { InstanceStore } from "@/project/instance-store" +import { InstanceBootstrap } from "@/project/bootstrap" +import { RuntimeFlags } from "@/effect/runtime-flags" +import { Ripgrep } from "@opencode-ai/core/ripgrep" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" + +const originalWorkspacesFlag = process.env.OPENCODE_EXPERIMENTAL_WORKSPACES + +const it = testEffect( + Layer.mergeAll( + NodeHttpServer.layer(Http.createServer, { host: "127.0.0.1", port: 0 }), + AppNodeBuilder.build( + LayerNode.group([ + Workspace.node, + SessionNs.node, + SessionProjector.node, + Database.node, + InstanceStore.node, + Ripgrep.node, + ]), + [ + [RuntimeFlags.node, RuntimeFlags.layer({ experimentalWorkspaces: true })], + [ + InstanceStore.bootstrapNode, + Layer.succeed(InstanceBootstrap.Service, InstanceBootstrap.Service.of({ run: Effect.void })), + ], + ], + ), + ), +) + +beforeEach(() => { + process.env.OPENCODE_EXPERIMENTAL_WORKSPACES = "true" +}) + +afterEach(async () => { + mock.restore() + await disposeAllInstances() + if (originalWorkspacesFlag === undefined) delete process.env.OPENCODE_EXPERIMENTAL_WORKSPACES + else process.env.OPENCODE_EXPERIMENTAL_WORKSPACES = originalWorkspacesFlag + await resetDatabase() +}) + +const unique = (prefix: string) => `${prefix}-${Math.random().toString(36).slice(2)}` + +function remoteAdapter(url: string): WorkspaceAdapter { + return { + name: "remote", + description: "remote", + configure: (info) => info, + async create() {}, + async remove() {}, + target: () => ({ type: "remote", url }), + } +} + +/** An engine-shaped global stream: `id: :` + `data:` per frame, then end. */ +function engineStream(frames: { seq: number; event: unknown }[]) { + const encoder = new TextEncoder() + return new Response( + new ReadableStream({ + start(controller) { + for (const frame of frames) + controller.enqueue(encoder.encode(`id: ep0ch:${frame.seq}\ndata: ${JSON.stringify(frame.event)}\n\n`)) + controller.close() + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ) +} + +function captureGlobalEvents() { + const events: GlobalEvent[] = [] + const handler = (event: GlobalEvent) => events.push(event) + GlobalBus.on("event", handler) + return { events, dispose: () => GlobalBus.off("event", handler) } +} + +function eventuallyEffect(effect: Effect.Effect, timeout = 3000) { + return Effect.gen(function* () { + const started = Date.now() + let last: unknown + while (Date.now() - started < timeout) { + const exit = yield* Effect.exit(effect) + if (exit._tag === "Success") return + last = exit.cause + yield* Effect.sleep("10 millis") + } + throw last ?? new Error("Timed out waiting for condition") + }) +} + +const insertWorkspace = (info: Workspace.Info) => + Database.Service.use(({ db }) => + db + .insert(WorkspaceTable) + .values({ + id: info.id, + type: info.type, + branch: info.branch, + name: info.name, + directory: info.directory, + extra: info.extra, + project_id: info.projectID, + time_used: info.timeUsed, + }) + .run() + .pipe(Effect.orDie), + ) + +const attachSessionToWorkspace = (sessionID: SessionID, workspaceID: WorkspaceV2.ID) => + Database.Service.use(({ db }) => + db + .update(SessionTable) + .set({ workspace_id: workspaceID }) + .where(eq(SessionTable.id, sessionID)) + .run() + .pipe(Effect.orDie), + ) + +const sessionSequence = (sessionID: SessionID) => + Database.Service.use(({ db }) => + db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, sessionID)) + .get() + .pipe( + Effect.orDie, + Effect.map((row) => row?.seq), + ), + ) + +describe("workspace control plane — sync copies are opt-in (#1778)", () => { + it.live("the control-plane subscription requests sync copies and replays them into the local store", () => { + const requests: { url: URL; syncCopies: string | undefined }[] = [] + let remote: { sessionID: SessionID; info: SessionNs.Info; seq: number } | undefined + return Effect.gen(function* () { + yield* HttpServer.serveEffect()( + Effect.gen(function* () { + const req = yield* HttpServerRequest.HttpServerRequest + const url = new URL(req.url, "http://localhost") + if (url.pathname === "/optin/global/event") { + const syncCopies = req.headers["x-opencode-sync-copies"] + requests.push({ url, syncCopies }) + if (!remote) return HttpServerResponse.fromWeb(engineStream([])) + const id = `evt_${unique("optin")}` + const event = { type: "session.updated", properties: { sessionID: remote.sessionID, info: remote.info } } + // the engine's twin emission: the durable event, then its sync copy (same sequence) + const frames = [{ seq: 7, event: { directory: "remote-dir", payload: { id, ...event } } }] + if (syncCopies === "true") + frames.push({ + seq: 7, + event: { + directory: "remote-dir", + payload: { + id, + type: "sync", + syncEvent: { + id, + aggregateID: remote.sessionID, + seq: remote.seq, + type: "session.updated.1", + data: event.properties, + }, + }, + } as never, + }) + return HttpServerResponse.fromWeb(engineStream(frames)) + } + if (url.pathname === "/optin/sync/history") return HttpServerResponse.fromWeb(Response.json([])) + return HttpServerResponse.text("unexpected", { status: 500 }) + }), + ) + const url = HttpServer.formatAddress((yield* HttpServer.HttpServer).address) + yield* provideTmpdirInstance( + () => + Effect.gen(function* () { + const workspace = yield* Workspace.Service + const sessions = yield* SessionNs.Service + const instance = yield* requireInstance + const captured = captureGlobalEvents() + try { + const type = unique("optin") + const info: Workspace.Info = { + id: WorkspaceV2.ID.ascending(), + type, + name: unique("workspace"), + branch: null, + directory: null, + extra: null, + projectID: instance.project.id as ProjectV2.ID, + timeUsed: Date.now(), + } + yield* insertWorkspace(info) + registerAdapter(instance.project.id, type, remoteAdapter(`${url}/optin`)) + const session = yield* sessions.create({ title: "before sync" }) + yield* attachSessionToWorkspace(session.id, info.id) + remote = { + sessionID: session.id, + info: { ...session, workspaceID: info.id, title: "from the remote's sync copy" }, + seq: ((yield* sessionSequence(session.id)) ?? -1) + 1, + } + + yield* workspace.startWorkspaceSyncing(instance.project.id) + + // replayed correctly: the sync copy reached the local store + yield* eventuallyEffect( + Effect.gen(function* () { + expect((yield* sessions.get(session.id).pipe(Effect.orDie)).title).toBe("from the remote's sync copy") + }), + ) + expect(requests.length).toBeGreaterThan(0) + expect(requests.every((request) => request.syncCopies === "true")).toBe(true) + // the opt-in rides a header: the event URL keeps its routed form + expect( + requests.every((request) => request.url.pathname === "/optin/global/event" && !request.url.search), + ).toBe(true) + // and the remote's frames are still forwarded onto the local bus + expect( + captured.events.some( + (event) => event.workspace === info.id && event.payload.type === "session.updated", + ), + ).toBe(true) + yield* workspace.remove(info.id) + } finally { + captured.dispose() + } + }), + { git: true }, + ) + }) + }) +}) diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/global-event-replay.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/global-event-replay.test.ts new file mode 100644 index 000000000..b63645966 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/server/global-event-replay.test.ts @@ -0,0 +1,85 @@ +// #1776 — the global stream's replay buffer: one shared buffer bounded by +// bytes and age, evicting oldest-first, both bounds configurable by environment. +import { describe, expect, test } from "bun:test" +import path from "node:path" +import { DEFAULT_BOUNDS, ReplayBuffer, boundsFromEnv } from "../../src/bus/global-replay" + +const frameOf = (bytes: number) => "x".repeat(bytes) + +function buffer(bounds: { maxBytes: number; maxAgeMs: number }, clock = { now: 0 }) { + return new ReplayBuffer({ epoch: "ep0ch", bounds, now: () => clock.now }) +} + +const seqs = (b: ReplayBuffer, cursor: string) => { + const sub = b.subscribe(cursor) + sub.close() + return sub.reset ?? sub.replay.map((frame) => frame.seq) +} + +describe("global event replay buffer — bounds (#1776)", () => { + test("evicts oldest-first once retained bytes exceed the byte bound", () => { + const b = buffer({ maxBytes: 30, maxAgeMs: 60_000 }) + for (const seq of [1, 2, 3]) b.push(seq, frameOf(10)) + expect(seqs(b, "ep0ch:0")).toEqual([1, 2, 3]) + + b.push(4, frameOf(10)) // 40 bytes > 30: the oldest frame goes + expect(seqs(b, "ep0ch:0")).toBe("evicted") + expect(seqs(b, "ep0ch:1")).toEqual([2, 3, 4]) + + b.push(5, frameOf(25)) // 55 bytes: evict 2, 3, 4 (oldest first) until within bound + expect(seqs(b, "ep0ch:3")).toBe("evicted") + expect(seqs(b, "ep0ch:4")).toEqual([5]) + }) + + test("evicts oldest-first once frames are older than the age bound", () => { + const clock = { now: 0 } + const b = buffer({ maxBytes: 1 << 20, maxAgeMs: 1_500 }, clock) + b.push(1, frameOf(10)) + clock.now = 1_000 + b.push(2, frameOf(10)) + clock.now = 1_400 + expect(seqs(b, "ep0ch:0")).toEqual([1, 2]) + + clock.now = 2_000 // frame 1 is 2000 ms old (> 1500), frame 2 is 1000 ms old + expect(seqs(b, "ep0ch:0")).toBe("evicted") + expect(seqs(b, "ep0ch:1")).toEqual([2]) + + clock.now = 3_000 // both aged out; a cursor at the current sequence is still servable + b.push(3, frameOf(10)) + expect(seqs(b, "ep0ch:1")).toBe("evicted") + expect(seqs(b, "ep0ch:2")).toEqual([3]) + expect(seqs(b, "ep0ch:3")).toEqual([]) + }) + + test("both bounds are read from the environment, defaulting to 32 MiB and 5 minutes", () => { + expect(DEFAULT_BOUNDS).toEqual({ maxBytes: 32 * 1024 * 1024, maxAgeMs: 5 * 60_000 }) + expect( + boundsFromEnv({ + OPENCODE_GLOBAL_EVENT_REPLAY_MAX_BYTES: "1024", + OPENCODE_GLOBAL_EVENT_REPLAY_MAX_AGE_MS: "60000", + }), + ).toEqual({ maxBytes: 1024, maxAgeMs: 60_000 }) + expect(boundsFromEnv({})).toEqual(DEFAULT_BOUNDS) + expect( + boundsFromEnv({ OPENCODE_GLOBAL_EVENT_REPLAY_MAX_BYTES: "lots", OPENCODE_GLOBAL_EVENT_REPLAY_MAX_AGE_MS: "-5" }), + ).toEqual(DEFAULT_BOUNDS) + }) + + test("the engine's shared buffer takes its bounds from the process environment", () => { + const result = Bun.spawnSync({ + cmd: [ + process.execPath, + "-e", + 'const { GlobalEventReplay } = await import("./src/bus/global-replay.ts"); console.log(JSON.stringify(GlobalEventReplay.bounds))', + ], + cwd: path.join(import.meta.dir, "..", ".."), + env: { + ...process.env, + OPENCODE_GLOBAL_EVENT_REPLAY_MAX_BYTES: "4096", + OPENCODE_GLOBAL_EVENT_REPLAY_MAX_AGE_MS: "2500", + }, + }) + expect(result.stderr.toString()).toBe("") + expect(JSON.parse(result.stdout.toString())).toEqual({ maxBytes: 4096, maxAgeMs: 2500 }) + }) +}) diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts new file mode 100644 index 000000000..970d82786 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -0,0 +1,771 @@ +// #1776 — the global event stream is sequenced and replayable: every frame +// carries an `:` SSE id, a resume cursor replays the gap, an +// unservable cursor gets an explicit reset, heartbeats carry time + seq, and a +// client abort releases its subscription. Exercised with a raw fetch client +// (exact headers, real aborts) against the in-process server, which runs on the +// same Bun node:http shim as the compiled binary. +import { NodeHttpServer } from "@effect/platform-node" +import Http from "node:http" +import { describe, expect } from "bun:test" +import { Clock, Context, Effect, Fiber, Layer, Logger, Option, Queue, References, Scope } from "effect" +import * as TestClock from "effect/testing/TestClock" +import { HttpRouter, HttpServer } from "effect/unstable/http" +import { HttpApiBuilder } from "effect/unstable/httpapi" +import { Auth } from "../../src/auth" +import { GlobalBus, type GlobalEvent } from "../../src/bus/global" +import { GlobalEventReplay } from "../../src/bus/global-replay" +import { Config } from "../../src/config/config" +import { Installation } from "../../src/installation" +import { MoveSession } from "@opencode-ai/core/control-plane/move-session" +import { ServerAuth } from "../../src/server/auth" +import { RootHttpApi } from "../../src/server/routes/instance/httpapi/api" +import { GlobalPaths } from "../../src/server/routes/instance/httpapi/groups/global" +import { controlHandlers } from "../../src/server/routes/instance/httpapi/handlers/control" +import { controlPlaneHandlers } from "../../src/server/routes/instance/httpapi/handlers/control-plane" +import { globalHandlers } from "../../src/server/routes/instance/httpapi/handlers/global" +import { authorizationLayer } from "../../src/server/routes/instance/httpapi/middleware/authorization" +import { schemaErrorLayer } from "../../src/server/routes/instance/httpapi/middleware/schema-error" +import { testEffect } from "../lib/effect" +import { OUTPUT_THROTTLE_MS, makeOutputThrottle } from "../../src/tool/shell-output-throttle" + +const apiLayer = HttpRouter.serve( + HttpApiBuilder.layer(RootHttpApi).pipe( + Layer.provide([controlHandlers, controlPlaneHandlers, globalHandlers]), + Layer.provide([authorizationLayer, schemaErrorLayer]), + // Raw HttpApi routes expose an opaque handler context at the request boundary. + // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion + HttpRouter.provideRequest(Layer.succeedContext(Context.empty() as Context.Context)), + ), + { disableListenLog: true, disableLogger: true }, +).pipe( + // Bound to 127.0.0.1 — the address the raw fetch client dials — not + // `NodeHttpServer.layerTest`'s wildcard. The kernel only skips ephemeral ports + // already bound on the SAME address, so a wildcard bind can share its port + // with another process's 127.0.0.1 listener, which then answers our requests. + Layer.provideMerge(NodeHttpServer.layer(Http.createServer, { host: "127.0.0.1", port: 0 })), + Layer.provide(Layer.mock(Auth.Service)({})), + Layer.provide(Layer.mock(Config.Service)({})), + Layer.provide(Layer.mock(MoveSession.Service)({})), + Layer.provide(Layer.mock(Installation.Service)({})), + Layer.provide(ServerAuth.Config.configLayer({ password: Option.none(), username: "opencode" })), + Layer.provide( + Logger.layer([ + Logger.make((options) => { + logs.push({ + message: ([] as unknown[]).concat(options.message).join(" "), + annotations: { ...options.fiber.getRef(References.CurrentLogAnnotations) }, + }) + }), + ]), + ), +) +const it = testEffect(apiLayer) + +// The server's own log lines (the stream's connect/disconnect census). +const logs: { message: string; annotations: Record }[] = [] +const logged = (message: string) => logs.filter((line) => line.message === message) + +/** Poll until `probe` holds (true), or give up after `ms` (false). */ +const eventually = (probe: () => boolean, ms = 3_000) => + Effect.gen(function* () { + while (!probe()) yield* Effect.sleep("20 millis") + return true + }).pipe(Effect.timeoutOrElse({ duration: ms, orElse: () => Effect.succeed(false) })) + +const ID = /^([A-Za-z0-9_-]+):(\d+)$/ + +/** One SSE frame; `raw` is its `data` exactly as sent, `bytes` the whole frame on the wire. */ +type Frame = { id: string | undefined; event: string | undefined; data: any; raw: string; bytes: number } + +/** Split an SSE id into its epoch and sequence (fails the test if malformed). */ +function idOf(frame: Frame) { + const match = ID.exec(frame.id ?? "") + if (!match) + throw new Error(`frame id ${JSON.stringify(frame.id)} is not : (${JSON.stringify(frame.data)})`) + return { epoch: match[1], seq: Number(match[2]) } +} + +function parseFrame(block: string): Frame { + const frame: Frame = { + id: undefined, + event: undefined, + data: undefined, + raw: "", + bytes: Buffer.byteLength(block) + 2, + } + const data: string[] = [] + for (const line of block.split("\n")) { + if (line.startsWith("id: ")) frame.id = line.slice(4) + else if (line.startsWith("event: ")) frame.event = line.slice(7) + else if (line.startsWith("data: ")) data.push(line.slice(6)) + } + frame.raw = data.join("\n") + frame.data = data.length ? JSON.parse(frame.raw) : undefined + return frame +} + +/** Open a raw SSE connection to the in-process server's global stream. */ +const connect = (input: { headers?: Record; query?: string } = {}) => + Effect.gen(function* () { + // The router is attached while the layer builds, so the server routes from + // the first request; dial exactly the address it is bound to. + const url = HttpServer.formatAddress((yield* HttpServer.HttpServer).address) + const frames = yield* Queue.unbounded() + const seen: string[] = [] + let ended = "" + const controller = new AbortController() + const response = yield* Effect.promise(() => + fetch(`${url}${GlobalPaths.event}${input.query ?? ""}`, { + headers: { accept: "text/event-stream", ...input.headers }, + signal: controller.signal, + }), + ) + const status = `${response.status} ${response.headers.get("content-type") ?? "-"}` + // Read the body in the background, splitting it into SSE frames. + void (async () => { + const reader = response.body!.pipeThrough(new TextDecoderStream()).getReader() + let buffer = "" + for (;;) { + const chunk = await reader.read() + if (chunk.done) return + buffer += chunk.value + for (let end = buffer.indexOf("\n\n"); end !== -1; end = buffer.indexOf("\n\n")) { + const block = buffer.slice(0, end) + buffer = buffer.slice(end + 2) + if (block.trim() === "") continue + const frame = parseFrame(block) + seen.push(`${frame.id}=${frame.data?.payload?.type}`) + Queue.offerUnsafe(frames, frame) + } + } + })().then( + () => (ended ||= `end (${status})`), + (error) => (ended ||= `${error?.name ?? "error"} (${status})`), + ) + const abort = () => controller.abort() + yield* Effect.addFinalizer(() => Effect.sync(abort)) + const next = Queue.take(frames).pipe( + Effect.timeoutOrElse({ + duration: "4 seconds", + orElse: () => + Effect.fail(new Error(`timed out waiting for frame; seen [${seen.join(", ")}], response ${ended || "open"}`)), + }), + ) + const take = (n: number) => Effect.forEach(Array.from({ length: n }), () => next) + return { next, take, abort } + }) + +const emit = (type: string, properties: Record = {}) => + GlobalBus.emit("event", { directory: "/tmp/i1776", payload: { type, properties } }) + +/** The event-v2 bridge's twin emission: a durable event, then its sync copy. */ +const twin = (id: string, type = "test.durable") => { + GlobalBus.emit("event", { directory: "/tmp/i1778", payload: { id, type, properties: {} } }) + GlobalBus.emit("event", { + directory: "/tmp/i1778", + payload: { type: "sync", syncEvent: { id, type: `${type}.1`, seq: 1, aggregateID: "agg", data: {} } }, + }) +} + +const typesOf = (frames: Frame[]) => frames.map((frame) => frame.data.payload.type) +const seqsOf = (frames: Frame[]) => frames.map((frame) => idOf(frame).seq) +/** Resolves `undefined` when no further frame arrives within `ms`. */ +const quiet = (stream: { next: Effect.Effect }, ms = 200) => + stream.next.pipe(Effect.timeoutOrElse({ duration: ms, orElse: () => Effect.succeed(undefined) })) + +describe("global event stream — sequenced ids (#1776)", () => { + it.live( + "every frame carries an : id shared by all subscribers; a sync copy carries its source's id", + () => + Effect.gen(function* () { + // sync copies are opt-in (#1778): both subscribers request them + const a = yield* connect({ query: "?sync=true" }) + const b = yield* connect({ query: "?sync=true" }) + const connectedA = yield* a.next + const connectedB = yield* b.next + expect(connectedA.data.payload.type).toBe("server.connected") + const start = idOf(connectedA) + expect(idOf(connectedB)).toEqual(start) + + emit("test.first") + // the event-v2 bridge's twin emission: the durable event, then its sync copy + GlobalBus.emit("event", { + directory: "/tmp/i1776", + payload: { id: "evt_i1776_durable", type: "test.durable", properties: {} }, + }) + GlobalBus.emit("event", { + directory: "/tmp/i1776", + payload: { + type: "sync", + syncEvent: { id: "evt_i1776_durable", type: "test.durable.1", seq: 1, aggregateID: "agg", data: {} }, + }, + }) + emit("test.last") + + const framesA = yield* a.take(4) + const framesB = yield* b.take(4) + expect(framesA.map((f) => f.data.payload.type)).toEqual(["test.first", "test.durable", "sync", "test.last"]) + expect(framesA.map((f) => f.id)).toEqual(framesB.map((f) => f.id)) + const seqs = framesA.map((f) => idOf(f).seq) + expect(framesA.every((f) => idOf(f).epoch === start.epoch)).toBe(true) + expect(seqs).toEqual([start.seq + 1, start.seq + 2, start.seq + 2, start.seq + 3]) + }), + ) +}) + +describe("global event stream — sync copies are opt-in (#1778)", () => { + it.live( + "a default subscriber gets no sync copies and its sequence stays gap-free; an opt-in subscriber gets them all", + () => + Effect.gen(function* () { + const plain = yield* connect() + // the header carrier (what the workspace control plane sends); `?sync=true` is the query twin + const synced = yield* connect({ headers: { "x-opencode-sync-copies": "true" } }) + const start = idOf(yield* plain.next) + expect(idOf(yield* synced.next)).toEqual(start) + + emit("test.first") + twin("evt_i1778_optin_a") + twin("evt_i1778_optin_b") + emit("test.last") + + const plainFrames = yield* plain.take(4) + expect(typesOf(plainFrames)).toEqual(["test.first", "test.durable", "test.durable", "test.last"]) + // filtering the copies punches no hole: a copy shares its source's sequence + expect(seqsOf(plainFrames)).toEqual([1, 2, 3, 4].map((n) => start.seq + n)) + expect(yield* quiet(plain)).toBeUndefined() + + const syncedFrames = yield* synced.take(6) + expect(typesOf(syncedFrames)).toEqual([ + "test.first", + "test.durable", + "sync", + "test.durable", + "sync", + "test.last", + ]) + expect(seqsOf(syncedFrames)).toEqual([1, 2, 2, 3, 3, 4].map((n) => start.seq + n)) + // the frames both subscribers get are the same frames + expect(syncedFrames.filter((f) => f.data.payload.type !== "sync").map((f) => f.id)).toEqual( + plainFrames.map((f) => f.id), + ) + }), + ) + + it.live("a resumed connection replays sync copies only when it requests them", () => + Effect.gen(function* () { + const a = yield* connect() + const cursor = (yield* a.next).id! + a.abort() + twin("evt_i1778_replay_a") + emit("test.between") + twin("evt_i1778_replay_b") + + const plain = yield* connect({ headers: { "last-event-id": cursor } }) + const synced = yield* connect({ headers: { "last-event-id": cursor }, query: "?sync=true" }) + const base = idOf(yield* plain.next).seq + yield* synced.next + emit("test.live") + + const plainFrames = yield* plain.take(4) + expect(typesOf(plainFrames)).toEqual(["test.durable", "test.between", "test.durable", "test.live"]) + expect(seqsOf(plainFrames)).toEqual([1, 2, 3, 4].map((n) => base + n)) + expect(yield* quiet(plain)).toBeUndefined() + + const syncedFrames = yield* synced.take(6) + expect(typesOf(syncedFrames)).toEqual([ + "test.durable", + "sync", + "test.between", + "test.durable", + "sync", + "test.live", + ]) + expect(seqsOf(syncedFrames)).toEqual([1, 1, 2, 3, 3, 4].map((n) => base + n)) + }), + ) + + it.live( + "a sync copy with no framed source keeps its own sequence and reaches every subscriber, so no hole opens", + () => + Effect.gen(function* () { + const plain = yield* connect() + const start = idOf(yield* plain.next) + emit("test.before") + GlobalBus.emit("event", { + directory: "/tmp/i1778", + payload: { + type: "sync", + syncEvent: { id: "evt_i1778_orphan", type: "test.orphan.1", seq: 1, aggregateID: "agg", data: {} }, + }, + }) + emit("test.after") + const frames = yield* plain.take(3) + expect(typesOf(frames)).toEqual(["test.before", "sync", "test.after"]) + expect(seqsOf(frames)).toEqual([1, 2, 3].map((n) => start.seq + n)) + }), + ) +}) + +// A per-turn summary as the engine publishes it: every changed file with its +// full patch text (measured up to 12.3 MB in one event on the Studio). +const summaryDiffs = () => [ + { file: "src/app.ts", patch: "@@ -1,2 +1,3 @@\n-old\n+new\n+more\n", additions: 2, deletions: 1, status: "modified" }, + { + file: "src/new.ts", + patch: `@@ -0,0 +1,400 @@\n${"+line\n".repeat(400)}`, + additions: 400, + deletions: 0, + status: "added", + }, + { file: "src/gone.ts", patch: "@@ -1 +0,0 @@\n-gone\n", additions: 0, deletions: 1, status: "deleted" }, +] +const stripped = (diffs: ReturnType) => diffs.map(({ patch: _patch, ...rest }) => rest) + +/** A summary-bearing durable event and its sync copy, as the event-v2 bridge emits them. */ +const summaryTwin = (id: string, type: "message.updated" | "session.updated") => { + const info = + type === "message.updated" + ? { id: "msg_i1778", sessionID: "ses_i1778", role: "user", summary: { title: "Edit", diffs: summaryDiffs() } } + : { + id: "ses_i1778", + title: "Session", + summary: { additions: 402, deletions: 2, files: 3, diffs: summaryDiffs() }, + } + const properties = { sessionID: "ses_i1778", info } + const source = { directory: "/tmp/i1778", payload: { id, type, properties } } + const copy = { + directory: "/tmp/i1778", + payload: { type: "sync", syncEvent: { id, type: `${type}.1`, seq: 3, aggregateID: "ses_i1778", data: properties } }, + } + return { source, copy } +} + +describe("global event stream — summary diffs carry no patch bodies on the wire (#1778)", () => { + for (const type of ["message.updated", "session.updated"] as const) { + it.live(`a live ${type} keeps file, status and counts but drops patch bodies; the bus payload keeps them`, () => + Effect.gen(function* () { + const plain = yield* connect() + const synced = yield* connect({ headers: { "x-opencode-sync-copies": "true" } }) + yield* plain.next + yield* synced.next + const seen: GlobalEvent[] = [] + const listener = (event: GlobalEvent) => seen.push(event) + GlobalBus.on("event", listener) + yield* Effect.addFinalizer(() => Effect.sync(() => GlobalBus.off("event", listener))) + + const { source, copy } = summaryTwin(`evt_i1778_${type}`, type) + GlobalBus.emit("event", source) + GlobalBus.emit("event", copy) + + const live = yield* plain.next + expect(live.data.payload.type).toBe(type) + const info = live.data.payload.properties.info + expect(info.summary.diffs).toEqual(stripped(summaryDiffs())) + expect(live.raw).not.toContain('"patch"') + // everything else in the event is as published + expect({ ...info, summary: { ...info.summary, diffs: undefined } }).toEqual({ + ...source.payload.properties.info, + summary: { ...source.payload.properties.info.summary, diffs: undefined }, + }) + // the projection is wire-only: in-process subscribers see the full event, untouched + expect(seen.find((event) => event.payload.type === type)?.payload.properties.info.summary.diffs).toEqual( + summaryDiffs(), + ) + + // the opt-in subscriber gets the same stripped frame, and a full sync copy + // (the replica replays it into its own store, so it keeps the patches) + const [same, sync] = yield* synced.take(2) + expect(same.raw).toBe(live.raw) + expect(same.id).toBe(live.id) + expect(sync.data.payload.type).toBe("sync") + expect(sync.data.payload.syncEvent.data.info.summary.diffs).toEqual(summaryDiffs()) + }), + ) + } + + it.live("a replayed summary frame is byte-identical to the live one", () => + Effect.gen(function* () { + const a = yield* connect() + const cursor = (yield* a.next).id! + const { source } = summaryTwin("evt_i1778_replayed", "message.updated") + GlobalBus.emit("event", source) + const live = yield* a.next + a.abort() + + const b = yield* connect({ headers: { "last-event-id": cursor } }) + yield* b.next + const replayed = yield* b.next + expect(replayed.id).toBe(live.id) + expect(replayed.raw).toBe(live.raw) + expect(replayed.raw).not.toContain('"patch"') + }), + ) +}) + +describe("global event stream — resume from a cursor (#1776)", () => { + for (const carrier of ["Last-Event-ID header", "lastEventID query"] as const) { + it.live(`replays exactly the frames after a ${carrier} cursor, then continues live`, () => + Effect.gen(function* () { + const a = yield* connect() + yield* a.next + emit("test.one") + emit("test.two") + emit("test.three") + const [one] = yield* a.take(3) + const cursor = idOf(one) + a.abort() + + emit("test.gap") + const resume = + carrier === "Last-Event-ID header" ? { headers: { "last-event-id": one.id! } } : { query: `?lastEventID=${encodeURIComponent(one.id!)}` } + // events keep arriving while the reconnect is in flight + const reconnect = yield* Effect.forkChild(connect(resume)) + for (let i = 0; i < 20; i++) { + emit("test.inflight", { i }) + yield* Effect.sleep("1 millis") + } + const b = yield* Fiber.join(reconnect) + emit("test.after") + + const connected = yield* b.next + expect(connected.data.payload.type).toBe("server.connected") + expect(idOf(connected)).toEqual(cursor) + const frames = yield* b.take(2 + 1 + 20 + 1) + const types = frames.map((f) => f.data.payload.type) + expect(types).toEqual(["test.two", "test.three", "test.gap", ...Array(20).fill("test.inflight"), "test.after"]) + expect(frames.map((f) => idOf(f).seq)).toEqual(Array.from({ length: 24 }, (_, i) => cursor.seq + 1 + i)) + expect(frames.filter((f) => f.data.payload.type === "test.inflight").map((f) => f.data.payload.properties.i)).toEqual( + Array.from({ length: 20 }, (_, i) => i), + ) + // nothing further: no duplicate trails the live frames + const extra = yield* b.next.pipe(Effect.timeoutOrElse({ duration: "200 millis", orElse: () => Effect.succeed(undefined) })) + expect(extra).toBeUndefined() + }), + ) + } +}) + +describe("global event stream — unservable cursor resets (#1776)", () => { + const unservable: Record Effect.Effect> = { + "a cursor from another epoch": () => Effect.succeed(`x${GlobalBus.epoch}:1`), + "a cursor that is not : (a payload id)": () => Effect.succeed("evt_131b06142001C9nKEJ5DvEnx4G"), + "a cursor ahead of the stream": () => Effect.succeed(`${GlobalBus.epoch}:${GlobalBus.seq + 1000}`), + "a cursor older than the buffer's oldest entry": () => + Effect.gen(function* () { + const a = yield* connect() + const cursor = (yield* a.next).id! + a.abort() + // more than the default 32 MiB byte bound: the frames after the cursor are evicted + const big = "x".repeat(1024 * 1024) + for (let i = 0; i < 33; i++) emit("test.big", { big }) + return cursor + }), + } + for (const [name, cursorFor] of Object.entries(unservable)) { + it.live(`${name} yields a reset frame first, then live frames`, () => + Effect.gen(function* () { + emit("test.before") + const cursor = yield* cursorFor() + const b = yield* connect({ headers: { "last-event-id": cursor } }) + const current = { epoch: GlobalBus.epoch, seq: GlobalBus.seq } + + const reset = yield* b.next + expect(reset.data.payload.type).toBe("server.cursor.unservable") + expect(reset.data.payload.properties).toEqual(current) + expect(idOf(reset)).toEqual(current) + const connected = yield* b.next + expect(connected.data.payload.type).toBe("server.connected") + expect(idOf(connected)).toEqual(current) + + emit("test.live") + const live = yield* b.next + expect(live.data.payload.type).toBe("test.live") + expect(idOf(live)).toEqual({ epoch: current.epoch, seq: current.seq + 1 }) + }), + ) + } +}) + +describe("global event stream — heartbeats (#1776)", () => { + it.effect("a heartbeat carries server time in milliseconds and the current sequence, and ids itself with it", () => + Effect.gen(function* () { + const a = yield* connect() + yield* a.next + emit("test.beat") + const live = yield* a.next + + yield* TestClock.adjust("10 seconds") + const beat = yield* a.next + expect(beat.data.payload.type).toBe("server.heartbeat") + expect(beat.data.payload.properties).toEqual({ time: yield* Clock.currentTimeMillis, seq: idOf(live).seq }) + expect(idOf(beat)).toEqual(idOf(live)) + + emit("test.after-beat") + yield* TestClock.adjust("10 seconds") + const [after, second] = yield* a.take(2) + expect(after.data.payload.type).toBe("test.after-beat") + expect(second.data.payload.type).toBe("server.heartbeat") + expect(second.data.payload.properties).toEqual({ time: yield* Clock.currentTimeMillis, seq: idOf(after).seq }) + expect(idOf(second)).toEqual(idOf(after)) + }), + ) +}) + +describe("global event stream — client abort (#1776)", () => { + it.live("a client that disconnects mid-stream releases its subscription and logs the disconnect", () => + Effect.gen(function* () { + // earlier tests' connections (destroyed at their end) finish releasing first + const settled = () => logged("global event connected").length === logged("global event disconnected").length + expect(yield* eventually(settled)).toBe(true) + const baseline = GlobalEventReplay.subscribers + const disconnects = logged("global event disconnected").length + const a = yield* connect() + yield* a.next + expect(GlobalEventReplay.subscribers).toBe(baseline + 1) + expect(logged("global event connected").at(-1)?.annotations).toMatchObject({ subscribers: baseline + 1 }) + + emit("test.unread") + emit("test.unread") + a.abort() + + expect(yield* eventually(() => GlobalEventReplay.subscribers === baseline)).toBe(true) + const lines = logged("global event disconnected").slice(disconnects) + expect(lines).toHaveLength(1) + expect(lines[0].annotations).toMatchObject({ subscribers: baseline }) + + // released for good: later events are queued for no one + for (let i = 0; i < 100; i++) emit("test.after-abort", { i }) + expect(GlobalEventReplay.subscribers).toBe(baseline) + }), + ) +}) + +// ── #1778 fixture turn: a multi-file edit plus a chatty shell command ────────── +// +// The same turn, two ways. BEFORE is what #1776's engine put on every +// subscriber's wire: each event serialized whole, each durable event followed by +// its sync copy, one shell preview update per output chunk. AFTER is measured +// on the live wire of a default subscriber of this engine: no sync copies, +// summary patch bodies stripped, shell updates through the real throttle +// (driven by TestClock). Sizes mirror the Studio measurement: one summary of +// 88 files carrying ~2.6 MB of patch text. +const FIXTURE = { files: 88, patchBytes: 30_000, chunks: 300, chunkEveryMs: 10, chunkBytes: 160, deltas: 120 } +const PREVIEW_MAX = 30_000 // the shell tool's MAX_METADATA_LENGTH +const KB = 1024 + +// the engine's SSE framing (the encoder omits the default `event: message` line) +const sseBytes = (id: string, data: string) => Buffer.byteLength(`id: ${id}\ndata: ${data}\n\n`) + +const patchFor = (file: string, bytes: number) => { + const line = `+ export const value_${file.length} = "${"x".repeat(60)}"\n` + return `--- a/${file}\n+++ b/${file}\n@@ -1,0 +1,${Math.ceil(bytes / line.length)} @@\n${line.repeat(Math.ceil(bytes / line.length))}` +} + +describe("global event stream — fixture turn byte report (#1778)", () => { + it.effect("a multi-file edit plus a chatty shell command keeps every live frame under 256 KB", () => + Effect.gen(function* () { + const plain = yield* connect() + yield* plain.next + const before: { type: string; bytes: number; sync: boolean }[] = [] + let liveEvents = 0 + const base = { directory: "/tmp/i1778-fixture", project: "prj_i1778" } + const sessionID = "ses_i1778_fixture" + + // Emit like the event-v2 bridge (the event, then for a durable one its sync + // copy) and record the frame #1776's tap would have serialized for each. + const publish = ( + type: string, + properties: Record, + options: { durable?: boolean; record?: boolean } = {}, + ) => { + const id = `evt_i1778_fx_${GlobalBus.seq + 1}` + const event = { ...base, payload: { id, type, properties } } + GlobalBus.emit("event", event) + liveEvents++ + const record = options.record !== false + if (record) + before.push({ + type, + sync: false, + bytes: sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq}`, JSON.stringify(event)), + }) + if (options.durable === false) return + const copy = { + ...base, + payload: { + type: "sync", + syncEvent: { id, type: `${type}.1`, seq: before.length, aggregateID: sessionID, data: properties }, + }, + } + GlobalBus.emit("event", copy) + if (record) + before.push({ + type, + sync: true, + bytes: sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq}`, JSON.stringify(copy)), + }) + } + const user = { + id: "msg_i1778_user", + sessionID, + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: "p", modelID: "m" }, + } + const assistant = { id: "msg_i1778_asst", sessionID, role: "assistant", parentID: user.id, time: { created: 2 } } + const tool = (callID: string, toolName: string, state: Record) => ({ + sessionID, + part: { id: `prt_${callID}`, sessionID, messageID: assistant.id, type: "tool", callID, tool: toolName, state }, + time: 3, + }) + + publish("message.updated", { sessionID, info: user }) + publish("message.updated", { sessionID, info: assistant }) + for (let i = 0; i < FIXTURE.deltas; i++) + publish( + "message.part.delta", + { sessionID, messageID: assistant.id, partID: "prt_text", field: "text", delta: "Editing the module now. " }, + { durable: false }, + ) + + // the multi-file edit: each edit tool part runs then completes (its own + // metadata keeps its patch — other tools' metadata is out of scope) + const files = Array.from({ length: FIXTURE.files }, (_, i) => `packages/app/src/feature/module-${i}.ts`) + const diffs = files.map((file) => ({ + file, + patch: patchFor(file, FIXTURE.patchBytes), + additions: 400, + deletions: 12, + status: "modified", + })) + for (const [i, diff] of diffs.entries()) { + const input = { filePath: diff.file, oldString: "a", newString: "b" } + publish("message.part.updated", tool(`edit_${i}`, "edit", { status: "pending", input, raw: "" })) + const filediff = { file: diff.file, patch: diff.patch, additions: diff.additions, deletions: diff.deletions } + publish( + "message.part.updated", + tool(`edit_${i}`, "edit", { + status: "completed", + input, + output: "Edit applied successfully.", + title: diff.file, + metadata: { diff: diff.patch, filediff, diagnostics: {} }, + time: { start: 3, end: 4 }, + }), + ) + } + + // the chatty shell command: a running update per output chunk BEFORE, the + // throttled updates AFTER (first at once, <= 1 per 200 ms, final flush) + const command = { command: "bun run build --verbose" } + const running = (output: string) => + tool("shell_0", "bash", { status: "running", input: command, metadata: { output }, time: { start: 5 } }) + const previews: string[] = [] + let output = "" + for (let i = 0; i < FIXTURE.chunks; i++) { + output += `[build] step ${String(i).padStart(4, "0")} ${"·".repeat(FIXTURE.chunkBytes - 24)}\n` + previews.push(output.length <= PREVIEW_MAX ? output : "...\n\n" + output.slice(-PREVIEW_MAX)) + } + // BEFORE: one running update (and its sync copy) per output chunk + const frameBytes = (payload: Record) => + sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq + 1}`, JSON.stringify({ ...base, payload })) + for (const preview of previews) { + const id = `evt_i1778_fx_shell_${before.length}` + const data = running(preview) + before.push({ + type: "shell", + sync: false, + bytes: frameBytes({ id, type: "message.part.updated", properties: data }), + }) + before.push({ + type: "shell", + sync: true, + bytes: frameBytes({ + id, + type: "sync", + syncEvent: { id, type: "message.part.updated.1", seq: 0, aggregateID: sessionID, data }, + }), + }) + } + // AFTER: the same chunks, 10 ms apart, through the shell tool's throttle + let shellUpdates = 0 + const throttle = yield* makeOutputThrottle((preview) => + Effect.sync(() => { + shellUpdates++ + publish("message.part.updated", running(preview), { record: false }) + }), + ) + for (const preview of previews) { + yield* throttle.offer(preview) + yield* TestClock.adjust(`${FIXTURE.chunkEveryMs} millis`) + } + yield* throttle.flush + publish( + "message.part.updated", + tool("shell_0", "bash", { + status: "completed", + input: command, + output: previews.at(-1), + title: command.command, + metadata: { output: previews.at(-1), exit: 0 }, + time: { start: 5, end: 6 }, + }), + ) + + // the per-turn summary: every changed file with its full patch + publish("message.updated", { sessionID, info: { ...user, summary: { title: "Edit 88 modules", diffs } } }) + publish("session.updated", { + sessionID, + info: { id: sessionID, title: "fixture", summary: { additions: 400 * 88, deletions: 12 * 88, files: 88 } }, + }) + + const frames = yield* plain.take(liveEvents) + // the wire carries exactly the turn's events — no sync copy, no hole + expect(frames.filter((f) => f.data.payload.type === "sync")).toEqual([]) + expect(seqsOf(frames)).toEqual(frames.map((_, i) => seqsOf(frames)[0] + i)) + // the BEFORE formula is the wire's own framing + for (const frame of frames) expect(sseBytes(frame.id!, frame.raw)).toBe(frame.bytes) + + const after = frames.map((frame) => ({ type: frame.data.payload.type as string, bytes: frame.bytes })) + const total = (rows: { bytes: number }[]) => rows.reduce((sum, row) => sum + row.bytes, 0) + const largest = (rows: { bytes: number }[]) => Math.max(...rows.map((row) => row.bytes)) + const summaryAfter = frames.find((f) => f.data.payload.properties?.info?.summary?.diffs)! + const report = { + before: { + frames: before.length, + totalBytes: total(before), + largestFrameBytes: largest(before), + syncCopies: before.filter((r) => r.sync).length, + shellUpdates: previews.length, + }, + after: { + frames: after.length, + totalBytes: total(after), + largestFrameBytes: largest(after), + syncCopies: 0, + shellUpdates, + }, + summaryFrame: { + beforeBytes: before.filter((r) => r.type === "message.updated" && !r.sync).at(-1)!.bytes, + afterBytes: summaryAfter.bytes, + }, + } + console.log(`[#1778 fixture-turn byte report] ${JSON.stringify(report)}`) + console.log( + `[#1778 fixture-turn byte report] total ${(report.before.totalBytes / KB / KB).toFixed(2)} MiB -> ${(report.after.totalBytes / KB / KB).toFixed(2)} MiB` + + ` (${(100 * (1 - report.after.totalBytes / report.before.totalBytes)).toFixed(1)}% cut); largest frame ${(report.before.largestFrameBytes / KB).toFixed(1)} KiB -> ${(report.after.largestFrameBytes / KB).toFixed(1)} KiB;` + + ` frames ${report.before.frames} -> ${report.after.frames}; shell updates ${report.before.shellUpdates} -> ${report.after.shellUpdates}`, + ) + + expect(report.before.largestFrameBytes).toBeGreaterThan(256 * KB) // the fixture is honest: BEFORE breaks the bound + expect(report.after.largestFrameBytes).toBeLessThan(256 * KB) + expect(report.after.totalBytes).toBeLessThan(report.before.totalBytes) + expect(shellUpdates).toBeLessThanOrEqual(1 + (FIXTURE.chunks * FIXTURE.chunkEveryMs) / OUTPUT_THROTTLE_MS + 1) + // the final shell output reached the wire before completion + const lastRunning = frames.filter((f) => f.data.payload.properties?.part?.state?.status === "running").at(-1)! + expect(lastRunning.data.payload.properties.part.state.metadata.output).toBe(previews.at(-1)) + }), + ) +}) diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts new file mode 100644 index 000000000..3fcc85a00 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts @@ -0,0 +1,122 @@ +/** + * #1778 — summary patch bodies are cut on the wire only. A real user message + * with per-file summary diffs goes through the session store and the event-v2 + * bridge: its live global-stream frame carries file/status/counts and no patch + * text, while the stored message — what `GET /session/:id/message/:messageID` + * serves, the source the per-turn diff view fetches a patch from on demand — + * keeps every patch. The durable event's sync copy (sent only to subscribers + * that replay it into a store of their own) keeps them too. + */ +import { afterEach, describe, expect } from "bun:test" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { Database } from "@opencode-ai/core/database/database" +import { FSUtil } from "@opencode-ai/core/fs-util" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { ModelV2 } from "@opencode-ai/core/model" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import { Effect, Layer } from "effect" +import { GlobalEventReplay, type Frame } from "@/bus/global-replay" +import { Session } from "@/session/session" +import { Snapshot } from "@/snapshot" +import { Storage } from "@/storage/storage" +import { SessionPaths } from "@/server/routes/instance/httpapi/groups/session" +import { MessageID } from "@/session/schema" +import { resetDatabase } from "../fixture/db" +import { disposeAllInstances, TestInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" +import { httpApiLayer, requestInDirectory } from "./httpapi-layer" + +const it = testEffect( + Layer.mergeAll( + LayerNode.compile(LayerNode.group([Session.node, Snapshot.node, Storage.node, FSUtil.node, Database.node])), + httpApiLayer, + ), +) + +afterEach(async () => { + await disposeAllInstances() + await resetDatabase() +}) + +const pathFor = (template: string, params: Record) => + Object.entries(params).reduce((result, [key, value]) => result.replace(`:${key}`, value), template) + +const diffs = [ + { + file: "src/main.ts", + patch: "@@ -1 +1,2 @@\n-a\n+b\n+c\n", + additions: 2, + deletions: 1, + status: "modified" as const, + }, + { + file: "src/new.ts", + patch: `@@ -0,0 +1,300 @@\n${"+x\n".repeat(300)}`, + additions: 300, + deletions: 0, + status: "added" as const, + }, +] + +describe("summary diffs — wire-only projection (#1778)", () => { + it.instance( + "the live frame drops patch bodies; the stored message and the sync copy keep them", + () => + Effect.gen(function* () { + const test = yield* TestInstance + const frames: Frame[] = [] + const subscription = GlobalEventReplay.subscribe(undefined, { sync: true }) + subscription.start((frame) => frames.push(frame)) + yield* Effect.addFinalizer(() => Effect.sync(() => subscription.close())) + + const session = yield* Effect.acquireRelease(Session.use.create({ title: "wire-projection" }), (created) => + Session.use.remove(created.id).pipe(Effect.ignore), + ) + const messageID = MessageID.ascending() + yield* Session.use.updateMessage({ + id: messageID, + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + summary: { title: "Edit", diffs }, + } satisfies SessionV1.User) + + const ours = () => + frames + .map((frame) => ({ frame, event: JSON.parse(frame.data) })) + .filter(({ event }) => + event.payload.type === "sync" + ? event.payload.syncEvent.data?.info?.id === messageID + : event.payload.type === "message.updated" && event.payload.properties.info.id === messageID, + ) + // the bridge emits synchronously inside publish: both frames exist now + const [live, copy, ...extra] = ours() + expect(extra).toEqual([]) + + expect(live.event.payload.type).toBe("message.updated") + expect(live.frame.syncCopy).toBe(false) + expect(live.frame.data).not.toContain('"patch"') + expect(live.event.payload.properties.info.summary).toEqual({ + title: "Edit", + diffs: diffs.map(({ patch: _patch, ...rest }) => rest), + }) + + expect(copy.event.payload.type).toBe("sync") + expect(copy.frame.syncCopy).toBe(true) + expect(copy.frame.seq).toBe(live.frame.seq) + expect(copy.event.payload.syncEvent.data.info.summary.diffs).toEqual(diffs) + + // the stored record is unchanged — and it is the on-demand patch source + const response = yield* requestInDirectory( + pathFor(SessionPaths.message, { sessionID: session.id, messageID }), + test.directory, + ) + expect(response.status).toBe(200) + const stored = (yield* response.json) as { info: { summary?: { diffs: unknown } } } + expect(stored.info.summary?.diffs).toEqual(diffs) + }), + { git: true, config: { formatter: false, lsp: false } }, + ) +}) diff --git a/packages/app-bundle/overlay/packages/opencode/test/tool/shell-output-throttle.test.ts b/packages/app-bundle/overlay/packages/opencode/test/tool/shell-output-throttle.test.ts new file mode 100644 index 000000000..a09e07acc --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/tool/shell-output-throttle.test.ts @@ -0,0 +1,192 @@ +// #1778 — a running shell tool's output preview is throttled: the first update +// goes out at once, then at most one per 200 ms (a trailing update carries the +// latest output), and the final output is always delivered when the command +// completes. Every update is a `message.part.updated` on the live stream +// carrying up to 30 KB of preview, so an unthrottled chatty command floods it. +// The throttle itself runs on effect's Clock (driven by TestClock here); the +// shell tool is then exercised with real chatty commands. Harness trimmed from +// shell.test.ts (upstream). +import { describe, expect } from "bun:test" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { Clock, Effect, Layer } from "effect" +import * as TestClock from "effect/testing/TestClock" +import path from "path" +import { Config } from "@/config/config" +import { Shell } from "@opencode-ai/core/shell" +import { ShellTool } from "../../src/tool/shell" +import { OUTPUT_THROTTLE_MS, makeOutputThrottle } from "../../src/tool/shell-output-throttle" +import { provideInstance, testInstanceStoreLayer } from "../fixture/fixture" +import { Agent } from "../../src/agent/agent" +import { Truncate } from "@/tool/truncate" +import { SessionID, MessageID } from "../../src/session/schema" +import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" +import { FSUtil } from "@opencode-ai/core/fs-util" +import { Plugin } from "../../src/plugin" +import { testEffect } from "../lib/effect" +import type { Tool } from "@/tool/tool" +import { RuntimeFlags } from "@/effect/runtime-flags" + +const it = testEffect( + Layer.mergeAll( + LayerNode.compile( + LayerNode.group([ + CrossSpawnSpawner.node, + FSUtil.node, + Plugin.node, + Truncate.node, + Config.node, + Agent.node, + RuntimeFlags.node, + ]), + ), + testInstanceStoreLayer, + ), +) + +Shell.acceptable.reset() +const projectRoot = path.join(__dirname, "../..") +const bin = `"${process.execPath.replaceAll("\\", "/")}"` +const script = (code: string) => { + const shell = Shell.name(Shell.acceptable()) + const text = `${bin} -e ${shell === "cmd" ? `"${code}"` : `'${code}'`}` + return shell === "pwsh" || shell === "powershell" ? `& ${text}` : text +} + +/** Run the shell tool, recording every output preview update it publishes. */ +const runRecording = (command: string) => + Effect.gen(function* () { + const updates: { at: number; output: string }[] = [] + const ctx: Tool.Context = { + sessionID: SessionID.make("ses_i1778"), + messageID: MessageID.make("msg_i1778"), + callID: "", + agent: "build", + abort: AbortSignal.any([]), + messages: [], + metadata: (input) => + Effect.sync(() => { + const output = (input.metadata as { output?: string })?.output + if (output) updates.push({ at: performance.now(), output }) + }), + ask: () => Effect.void, + } + const tool = yield* (yield* ShellTool).init() + const started = performance.now() + const result = yield* tool.execute({ command }, ctx) + return { updates, result, elapsed: performance.now() - started } + }).pipe(provideInstance(projectRoot)) + +describe("shell output throttle (#1778)", () => { + it.effect("the first update goes out at once; updates inside 200 ms coalesce into one trailing update", () => + Effect.gen(function* () { + const sent: { at: number; output: string }[] = [] + const throttle = yield* makeOutputThrottle((output) => + Effect.map(Clock.currentTimeMillis, (at) => { + sent.push({ at, output }) + }), + ) + expect(OUTPUT_THROTTLE_MS).toBe(200) + + yield* throttle.offer("a") + expect(sent).toEqual([{ at: 0, output: "a" }]) + + yield* throttle.offer("ab") + yield* TestClock.adjust("50 millis") + yield* throttle.offer("abc") + yield* TestClock.adjust("149 millis") + expect(sent).toHaveLength(1) + yield* TestClock.adjust("1 millis") + expect(sent).toEqual([ + { at: 0, output: "a" }, + { at: 200, output: "abc" }, + ]) + + // quiet afterwards: nothing new, nothing sent + yield* TestClock.adjust("1 second") + expect(sent).toHaveLength(2) + // after a quiet spell the next update goes out at once again + yield* throttle.offer("abcd") + expect(sent.at(-1)).toEqual({ at: 1200, output: "abcd" }) + }), + ) + + it.effect("a steady stream of output yields at most one update per 200 ms, the latest each time", () => + Effect.gen(function* () { + const sent: { at: number; output: string }[] = [] + const throttle = yield* makeOutputThrottle((output) => + Effect.map(Clock.currentTimeMillis, (at) => { + sent.push({ at, output }) + }), + ) + let output = "" + for (let i = 0; i < 200; i++) { + output += `${i}\n` + yield* throttle.offer(output) + yield* TestClock.adjust("10 millis") + } + expect(sent.length).toBeLessThanOrEqual(1 + 2000 / 200) + expect(sent.length).toBeGreaterThanOrEqual(2000 / 200) + for (let i = 1; i < sent.length; i++) expect(sent[i].at - sent[i - 1].at).toBeGreaterThanOrEqual(200) + expect(sent.map((s) => s.output.length)).toEqual([...sent.map((s) => s.output.length)].sort((a, b) => a - b)) + }), + ) + + it.effect("flush delivers unsent output at once, cancels the pending trailing update, and is idempotent", () => + Effect.gen(function* () { + const sent: { at: number; output: string }[] = [] + const throttle = yield* makeOutputThrottle((output) => + Effect.map(Clock.currentTimeMillis, (at) => { + sent.push({ at, output }) + }), + ) + yield* throttle.offer("a") + yield* throttle.offer("ab") + yield* TestClock.adjust("20 millis") + yield* throttle.flush + expect(sent).toEqual([ + { at: 0, output: "a" }, + { at: 20, output: "ab" }, + ]) + yield* TestClock.adjust("1 second") + yield* throttle.flush + expect(sent).toHaveLength(2) + }), + ) +}) + +describe("tool.shell — throttled output updates (#1778)", () => { + it.live( + "a command printing continuously publishes at most about five output updates per second", + () => + Effect.gen(function* () { + const { updates, result, elapsed } = yield* runRecording( + script( + "for (let i = 0; i < 150; i++) { process.stdout.write(`line-${i}\\n`); await Bun.sleep(6) } process.stdout.write(`line-END\\n`)", + ), + ) + expect(result.metadata.exit).toBe(0) + // first update at once, then <= 1 per 200 ms, plus the final flush on completion + expect(updates.length).toBeLessThanOrEqual(2 + Math.ceil(elapsed / OUTPUT_THROTTLE_MS)) + expect(updates.length).toBeGreaterThan(1) + // the final output is delivered on completion: the last update is the full output + expect(updates.at(-1)!.output).toContain("line-END") + expect(updates.at(-1)!.output).toBe(result.metadata.output) + }), + 20_000, + ) + + it.live( + "output that lands inside the throttle window just before exit is still delivered", + () => + Effect.gen(function* () { + const { updates, result } = yield* runRecording( + script("process.stdout.write(`first\\n`); await Bun.sleep(30); process.stdout.write(`last\\n`)"), + ) + expect(result.metadata.exit).toBe(0) + expect(updates[0].output).toContain("first") + expect(updates.at(-1)!.output).toContain("last") + expect(updates.at(-1)!.output).toBe(result.metadata.output) + }), + 20_000, + ) +}) diff --git a/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts b/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts new file mode 100644 index 000000000..2bd527c63 --- /dev/null +++ b/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts @@ -0,0 +1,278 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import type { Config } from "./types.gen.js" + +export type ServerSentEventsOptions = Omit & + Pick & { + /** + * Fetch API implementation. You can use this option to provide a custom + * fetch instance. + * + * @default globalThis.fetch + */ + fetch?: typeof fetch + /** + * Implementing clients can call request interceptors inside this hook. + */ + onRequest?: (url: string, init: RequestInit) => Promise + /** + * Callback invoked when a network or parsing error occurs during streaming. + * + * This option applies only if the endpoint returns a stream of events. + * + * @param error The error that occurred. + */ + onSseError?: (error: unknown) => void + /** + * Callback invoked when an event is streamed from the server. + * + * This option applies only if the endpoint returns a stream of events. + * + * @param event Event streamed from the server. + * @returns Nothing (void). + */ + onSseEvent?: (event: StreamEvent) => void + serializedBody?: RequestInit["body"] + /** + * Default retry delay in milliseconds. + * + * This option applies only if the endpoint returns a stream of events. + * + * @default 3000 + */ + sseDefaultRetryDelay?: number + /** + * Maximum number of retry attempts before giving up. + */ + sseMaxRetryAttempts?: number + /** + * Maximum retry delay in milliseconds. + * + * Applies only when exponential backoff is used. + * + * This option applies only if the endpoint returns a stream of events. + * + * @default 30000 + */ + sseMaxRetryDelay?: number + /** + * Optional sleep function for retry backoff. + * + * Defaults to using `setTimeout`. + */ + sseSleepFn?: (ms: number) => Promise + url: string + } + +export interface StreamEvent { + data: TData + event?: string + id?: string + retry?: number +} + +export type ServerSentEventsResult = { + stream: AsyncGenerator ? TData[keyof TData] : TData, TReturn, TNext> +} + +export const createSseClient = ({ + onRequest, + onSseError, + onSseEvent, + responseTransformer, + responseValidator, + sseDefaultRetryDelay, + sseMaxRetryAttempts, + sseMaxRetryDelay, + sseSleepFn, + url, + ...options +}: ServerSentEventsOptions): ServerSentEventsResult => { + let lastEventId: string | undefined + + const sleep = sseSleepFn ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))) + + const createStream = async function* () { + let retryDelay: number = sseDefaultRetryDelay ?? 3000 + let attempt = 0 + const signal = options.signal ?? new AbortController().signal + + while (true) { + if (signal.aborted) break + + attempt++ + + const headers = + options.headers instanceof Headers + ? options.headers + : new Headers(options.headers as Record | undefined) + + if (lastEventId !== undefined) { + headers.set("Last-Event-ID", lastEventId) + } + + try { + const requestInit: RequestInit = { + redirect: "follow", + ...options, + body: options.serializedBody, + headers, + signal, + } + let request = new Request(url, requestInit) + if (onRequest) { + request = await onRequest(url, requestInit) + } + // fetch must be assigned here, otherwise it would throw the error: + // TypeError: Failed to execute 'fetch' on 'Window': Illegal invocation + const _fetch = options.fetch ?? globalThis.fetch + const response = await _fetch(request) + + if (!response.ok) throw new Error(`SSE failed: ${response.status} ${response.statusText}`) + + if (!response.body) throw new Error("No body in SSE response") + + const reader = response.body.pipeThrough(new TextDecoderStream()).getReader() + + // amicode #1779: split frames in LINEAR time. The generated reader + // appended every chunk to the whole partial buffer, then re-normalized + // and re-split all of it — quadratic on a multi-MB frame arriving in + // 64 KB chunks, on the webview's main thread. Now only the NEW text is + // normalized and searched; a "\n\n" split across two chunks is caught by + // a one-character carry; the partial frame is held as pieces and joined + // once, when it completes. A "\r" ending a chunk waits for the next one + // (it may be the first half of a CRLF — normalizing it alone made a CRLF + // split across chunks end its frame early). Frames come out as before. + let pending: Array = [] + let pendingLast = "" + let heldCR = false + // Normalize line endings: CRLF -> LF, then CR -> LF + const normalize = (value: string, final: boolean): string => { + let text = heldCR ? "\r" + value : value + heldCR = !final && text.endsWith("\r") + if (heldCR) text = text.slice(0, -1) + return text.includes("\r") ? text.replace(/\r\n/g, "\n").replace(/\r/g, "\n") : text + } + const splitFrames = (text: string): Array => { + const frames: Array = [] + if (!text) return frames + let start = 0 + if (pendingLast === "\n" && text.charCodeAt(0) === 10) { + // the delimiter straddles the chunk boundary + frames.push(pending.join("").slice(0, -1)) + pending = [] + start = 1 + } + let end: number + while ((end = text.indexOf("\n\n", start)) !== -1) { + pending.push(text.slice(start, end)) + frames.push(pending.join("")) + pending = [] + start = end + 2 + } + if (start < text.length) { + pending.push(start === 0 ? text : text.slice(start)) + pendingLast = text.charAt(text.length - 1) + } else { + pendingLast = "" + } + return frames + } + + const abortHandler = () => { + try { + reader.cancel() + } catch { + // noop + } + } + + signal.addEventListener("abort", abortHandler) + + try { + while (true) { + const { done, value } = await reader.read() + // at the end of the stream, a held "\r" ends its line after all + const chunks = splitFrames(normalize(done ? "" : value, done)) + + for (const chunk of chunks) { + const lines = chunk.split("\n") + const dataLines: Array = [] + let eventName: string | undefined + + for (const line of lines) { + if (line.startsWith("data:")) { + dataLines.push(line.replace(/^data:\s*/, "")) + } else if (line.startsWith("event:")) { + eventName = line.replace(/^event:\s*/, "") + } else if (line.startsWith("id:")) { + lastEventId = line.replace(/^id:\s*/, "") + } else if (line.startsWith("retry:")) { + const parsed = Number.parseInt(line.replace(/^retry:\s*/, ""), 10) + if (!Number.isNaN(parsed)) { + retryDelay = parsed + } + } + } + + let data: unknown + let parsedJson = false + + if (dataLines.length) { + const rawData = dataLines.join("\n") + try { + data = JSON.parse(rawData) + parsedJson = true + } catch { + data = rawData + } + } + + if (parsedJson) { + if (responseValidator) { + await responseValidator(data) + } + + if (responseTransformer) { + data = await responseTransformer(data) + } + } + + onSseEvent?.({ + data, + event: eventName, + id: lastEventId, + retry: retryDelay, + }) + + if (dataLines.length) { + yield data as any + } + } + if (done) break + } + } finally { + signal.removeEventListener("abort", abortHandler) + reader.releaseLock() + } + + break // exit loop on normal completion + } catch (error) { + // connection failed or aborted; retry after delay + onSseError?.(error) + + if (sseMaxRetryAttempts !== undefined && attempt >= sseMaxRetryAttempts) { + break // stop after firing error + } + + // exponential backoff: double retry each attempt, cap at 30s + const backoff = Math.min(retryDelay * 2 ** (attempt - 1), sseMaxRetryDelay ?? 30000) + await sleep(backoff) + } + } + } + + const stream = createStream() + + return { stream } +} diff --git a/packages/app-bundle/overlay/packages/session-ui/src/components/session-turn.tsx b/packages/app-bundle/overlay/packages/session-ui/src/components/session-turn.tsx index da8ed7d53..56028dd43 100644 --- a/packages/app-bundle/overlay/packages/session-ui/src/components/session-turn.tsx +++ b/packages/app-bundle/overlay/packages/session-ui/src/components/session-turn.tsx @@ -26,6 +26,7 @@ import { TextReveal } from "@opencode-ai/ui/text-reveal" import { createAutoScroll } from "@opencode-ai/ui/hooks" import { useI18n } from "@opencode-ai/ui/context/i18n" import { normalize } from "./session-diff" +import { hasBody } from "./summary-patch" function record(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value) @@ -470,9 +471,32 @@ export function SessionTurn( > {(diff) => { - const view = normalize(diff) const active = createMemo(() => expanded().includes(diff.file)) const [shown, setShown] = createSignal(false) + // #1778: a live summary carries no patch body — fetch the + // turn's stored diffs once this file is expanded and shown. + // A plain signal, not a resource: a pending fetch must not + // suspend the timeline around it. + const [fetched, setFetched] = createSignal() + createEffect( + on(shown, (value) => { + if (!value || hasBody(diff) || fetched() !== undefined) return + void data + .summaryPatch({ + sessionID: props.sessionID, + messageID: props.messageID, + file: diff.file, + summary: diffs(), + }) + .then((patch) => { + if (patch !== undefined) setFetched(patch) + }) + }), + ) + const view = createMemo(() => { + const patch = fetched() + return normalize(patch === undefined ? diff : { ...diff, patch }) + }) createEffect( on( @@ -519,7 +543,7 @@ export function SessionTurn(
- +
diff --git a/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts new file mode 100644 index 000000000..6b8941b7e --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, test } from "bun:test" +import { readFileSync } from "node:fs" +import { resolve } from "node:path" +import { normalize } from "./session-diff" +import { createSummaryPatchLoader, hasBody, type SummaryDiffSource } from "./summary-patch" + +// #1778 — live per-turn summaries carry each file's status and counts but no +// patch body (the engine strips patches on the wire; storage keeps them). The +// per-turn diff view fetches a patch on demand when a file is expanded, from +// the stored message. The loader is the behavior; session-turn.tsx's wiring is +// asserted on its source below (bun's test transpiler does not run Solid's +// JSX transform — the repo's component-source-assertion convention). + +const PATCH = "@@ -1,2 +1,2 @@\n-old line\n+new line\n context\n" +const stored = [ + { file: "src/app.ts", patch: PATCH, additions: 1, deletions: 1, status: "modified" as const }, + { file: "src/other.ts", patch: "@@ -0,0 +1 @@\n+other\n", additions: 1, deletions: 0, status: "added" as const }, +] +// what a live message.updated carries after the wire projection +const live = stored.map(({ patch: _patch, ...rest }) => rest) +const turn = { sessionID: "ses_1", messageID: "msg_1" } + +function recordingSource(answer: () => Promise = async () => stored) { + const calls: { sessionID: string; messageID: string }[] = [] + const source: SummaryDiffSource = (input) => { + calls.push(input) + return answer() + } + return { calls, source } +} + +describe("per-turn diff view — patch bodies on demand (#1778)", () => { + test("a live summary's file list and counts need no fetch; only a patch-less diff needs one", () => { + const { calls, source } = recordingSource() + createSummaryPatchLoader(source) + expect(live.map((diff) => [diff.file, diff.additions, diff.deletions, diff.status])).toEqual([ + ["src/app.ts", 1, 1, "modified"], + ["src/other.ts", 1, 0, "added"], + ]) + expect(live.map(hasBody)).toEqual([false, false]) + expect(stored.map(hasBody)).toEqual([true, true]) + expect(hasBody({ file: "a", before: "x", after: "y" })).toBe(true) + expect(calls).toEqual([]) + }) + + test("expanding a file fetches the turn's stored diffs once and renders that file's patch", async () => { + const { calls, source } = recordingSource() + const loader = createSummaryPatchLoader(source) + + const patch = await loader.patch({ ...turn, file: "src/app.ts", summary: live }) + expect(patch).toBe(PATCH) + const view = normalize({ ...live[0], patch }) + expect(view.fileDiff.additionLines.join("")).toContain("new line") + expect(view.fileDiff.deletionLines.join("")).toContain("old line") + expect({ additions: view.additions, deletions: view.deletions, status: view.status }).toEqual({ + additions: 1, + deletions: 1, + status: "modified", + }) + + // a second file of the same turn — and concurrent expands — share the one fetch + const [other, again] = await Promise.all([ + loader.patch({ ...turn, file: "src/other.ts", summary: live }), + loader.patch({ ...turn, file: "src/app.ts", summary: live }), + ]) + expect(other).toBe(stored[1].patch) + expect(again).toBe(PATCH) + expect(calls).toEqual([turn]) + }) + + test("a failed fetch resolves to no patch and is retried on the next expand", async () => { + let fail = true + const { calls, source } = recordingSource(async () => { + if (fail) throw new Error("offline") + return stored + }) + const loader = createSummaryPatchLoader(source) + expect(await loader.patch({ ...turn, file: "src/app.ts", summary: live })).toBeUndefined() + fail = false + expect(await loader.patch({ ...turn, file: "src/app.ts", summary: live })).toBe(PATCH) + expect(calls).toHaveLength(2) + }) + + test("a turn whose summary changed (a later message.updated) is fetched afresh", async () => { + const { calls, source } = recordingSource() + const loader = createSummaryPatchLoader(source) + await loader.patch({ ...turn, file: "src/app.ts", summary: live }) + await loader.patch({ ...turn, file: "src/app.ts", summary: [{ ...live[0], additions: 7 }, live[1]] }) + expect(calls).toHaveLength(2) + }) + + test("without a source (a provider that cannot fetch) a patch-less diff resolves to no patch", async () => { + expect( + await createSummaryPatchLoader(undefined).patch({ ...turn, file: "src/app.ts", summary: live }), + ).toBeUndefined() + }) +}) + +describe("per-turn diff view — wiring (#1778)", () => { + const turnSource = readFileSync(resolve(__dirname, "session-turn.tsx"), "utf8") + const dataSource = readFileSync(resolve(__dirname, "../context/data.tsx"), "utf8") + + test("the data provider takes an on-demand summary-diff source and shares one loader", () => { + expect(dataSource).toContain("onLoadSummaryDiffs?: SummaryDiffSource") + expect(dataSource).toContain("createSummaryPatchLoader(props.onLoadSummaryDiffs)") + expect(dataSource).toContain("summaryPatch:") + }) + + test("the accordion fetches a patch only once a patch-less file is expanded, and renders the fetched patch", () => { + // the fetch fires from the expanded+shown edge, only for a diff without a body + expect(turnSource).toContain("on(shown, (value) => {") + expect(turnSource).toContain("if (!value || hasBody(diff) || fetched() !== undefined) return") + expect(turnSource).toContain(".summaryPatch({") + expect(turnSource).toContain("normalize(patch === undefined ? diff : { ...diff, patch })") + expect(turnSource).toContain("fileDiff={view().fileDiff}") + // the per-file view is no longer parsed eagerly for collapsed files + expect(turnSource).not.toContain("const view = normalize(diff)") + }) + + test("the file list and counts render from the summary itself, with no fetch", () => { + expect(turnSource).toContain("") + expect(turnSource).toContain("") + const memo = turnSource.slice(turnSource.indexOf("const diffs = createMemo"), turnSource.indexOf("const MAX_FILES")) + expect(memo).not.toContain("summaryPatch") + }) +}) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.ts b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.ts new file mode 100644 index 000000000..ca250f917 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.ts @@ -0,0 +1,77 @@ +// #1778 — per-turn summary patches on demand. The engine sends a live summary's +// diffs with file, status and counts but no patch body (patches stay in +// storage); the per-turn diff view fetches the turn's stored diffs only when a +// patch-less file is expanded. Plain TypeScript (no reactivity) so the +// behavior is testable directly; session-turn.tsx drives it from its expand +// state. + +/** A diff as a summary carries it — the patch body is optional on the wire. */ +export type SummaryDiffLike = { + file?: string + patch?: string + before?: string + after?: string + additions?: number + deletions?: number + status?: string +} + +/** Fetch a turn's stored summary diffs (with patch bodies), e.g. from `GET /session/:id/message/:messageID`. */ +export type SummaryDiffSource = (input: { + sessionID: string + messageID: string +}) => Promise + +export type SummaryPatchRequest = { + sessionID: string + messageID: string + /** The expanded file. */ + file: string + /** The turn's summary diffs as currently shown — a changed summary is fetched afresh. */ + summary: readonly SummaryDiffLike[] +} + +/** Whether a diff can render without a fetch: it carries a patch, or before/after contents. */ +export function hasBody(diff: SummaryDiffLike) { + return typeof diff.patch === "string" || typeof diff.before === "string" || typeof diff.after === "string" +} + +const signature = (summary: readonly SummaryDiffLike[]) => + JSON.stringify(summary.map((diff) => [diff.file, diff.status, diff.additions, diff.deletions])) + +/** + * One loader per data provider. A turn's stored diffs are fetched at most once + * per summary (every file of the turn, and concurrent expands, share it); a + * failed fetch is forgotten so the next expand retries it. + */ +export function createSummaryPatchLoader(source: SummaryDiffSource | undefined) { + const turns = new Map>() + const limit = 32 + + const fetchTurn = (input: SummaryPatchRequest) => { + const key = `${input.sessionID}\0${input.messageID}\0${signature(input.summary)}` + const hit = turns.get(key) + if (hit) return hit + const pending = Promise.resolve() + .then(() => source?.({ sessionID: input.sessionID, messageID: input.messageID })) + .catch(() => { + turns.delete(key) + return undefined + }) + turns.set(key, pending) + while (turns.size > limit) turns.delete(turns.keys().next().value!) + return pending + } + + return { + /** The expanded file's patch body, or `undefined` when it cannot be had. */ + async patch(input: SummaryPatchRequest): Promise { + if (!source) return undefined + const diffs = await fetchTurn(input) + const match = diffs?.findLast((diff) => diff.file === input.file) + return typeof match?.patch === "string" ? match.patch : undefined + }, + } +} + +export type SummaryPatchLoader = ReturnType diff --git a/packages/app-bundle/overlay/packages/session-ui/src/context/data.tsx b/packages/app-bundle/overlay/packages/session-ui/src/context/data.tsx new file mode 100644 index 000000000..4541bd681 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/context/data.tsx @@ -0,0 +1,82 @@ +import type { Message, Session, Part, SnapshotFileDiff, SessionStatus, Provider } from "@opencode-ai/sdk/v2" +import type { FileDiffInfo } from "@opencode-ai/client/promise" +import { createSimpleContext } from "@opencode-ai/ui/context" +import { PreloadMultiFileDiffResult } from "@pierre/diffs/ssr" +import { createSummaryPatchLoader, type SummaryDiffSource } from "../components/summary-patch" + +export type NormalizedProviderListResponse = { + all: Map + defaultModel?: { + providerID: string + modelID: string + } | null + default: { + [key: string]: string + } + connected: Array +} + +type Data = { + agent?: { + name: string + color?: string + }[] + provider?: NormalizedProviderListResponse + session: Session[] + session_status: { + [sessionID: string]: SessionStatus + } + session_diff: { + [sessionID: string]: (SnapshotFileDiff | FileDiffInfo)[] + } + session_diff_preload?: { + [sessionID: string]: PreloadMultiFileDiffResult[] + } + message: { + [sessionID: string]: Message[] + } + part: { + [messageID: string]: Part[] + } + part_text_accum_delta?: { + [partID: string]: string + } +} + +export type NavigateToSessionFn = (sessionID: string) => void + +export type SessionHrefFn = (sessionID: string) => string + +export const { use: useData, provider: DataProvider } = createSimpleContext({ + name: "Data", + init: (props: { + data: Data + directory: string + sessionID?: string + onNavigateToSession?: NavigateToSessionFn + onSessionHref?: SessionHrefFn + /** + * #1778 (amicode overlay): fetch a turn's stored summary diffs, patch + * bodies included. Live summaries carry no patch bodies (the engine + * strips them on the wire), so the per-turn diff view asks for a patch + * when a file is expanded. Without it a patch-less diff renders empty. + */ + onLoadSummaryDiffs?: SummaryDiffSource + }) => { + const loader = createSummaryPatchLoader(props.onLoadSummaryDiffs) + return { + get store() { + return props.data + }, + get directory() { + return props.directory + }, + get sessionID() { + return props.sessionID + }, + navigateToSession: props.onNavigateToSession, + sessionHref: props.onSessionHref, + summaryPatch: loader.patch, + } + }, +}) diff --git a/packages/extension/scripts/assert_fleet_guard.sh b/packages/extension/scripts/assert_fleet_guard.sh index 44f4a56f2..956d38d6a 100755 --- a/packages/extension/scripts/assert_fleet_guard.sh +++ b/packages/extension/scripts/assert_fleet_guard.sh @@ -45,6 +45,16 @@ grep -q "TCPKeepAlive=yes" "$PLIST" || fail "plist TCPKeepAlive yes missing" grep -q "127.0.0.1:4096:127.0.0.1:4096" "$PLIST" || fail "plist LocalForward 4096 missing" ok "tunnel plist $PLIST" +# #1775: the per-peer forward template carries the SAME standard options. +PEER_PLIST="$ROOT/tools/fleet/co.harmoniqs.amico-peer-forward.plist" +[[ -f "$PEER_PLIST" ]] || fail "peer-forward plist template missing at $PEER_PLIST" +grep -q "ServerAliveInterval=15" "$PEER_PLIST" || fail "peer-forward plist ServerAliveInterval 15 missing" +grep -q "ServerAliveCountMax=2" "$PEER_PLIST" || fail "peer-forward plist ServerAliveCountMax 2 missing" +grep -q "TCPKeepAlive=yes" "$PEER_PLIST" || fail "peer-forward plist TCPKeepAlive yes missing" +grep -q "ExitOnForwardFailure=yes" "$PEER_PLIST" || fail "peer-forward plist ExitOnForwardFailure yes missing" +grep -q "co.harmoniqs.amico-peer-PEER_MACHINE_ID-forward" "$PEER_PLIST" || fail "peer-forward plist label is not the co.harmoniqs.amico-peer--forward convention" +ok "peer-forward plist $PEER_PLIST" + # The extension's health checks consume the projection topology state # (fleet_topology.ts) — the consumer import must be present (#1106). grep -q "fleet_topology" "$ROOT/packages/extension/src/fleet_health.ts" || fail "fleet_health.ts missing the fleet_topology consumer import (#1106)" @@ -53,11 +63,13 @@ grep -q "fleet_topology" "$ROOT/packages/extension/src/fleet_health.ts" || fail PKG_GUARD="$ROOT/packages/extension/tools/fleet/amico-opencode-fleet-guard" PKG_PLIST="$ROOT/packages/extension/tools/fleet/co.harmoniqs.amico-tunnel.plist" PKG_INSTALL="$ROOT/packages/extension/tools/fleet/install.sh" -for f in "$PKG_GUARD" "$PKG_PLIST" "$PKG_INSTALL"; do +PKG_PEER_PLIST="$ROOT/packages/extension/tools/fleet/co.harmoniqs.amico-peer-forward.plist" +for f in "$PKG_GUARD" "$PKG_PLIST" "$PKG_INSTALL" "$PKG_PEER_PLIST"; do [[ -f "$f" ]] || fail "packaged fleet file missing at $f (run: cp tools/fleet/* packages/extension/tools/fleet/)" done cmp -s "$GUARD" "$PKG_GUARD" || fail "packaged guard drift — cp tools/fleet/amico-opencode-fleet-guard packages/extension/tools/fleet/" cmp -s "$PLIST" "$PKG_PLIST" || fail "packaged plist drift — cp tools/fleet/co.harmoniqs.amico-tunnel.plist packages/extension/tools/fleet/" +cmp -s "$PEER_PLIST" "$PKG_PEER_PLIST" || fail "packaged peer-forward plist drift — cp tools/fleet/co.harmoniqs.amico-peer-forward.plist packages/extension/tools/fleet/" cmp -s "$INSTALL" "$PKG_INSTALL" || fail "packaged installer drift — cp tools/fleet/install.sh packages/extension/tools/fleet/" echo "[fleet-gate] all fleet gates passed" diff --git a/packages/extension/src/amicode_service/engine_proxy.ts b/packages/extension/src/amicode_service/engine_proxy.ts index a037b7a6f..255822df2 100644 --- a/packages/extension/src/amicode_service/engine_proxy.ts +++ b/packages/extension/src/amicode_service/engine_proxy.ts @@ -25,6 +25,19 @@ export interface EngineProxyOptions { * (Authorization included) rides unchanged. */ const HOP_BY_HOP = ["host", "connection"] as const; +/** #1773 — an upstream lost AFTER the response headers went out: ERROR-close + * the downstream (destroy it with the error) so the listener — or the next + * relay — sees a transport error. A clean `end()` here read as "the stream + * ended normally, nothing was lost" and the client reconnected without a + * resync. Never throws. */ +function errorClose(res: http.ServerResponse, err: unknown): void { + try { + res.destroy(err instanceof Error ? err : new Error(`engine upstream failed: ${String(err)}`)); + } catch { + /* already gone */ + } +} + export class EngineProxy { constructor(private readonly opts: EngineProxyOptions) {} @@ -46,13 +59,8 @@ export class EngineProxy { const upstream = http.request(target, { method: req.method, headers }, (up) => { res.writeHead(up.statusCode ?? 502, up.headers); up.pipe(res); - up.on("error", () => { - try { - res.end(); - } catch { - /* already gone */ - } - }); + // #1773: the engine's stream died mid-body — an error close, not an end. + up.on("error", (err) => errorClose(res, err)); }); // #1261 (AC5): exactly-once reattach — a client disconnect (a reloaded // window closing its stream) tears down the engine upstream so a reload @@ -64,14 +72,14 @@ export class EngineProxy { }); upstream.on("error", (err) => { // The engine went away mid-flight (restart gap, crash): honest 502 - // JSON when headers aren't sent yet, else just close the stream. + // JSON when headers aren't sent yet, else (#1773) an error close. + if (res.headersSent) { + errorClose(res, err); + return; + } try { - if (!res.headersSent) { - res.writeHead(502, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: `engine upstream failed: ${err}` })); - } else { - res.end(); - } + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: `engine upstream failed: ${err}` })); } catch { /* response already torn down — never throw into the server */ } @@ -80,14 +88,14 @@ export class EngineProxy { // (file uploads) pass through with no service-side memory cost. req.pipe(upstream); return true; - } catch { + } catch (err) { + if (res.headersSent) { + errorClose(res, err); + return true; + } try { - if (!res.headersSent) { - res.writeHead(502, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: "engine upstream unavailable" })); - } else { - res.end(); - } + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: "engine upstream unavailable" })); } catch { /* never throw into the server */ } diff --git a/packages/extension/src/amicode_service/fanin_census.ts b/packages/extension/src/amicode_service/fanin_census.ts new file mode 100644 index 000000000..ecb0fa620 --- /dev/null +++ b/packages/extension/src/amicode_service/fanin_census.ts @@ -0,0 +1,240 @@ +// FAN-IN CENSUS (#1771, part of #1770) — every fan-in service's counters and +// its log lines. A wedged, evicted, or flapping event arm used to look exactly +// like a quiet one in every log: nothing recorded who subscribed, why an arm +// re-opened, or what was dropped (the Oct 12 eviction storm was found only by +// reverse-engineering engine connect timestamps). The census makes that visible +// without a probe run: +// +// - one CENSUS line every 30 s (FANIN_CENSUS_INTERVAL_MS): active listeners, +// listeners accepted / closed since boot, evictions, refusals, and per arm +// (local and each peer, per listener connection) frames, bytes, largest +// frame, time since the last frame, opens, closes by reason, pending depth, +// overflow drops, and gaps emitted; +// - one TRANSITION line immediately on each listener accepted / closed / +// refused, arm opened / closed, gap emitted, overflow drop — never per frame. +// +// Observation only: the census writes to its SINK (the hub runner's log, the +// window's Amicode output channel), never to a stream. It adds no frame and +// changes no delivery decision. The clock and the sink are injected so tests +// drive it deterministically. +import type { ArmTraffic, GapCause } from "./sse_fanin_aggregator"; + +/** Why a listener or an upstream arm closed — an ENUMERATED set, so later + * slices and the verification slice can assert on it. `evicted` is kept even + * though #1772 removed the eviction path: it must read 0, and a regression + * that brings eviction back must have a name to show up under. */ +export const FANIN_CLOSE_REASONS = [ + "upstream-end", + "upstream-error", + "membership-change", + "listener-closed", + "evicted", + "shutdown", +] as const; +export type FanInCloseReason = (typeof FANIN_CLOSE_REASONS)[number]; + +/** The steady-state census cadence. */ +export const FANIN_CENSUS_INTERVAL_MS = 30_000; + +/** One arm of one live listener connection, as the census reports it. */ +export interface ArmCensus { + /** The listener connection (`c1`, `c2`, … in accept order). */ + conn: string; + /** The arm's namespace: `local` or the peer's machine id. */ + arm: string; + frames: number; + bytes: number; + largestFrame: number; + /** ms since the arm's last frame; null when it has delivered none. */ + sinceLastFrameMs: number | null; + opens: number; + closes: Record; + /** Frames waiting in the aggregator's pending stream for this arm. */ + pending: number; + /** Frames dropped on overflow, since the connection opened. */ + drops: number; + /** Gap frames emitted naming this arm. */ + gaps: number; +} + +export interface FanInCensusSnapshot { + plane: string; + listeners: number; + accepted: number; + closed: number; + evictions: number; + refusals: number; + arms: ArmCensus[]; +} + +/** What a live listener connection exposes to the census: its id and its + * arms' current counters (read at census time — nothing is pushed per frame). */ +export interface CensusConnection { + readonly id: string; + arms(): Array<{ + arm: string; + traffic: ArmTraffic; + pending: number; + opens: number; + closes: Partial>; + }>; +} + +export interface FanInCensusOptions { + /** Which fan-in this is (`observe` — the observation path; `premium` — the + * flag-gated fleet plane). Rides every line. */ + plane?: string; + /** Where the lines go. Default: nowhere (a census nobody wired is silent). */ + sink?: (line: string) => void; + /** The clock (ms). Default Date.now. */ + now?: () => number; + /** The census cadence. Default FANIN_CENSUS_INTERVAL_MS. */ + intervalMs?: number; +} + +export function emptyCloses(): Record { + const out = {} as Record; + for (const r of FANIN_CLOSE_REASONS) out[r] = 0; + return out; +} + +function closesToken(closes: Record): string { + const parts = FANIN_CLOSE_REASONS.filter((r) => closes[r] > 0).map((r) => `${r}:${closes[r]}`); + return parts.length === 0 ? "-" : parts.join(","); +} + +export class FanInCensus { + readonly plane: string; + readonly now: () => number; + private readonly sink: (line: string) => void; + private readonly intervalMs: number; + private readonly live = new Map(); + private accepted = 0; + private closed = 0; + private evictions = 0; + private refusals = 0; + private timer?: ReturnType; + + constructor(opts: FanInCensusOptions = {}) { + this.plane = opts.plane ?? "fanin"; + this.sink = opts.sink ?? (() => undefined); + this.now = opts.now ?? Date.now; + this.intervalMs = opts.intervalMs ?? FANIN_CENSUS_INTERVAL_MS; + } + + /** Start the steady-state cadence (idempotent; the timer never keeps the + * process alive on its own). */ + start(): void { + if (this.timer) return; + this.timer = setInterval(() => this.emit(), this.intervalMs); + this.timer.unref?.(); + } + + /** Stop the cadence (idempotent). Transition lines still log. */ + stop(): void { + if (!this.timer) return; + clearInterval(this.timer); + this.timer = undefined; + } + + // ── transitions (each logs ONE line, immediately) ───────────────────────── + + listenerAccepted(conn: CensusConnection): void { + this.accepted += 1; + this.live.set(conn.id, conn); + this.log("listener-accepted", `conn=${conn.id} listeners=${this.live.size}`); + } + + listenerClosed(connId: string, reason: FanInCloseReason): void { + if (!this.live.delete(connId)) return; + this.closed += 1; + if (reason === "evicted") this.evictions += 1; + this.log("listener-closed", `conn=${connId} reason=${reason} listeners=${this.live.size}`); + } + + listenerRefused(cap: number): void { + this.refusals += 1; + this.log("listener-refused", `listeners=${this.live.size} cap=${cap}`); + } + + armOpened(connId: string, arm: string, opens: number): void { + this.log("arm-opened", `conn=${connId} arm=${arm} opens=${opens}`); + } + + armClosed(connId: string, arm: string, reason: FanInCloseReason): void { + this.log("arm-closed", `conn=${connId} arm=${arm} reason=${reason}`); + } + + gapEmitted(connId: string, arms: string[], cause: GapCause): void { + this.log("gap-emitted", `conn=${connId} arm=${arms.join(",")} cause=${cause}`); + } + + /** The FIRST drop of an overflow episode (the census counts every drop; the + * line is per episode — a drop storm never becomes a per-frame log). */ + overflowDrop(connId: string, arm: string, pending: number): void { + this.log("overflow-drop", `conn=${connId} arm=${arm} pending=${pending}`); + } + + // ── the census ──────────────────────────────────────────────────────────── + + snapshot(): FanInCensusSnapshot { + const now = this.now(); + const arms: ArmCensus[] = []; + for (const conn of this.live.values()) { + for (const a of conn.arms()) { + arms.push({ + conn: conn.id, + arm: a.arm, + frames: a.traffic.frames, + bytes: a.traffic.bytes, + largestFrame: a.traffic.largestFrame, + sinceLastFrameMs: a.traffic.lastFrameAt === undefined ? null : Math.max(0, now - a.traffic.lastFrameAt), + opens: a.opens, + closes: { ...emptyCloses(), ...a.closes }, + pending: a.pending, + drops: a.traffic.drops, + gaps: a.traffic.gaps, + }); + } + } + return { + plane: this.plane, + listeners: this.live.size, + accepted: this.accepted, + closed: this.closed, + evictions: this.evictions, + refusals: this.refusals, + arms, + }; + } + + /** The census line: the service-level head, then ` | `-separated per-arm + * segments (`key=value` tokens throughout). */ + line(): string { + const s = this.snapshot(); + const head = `[fanin] census plane=${s.plane} listeners=${s.listeners} accepted=${s.accepted} closed=${s.closed} evictions=${s.evictions} refusals=${s.refusals} arms=${s.arms.length}`; + const segs = s.arms.map( + (a) => + `conn=${a.conn} arm=${a.arm} frames=${a.frames} bytes=${a.bytes} largest=${a.largestFrame} since_last_ms=${a.sinceLastFrameMs ?? "-"} opens=${a.opens} closes=${closesToken(a.closes)} pending=${a.pending} drops=${a.drops} gaps=${a.gaps}`, + ); + return [head, ...segs].join(" | "); + } + + /** Write the census line to the sink now (the cadence calls this). */ + emit(): void { + this.write(this.line()); + } + + /** One transition line: `[fanin] plane= `. */ + private log(kind: string, fields: string): void { + this.write(`[fanin] ${kind} plane=${this.plane} ${fields}`); + } + + private write(line: string): void { + try { + this.sink(line); + } catch { + /* a broken sink must never break the fan-in */ + } + } +} diff --git a/packages/extension/src/amicode_service/fleet_peer_store.ts b/packages/extension/src/amicode_service/fleet_peer_store.ts index 3b3d43341..3ecc94617 100644 --- a/packages/extension/src/amicode_service/fleet_peer_store.ts +++ b/packages/extension/src/amicode_service/fleet_peer_store.ts @@ -78,6 +78,23 @@ export function hasPeerToken(machineId: string, deps: PeerStoreDeps = {}): boole return readPeerToken(machineId, deps).ok; } +/** Every target this machine is attached to, with its base_url — NEVER the + * token (#1775: the per-peer SSH forward is keyed on the machine id and its + * local port is the loopback base_url's port). Sorted by machine id; an entry + * without a string base_url is skipped (absent/corrupt store → []). */ +export function listPeerBaseUrls(deps: PeerStoreDeps = {}): Array<{ machineId: string; baseUrl: string }> { + const peers = readKeyedCollection(peerTokenStorePath(deps), COLLECTION); + const out: Array<{ machineId: string; baseUrl: string }> = []; + for (const machineId of Object.keys(peers).sort()) { + const e = peers[machineId]; + if (typeof e !== "object" || e === null || Array.isArray(e)) continue; + const baseUrl = (e as Record).base_url; + if (typeof baseUrl !== "string" || baseUrl.trim() === "") continue; + out.push({ machineId, baseUrl: baseUrl.trim() }); + } + return out; +} + /** Remove the entire reader peer-store file; absent is a no-op. */ export function clearAllPeerTokens(deps: PeerStoreDeps = {}): void { clearKeyedStoreFile(peerTokenStorePath(deps)); diff --git a/packages/extension/src/amicode_service/hub_proxy.ts b/packages/extension/src/amicode_service/hub_proxy.ts index 76fba2fb9..82d733769 100644 --- a/packages/extension/src/amicode_service/hub_proxy.ts +++ b/packages/extension/src/amicode_service/hub_proxy.ts @@ -63,6 +63,18 @@ const DROPPED_HEADERS = ["host", "connection", "authorization"] as const; * `?ticket=`/`?cursor=` query all ride through verbatim. */ const UPGRADE_DROPPED_HEADERS = ["host", "authorization"] as const; +/** #1773 — the hub stream was lost AFTER the response headers went out: + * ERROR-close the downstream (destroy it with the error) so the listener — or + * the next relay — sees a transport error. A clean `end()` here read as "the + * stream ended normally, nothing was lost". Never throws. */ +function errorClose(res: http.ServerResponse, err: unknown): void { + try { + res.destroy(err instanceof Error ? err : new Error(`hub upstream failed: ${String(err)}`)); + } catch { + /* already gone */ + } +} + export class HubProxy { constructor(private readonly opts: HubProxyOptions) {} @@ -104,13 +116,13 @@ export class HubProxy { } else { const cred = this.opts.credential(); if (!cred.ok) { + if (res.headersSent) { + errorClose(res, new Error(`hub-credential-missing (${cred.reason})`)); + return true; + } try { - if (!res.headersSent) { - res.writeHead(503, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: "hub-credential-missing", reason: cred.reason })); - } else { - res.end(); - } + res.writeHead(503, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: "hub-credential-missing", reason: cred.reason })); } catch { /* never throw into the server */ } @@ -200,13 +212,9 @@ export class HubProxy { } else { up.pipe(res); } - up.on("error", () => { - try { - res.end(); - } catch { - /* already gone */ - } - }); + // #1773: the hub's stream died mid-body — an error close, not an end + // (covers the resume-filtered per-session path and the verbatim pipe). + up.on("error", (err) => errorClose(res, err)); }); // #1261 (AC5): exactly-once reattach. When the CLIENT disconnects (a // reloaded window closing its stream), tear down the host upstream so a @@ -228,27 +236,27 @@ export class HubProxy { detail: err instanceof Error ? err.message : String(err), }); } + if (res.headersSent) { + errorClose(res, err); // #1773: lost after headers → an error close + return; + } try { - if (!res.headersSent) { - res.writeHead(502, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: `hub upstream failed: ${err}` })); - } else { - res.end(); - } + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: `hub upstream failed: ${err}` })); } catch { /* response already torn down — never throw into the server */ } }); req.pipe(upstream); return true; - } catch { + } catch (err) { + if (res.headersSent) { + errorClose(res, err); + return true; + } try { - if (!res.headersSent) { - res.writeHead(502, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: "hub upstream unavailable" })); - } else { - res.end(); - } + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: "hub upstream unavailable" })); } catch { /* never throw into the server */ } diff --git a/packages/extension/src/amicode_service/index.ts b/packages/extension/src/amicode_service/index.ts index 3e02f2134..770d36423 100644 --- a/packages/extension/src/amicode_service/index.ts +++ b/packages/extension/src/amicode_service/index.ts @@ -40,11 +40,13 @@ import { SessionMultiplexProxy, SessionOwnerMap, OwnerMapFeed, + fanInReachable, type PeerTransport, type MultiplexResolver, type SessionEntry, } from "./session_multiplexer"; import { SseFanInDriver } from "./sse_fanin_driver"; +import { FanInCensus } from "./fanin_census"; import { readFleetSessionStatus, type StatusOwnerLookup } from "./fleet_session_status"; import { createObservationReadPlane } from "./observation_read_plane"; import { createObservationWritePlane } from "./observation_write_plane"; @@ -60,7 +62,7 @@ import { readPendingRequests, } from "./fleet_control_request"; import { HubCredentialRead, mintRegistry, readHubCredential } from "./hub_credential"; -import { buildMergedProjection, buildFleetProjection, buildOwnerRoutingProjection, type UpstreamMode, type MergedProjection, type FleetProjection } from "./merged_projection"; +import { buildMergedProjection, buildFleetProjection, buildOwnerRoutingProjection, type UpstreamMode, type MergedProjection, type FleetProjection, type OwnerRefreshPeer } from "./merged_projection"; import { buildControlResolver } from "./remote_session_state"; import { FleetPostureDetector, type FleetPostureTuning } from "./fleet_posture"; import { handleFleetWrite, type FleetWriteDeps } from "./fleet_writes"; @@ -930,6 +932,10 @@ export function createAmicodeService( * credential gate's refresh loop). Absent → the surface renders empty * roles (no cards known) with providers from the stored snapshot. */ modelRouting?: ModelRoutingDeps; + /** #1771: the service's diagnostic log (the hub runner's stdout → + * /tmp/amico-hub.log; the window's Amicode output channel). The fan-in + * census and the owner-map refresh outcomes write here. Absent → silent. */ + log?: (line: string) => void; fleet?: { /** Resolved entitlements (injectable for tests); null resolves the * machine's real set. */ @@ -1168,7 +1174,7 @@ export function createAmicodeService( if (fleetPeers) { const ownerMapFeed = new OwnerMapFeed({ ownerMap, - buildProjection: async (): Promise<{ sessions: SessionEntry[] }> => { + buildProjection: async (): Promise<{ sessions: SessionEntry[]; peers: OwnerRefreshPeer[] }> => { const peers = fleetPeers.getServingPeers().map((p) => { const tokenRead = fleetPeers.readPeerToken(p.machineId); return { @@ -1183,8 +1189,10 @@ export function createAmicodeService( peers, rosterLookup: fleetPeers.rosterLookup, }); - return { sessions: projection.sessions as SessionEntry[] }; + return { sessions: projection.sessions as SessionEntry[], peers: projection.peers }; }, + // #1771: each refresh's per-peer outcome → the service log. + ...(opts.log ? { log: opts.log } : {}), }); ownerMapFeed.start(); server.registerCleanup(() => ownerMapFeed.stop()); @@ -1207,14 +1215,23 @@ export function createAmicodeService( let eventFanIn: SseFanInDriver | undefined; if (fleetPeers && !isClient) { const peers = fleetPeers; - eventFanIn = new SseFanInDriver({ + // #1771: the fan-in census — 30 s census lines + immediate transition + // lines to the service log; torn down (and closes attributed to + // `shutdown`) with the server. + const census = new FanInCensus({ plane: "premium", ...(opts.log ? { sink: opts.log } : {}) }); + census.start(); + const premiumPeerBaseUrl = (machineId: string): string | undefined => { + const r = peers.readPeerToken(machineId); + return r.ok ? r.credential.baseUrl : undefined; + }; + const driver = new SseFanInDriver({ ownerMap, localMachineId: peers.localMachineId, localEventUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), - peerBaseUrl: (machineId) => { - const r = peers.readPeerToken(machineId); - return r.ok ? r.credential.baseUrl : undefined; - }, + peerBaseUrl: premiumPeerBaseUrl, + // #1774: a peer the owner map names unavailable (retention lapsed, or + // its refresh was refused) is never an arm — the aggregator announces it. + reachable: fanInReachable(ownerMap, (machineId) => premiumPeerBaseUrl(machineId) !== undefined), peerToken: (machineId) => { const r = peers.readPeerToken(machineId); return r.ok ? { ok: true, credential: r.credential } : { ok: false, reason: "absent" }; @@ -1223,7 +1240,10 @@ export function createAmicodeService( // the aggregator so the connect frame carries real focus data (not the // named-empty default). Absent → the aggregator's named-empty fallback. ...(opts.fleet.focusSnapshot ? { focusSnapshot: opts.fleet.focusSnapshot } : {}), + census, }); + server.registerCleanup(() => driver.shutdown()); + eventFanIn = driver; } const fleetPlaneObj: import("./server").FleetPlane = { getMode, @@ -1353,7 +1373,7 @@ export function createAmicodeService( const ownerMap = new SessionOwnerMap(); const ownerMapFeed = new OwnerMapFeed({ ownerMap, - buildProjection: async (): Promise<{ sessions: SessionEntry[] }> => { + buildProjection: async (): Promise<{ sessions: SessionEntry[]; peers: OwnerRefreshPeer[] }> => { const peers = fleetPeers.getServingPeers().map((p) => { const tokenRead = fleetPeers.readPeerToken(p.machineId); return { @@ -1368,8 +1388,10 @@ export function createAmicodeService( peers, rosterLookup: fleetPeers.rosterLookup, }); - return { sessions: projection.sessions as SessionEntry[] }; + return { sessions: projection.sessions as SessionEntry[], peers: projection.peers }; }, + // #1771: each refresh's per-peer outcome → the service log. + ...(opts.log ? { log: opts.log } : {}), }); ownerMapFeed.start(); server.registerCleanup(() => ownerMapFeed.stop()); @@ -1391,6 +1413,8 @@ export function createAmicodeService( const r = fleetPeers.readPeerToken(machineId); return r.ok ? { getUrl: () => r.credential.baseUrl, token: r.credential.token } : { getUrl: () => undefined }; }, + // #1774: the local-first probe for an unknown-owner read. + localEngineUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), ...(opts.fleet.dataPlaneTimeoutMs !== undefined ? { timeoutMs: opts.fleet.dataPlaneTimeoutMs } : {}), }), ); @@ -1422,6 +1446,8 @@ export function createAmicodeService( const g = findControlGrantByTarget(owner); return g ? { scope: g.scope, state: g.state } : undefined; }, + // #1774: the local-first probe for an unknown-owner write. + localEngineUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), ...(opts.fleet.dataPlaneTimeoutMs !== undefined ? { timeoutMs: opts.fleet.dataPlaneTimeoutMs } : {}), }), ); @@ -1436,22 +1462,30 @@ export function createAmicodeService( // — fleet-of-one byte-identity holds only absent a focus provider (D6). // Per-peer arms auth AS THEMSELVES with their OWN reader token (decision // A); the local arm rides the app's incoming credential. - server.attachObservationEventPlane( - new SseFanInDriver({ - ownerMap, - localMachineId: fleetPeers.localMachineId, - localEventUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), - peerBaseUrl: (machineId) => { - if (!fleetPeers.getServingPeers().some((p) => p.machineId === machineId)) return undefined; - const r = fleetPeers.readPeerToken(machineId); - return r.ok ? r.credential.baseUrl : undefined; - }, - peerToken: (machineId) => { - const r = fleetPeers.readPeerToken(machineId); - return r.ok ? { ok: true, credential: r.credential } : { ok: false, reason: "absent" }; - }, - }), - ); + // #1771: with its census (30 s census + transition lines → the log). + const observeCensus = new FanInCensus({ plane: "observe", ...(opts.log ? { sink: opts.log } : {}) }); + observeCensus.start(); + const observePeerBaseUrl = (machineId: string): string | undefined => { + if (!fleetPeers.getServingPeers().some((p) => p.machineId === machineId)) return undefined; + const r = fleetPeers.readPeerToken(machineId); + return r.ok ? r.credential.baseUrl : undefined; + }; + const observeDriver = new SseFanInDriver({ + ownerMap, + localMachineId: fleetPeers.localMachineId, + localEventUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), + peerBaseUrl: observePeerBaseUrl, + // #1774: a peer the owner map names unavailable (retention lapsed, or + // its refresh was refused) is never an arm — the aggregator announces it. + reachable: fanInReachable(ownerMap, (machineId) => observePeerBaseUrl(machineId) !== undefined), + peerToken: (machineId) => { + const r = fleetPeers.readPeerToken(machineId); + return r.ok ? { ok: true, credential: r.credential } : { ok: false, reason: "absent" }; + }, + census: observeCensus, + }); + server.registerCleanup(() => observeDriver.shutdown()); + server.attachObservationEventPlane(observeDriver); } } } diff --git a/packages/extension/src/amicode_service/local_session_probe.ts b/packages/extension/src/amicode_service/local_session_probe.ts new file mode 100644 index 000000000..178a67467 --- /dev/null +++ b/packages/extension/src/amicode_service/local_session_probe.ts @@ -0,0 +1,62 @@ +// LOCAL SESSION PROBE (#1774): "does the LOCAL engine hold this session?" — the +// local-first half of the unknown-owner outcome. A session-scoped request whose +// id has NO known owner (the owner map has not caught up — a just-created local +// session, or a peer whose entries expired) is tried on the local engine first; +// only the engine's own 404 for the session detail read means "not here", and +// the caller then answers a retryable owner-unknown 503 instead of letting the +// request become a 404 or an empty result. +// +// The probe is the engine's own session detail read (`GET /session/{id}` → 404 +// on a storage miss), sent with the request's OWN credential and directory scope +// (the same Authorization the engine proxy would forward), never a body. Any +// other answer — 2xx, an auth refusal, a 5xx, a transport error, a timeout — is +// not a verdict: the request falls through to the local engine, which answers +// for itself exactly as before. +import type * as http from "node:http"; + +/** present → the local engine holds it; absent → its own 404; unknown → no verdict. */ +export type LocalSessionPresence = "present" | "absent" | "unknown"; + +/** The probe's bound when the caller sets none — short: it sits in front of a request. */ +export const LOCAL_SESSION_PROBE_TIMEOUT_MS = 3_000; + +/** The directory/workspace scope carriers a request may hold (header or query). */ +const SCOPE_HEADERS = ["x-opencode-directory", "x-opencode-workspace"] as const; +const SCOPE_PARAMS = ["directory", "workspace", "auth_token"] as const; + +export async function probeLocalSession(opts: { + /** The local engine origin, read per request; undefined → no verdict. */ + localUrl: string | undefined; + sessionId: string; + /** The incoming request: its credential and scope ride on the probe. */ + req: Pick; + fetchImpl?: typeof fetch; + timeoutMs?: number; +}): Promise { + if (!opts.localUrl) return "unknown"; + try { + const target = new URL(`/session/${encodeURIComponent(opts.sessionId)}`, opts.localUrl); + const incoming = new URL(opts.req.url ?? "/", "http://origin"); + for (const p of SCOPE_PARAMS) { + const v = incoming.searchParams.get(p); + if (v !== null) target.searchParams.set(p, v); + } + const headers: Record = {}; + const auth = opts.req.headers.authorization; + if (typeof auth === "string") headers.authorization = auth; + for (const h of SCOPE_HEADERS) { + const v = opts.req.headers[h]; + if (typeof v === "string") headers[h] = v; + } + const res = await (opts.fetchImpl ?? fetch)(target.toString(), { + method: "GET", + headers, + signal: AbortSignal.timeout(opts.timeoutMs ?? LOCAL_SESSION_PROBE_TIMEOUT_MS), + }); + await res.body?.cancel().catch(() => undefined); + if (res.status === 404) return "absent"; + return res.ok ? "present" : "unknown"; + } catch { + return "unknown"; + } +} diff --git a/packages/extension/src/amicode_service/merged_projection.ts b/packages/extension/src/amicode_service/merged_projection.ts index 03e7efd41..e2446199c 100644 --- a/packages/extension/src/amicode_service/merged_projection.ts +++ b/packages/extension/src/amicode_service/merged_projection.ts @@ -98,6 +98,26 @@ export interface FleetProjectionOptions { archived?: boolean; fetchImpl?: typeof fetch; timeoutMs?: number; + /** #1771: the clock timing each source's fetch (the owner-map refresh's + * per-peer duration). Default Date.now. */ + now?: () => number; +} + +/** #1771: one peer's ownership-refresh outcome — what the OwnerMapFeed logs. + * `timeout` is the fetch guard firing (AbortSignal.timeout); `unauthorized` a + * 401; every other absence (fetch failure, no upstream, missing credential, + * untrusted) is `error`, with its named `reason`. */ +export type OwnerRefreshOutcome = "ok" | "timeout" | "error" | "unauthorized"; + +export interface OwnerRefreshPeer { + machineId: string; + outcome: OwnerRefreshOutcome; + /** The peer's fetch duration (ms; the slower of its active/archived reads). */ + durationMs: number; + /** The peer-owned sessions this refresh put in the routing union. */ + entries: number; + reason?: SourceAbsenceReason; + detail?: string; } /** The fleet-wide projection result — N sources, machine-keyed. */ @@ -213,19 +233,27 @@ async function fetchSessions( * Peers/hub keep the old behavior (false): no credential ⇒ do not fire an * unauthenticated cross-machine request. */ allowUnauthenticated = false, -): Promise<{ record: SourceFetchRecord; entries: Record[] }> { + /** #1771: the clock timing this fetch. */ + now: () => number = Date.now, +): Promise { + const started = now(); + const done = (r: { record: SourceFetchRecord; entries: Record[] }, timedOut = false): SourceFetch => ({ + ...r, + durationMs: Math.max(0, now() - started), + timedOut, + }); const base: SourceFetchRecord = { source: tag, present: false }; const origin = opts.getUrl(); - if (!origin) return { record: { ...base, reason: "no-upstream" }, entries: [] }; + if (!origin) return done({ record: { ...base, reason: "no-upstream" }, entries: [] }); if (!authHeader && !allowUnauthenticated) { - return { + return done({ record: { ...base, reason: tag === "hub" ? "credential-missing" : "credential-missing", detail: tag === "hub" ? "hub credential absent/malformed/incomplete" : "engine mint not armed", }, entries: [], - }; + }); } // Send Authorization only when we actually hold one — the open loopback // engine is queried bare (allowUnauthenticated with no password). @@ -235,13 +263,13 @@ async function fetchSessions( headers: authHeaders, signal: AbortSignal.timeout(timeoutMs), }); - if (res.status === 401) return { record: { ...base, reason: "unauthorized" }, entries: [] }; + if (res.status === 401) return done({ record: { ...base, reason: "unauthorized" }, entries: [] }); if (!res.ok) { - return { record: { ...base, reason: "fetch-failed", detail: `HTTP ${res.status}` }, entries: [] }; + return done({ record: { ...base, reason: "fetch-failed", detail: `HTTP ${res.status}` }, entries: [] }); } const parsed: unknown = await res.json(); if (!Array.isArray(parsed)) { - return { record: { ...base, reason: "fetch-failed", detail: "session list is not an array" }, entries: [] }; + return done({ record: { ...base, reason: "fetch-failed", detail: "session list is not an array" }, entries: [] }); } const entries = parsed.filter( (e): e is Record => typeof e === "object" && e !== null && !Array.isArray(e), @@ -272,15 +300,31 @@ async function fetchSessions( /* the stamp fails open to null — the record stays honest via the field */ } const record: SourceFetchRecord = { source: tag, present: true, count: entries.length, max, sum, version }; - return { record, entries }; + return done({ record, entries }); } catch (e) { - return { - record: { ...base, reason: "fetch-failed", detail: e instanceof Error ? e.message : String(e) }, - entries: [], - }; + // #1771: the fetch guard firing is a TIMEOUT (the record keeps its named + // `fetch-failed` reason; only the refresh outcome distinguishes it). + const timedOut = (e as { name?: unknown } | null)?.name === "TimeoutError"; + return done( + { + record: { ...base, reason: "fetch-failed", detail: e instanceof Error ? e.message : String(e) }, + entries: [], + }, + timedOut, + ); } } +/** One source fetch: the NAMED record + entries, plus (#1771) how long it took + * and whether the fetch guard timed it out. Internal — the served projection + * carries only the record. */ +interface SourceFetch { + record: SourceFetchRecord; + entries: Record[]; + durationMs: number; + timedOut: boolean; +} + /** Merge both stores into ONE provenance-tagged list. The hub wins conflicts * (the store of record for fleet sessions); locals stay tagged local. */ export function mergeSessions( @@ -419,8 +463,15 @@ function mergeNSources( * Never throws: every peer failure is a NAMED record inside an otherwise-valid * projection — the read path degrades by naming, not by vanishing. */ export async function buildFleetProjection(opts: FleetProjectionOptions): Promise { + return (await fleetFanOut(opts)).projection; +} + +/** The fan-out behind buildFleetProjection, also returning each PEER's raw + * fetch (#1771: its duration + timeout flag) in `opts.peers` order. */ +async function fleetFanOut(opts: FleetProjectionOptions): Promise<{ projection: FleetProjection; peerFetches: SourceFetch[] }> { const fetchImpl = opts.fetchImpl ?? fetch; const timeoutMs = opts.timeoutMs ?? 10_000; + const now = opts.now ?? Date.now; const localAuth = opts.local.password !== undefined ? serverAuthHeader(opts.local.password) : undefined; // Fan out: local + all peers in parallel. @@ -436,8 +487,8 @@ export async function buildFleetProjection(opts: FleetProjectionOptions): Promis const fleetEndpoint = opts.archived ? "/experimental/session?archived=true&limit=10000" : "/experimental/session?limit=10000"; - const localPromise = fetchSessions(opts.localMachineId, opts.local, localAuth, fetchImpl, timeoutMs, fleetEndpoint, true); - const peerPromises = opts.peers.map((peer) => { + const localPromise = fetchSessions(opts.localMachineId, opts.local, localAuth, fetchImpl, timeoutMs, fleetEndpoint, true, now); + const peerPromises = opts.peers.map((peer): Promise => { // #1481 (AC1): TRUST gates Observe. An untrusted peer (no Observe grant) is // never contacted — it resolves to a NAMED `untrusted` record with zero // entries, so no session metadata can leak. Trust is checked BEFORE the @@ -446,10 +497,12 @@ export async function buildFleetProjection(opts: FleetProjectionOptions): Promis return Promise.resolve({ record: { source: peer.machineId, present: false, reason: "untrusted" as const } satisfies SourceFetchRecord, entries: [] as Record[], + durationMs: 0, + timedOut: false, }); } const auth = peer.token !== undefined ? peerAuthHeader(peer.token) : undefined; - return fetchSessions(peer.machineId, { getUrl: peer.getUrl }, auth, fetchImpl, timeoutMs, fleetEndpoint); + return fetchSessions(peer.machineId, { getUrl: peer.getUrl }, auth, fetchImpl, timeoutMs, fleetEndpoint, false, now); }); const [localResult, ...peerResults] = await Promise.all([localPromise, ...peerPromises]); @@ -496,7 +549,18 @@ export async function buildFleetProjection(opts: FleetProjectionOptions): Promis const allRecords = [localResult.record, ...peerResults.map((r) => r.record)]; const currency = deriveCurrency(allRecords); - return { ok: true, mode: "fleet", sessions, sources, currency: { ...currency, derived_over: "fetched" } }; + return { + projection: { ok: true, mode: "fleet", sessions, sources, currency: { ...currency, derived_over: "fetched" } }, + peerFetches: peerResults, + }; +} + +/** #1771: classify one peer fetch as a refresh outcome. */ +function refreshOutcomeOf(f: SourceFetch): OwnerRefreshOutcome { + if (f.record.present) return "ok"; + if (f.timedOut) return "timeout"; + if (f.record.reason === "unauthorized") return "unauthorized"; + return "error"; } /** #1647 (S2): the OWNER-ROUTING session set — active AND archived unioned. @@ -505,18 +569,41 @@ export async function buildFleetProjection(opts: FleetProjectionOptions): Promis * ARCHIVED session (unarchive / delete of an archived row) has no owner entry * and misroutes to the LOCAL engine (the #1382 silent-local failure). This * runs BOTH variants and unions by id so every peer-owned session — active or - * archived — is routable. Owner tags are carried through unchanged. */ + * archived — is routable. Owner tags are carried through unchanged. + * #1771: also returns each PEER's refresh outcome (`peers`, in `opts.peers` + * order) — the first non-ok of its two reads, the slower read's duration, and + * the peer-owned entries this union carries — which the OwnerMapFeed logs. */ export async function buildOwnerRoutingProjection( opts: Omit, -): Promise<{ sessions: FleetProjection["sessions"] }> { +): Promise<{ sessions: FleetProjection["sessions"]; peers: OwnerRefreshPeer[] }> { const [active, archived] = await Promise.all([ - buildFleetProjection({ ...opts, archived: false }), - buildFleetProjection({ ...opts, archived: true }), + fleetFanOut({ ...opts, archived: false }), + fleetFanOut({ ...opts, archived: true }), ]); const byId = new Map(); - for (const s of [...active.sessions, ...archived.sessions]) { + for (const s of [...active.projection.sessions, ...archived.projection.sessions]) { const id = typeof s.id === "string" ? s.id : undefined; if (id && !byId.has(id)) byId.set(id, s); } - return { sessions: [...byId.values()] }; + const sessions = [...byId.values()]; + const owned = new Map(); + for (const s of sessions) { + const owner = s.amicode_owner?.owner_machine_id; + if (owner !== undefined) owned.set(owner, (owned.get(owner) ?? 0) + 1); + } + const peers = opts.peers.map((peer, i): OwnerRefreshPeer => { + const a = active.peerFetches[i]; + const b = archived.peerFetches[i]; + const worst = refreshOutcomeOf(a) !== "ok" ? a : b; + const outcome = refreshOutcomeOf(worst); + return { + machineId: peer.machineId, + outcome, + durationMs: Math.max(a.durationMs, b.durationMs), + entries: owned.get(peer.machineId) ?? 0, + ...(outcome !== "ok" && worst.record.reason !== undefined ? { reason: worst.record.reason } : {}), + ...(outcome !== "ok" && worst.record.detail !== undefined ? { detail: worst.record.detail } : {}), + }; + }); + return { sessions, peers }; } diff --git a/packages/extension/src/amicode_service/observation_read_plane.ts b/packages/extension/src/amicode_service/observation_read_plane.ts index b8453ef98..dc0a0df96 100644 --- a/packages/extension/src/amicode_service/observation_read_plane.ts +++ b/packages/extension/src/amicode_service/observation_read_plane.ts @@ -22,17 +22,24 @@ import { type ObservationReadRouterOpts, type ObservationReadPeerTarget, } from "./session_multiplexer"; +import { probeLocalSession } from "./local_session_probe"; import type { ObservationReadPlane } from "./server"; /** Build the observation-only read plane: a resolver (path→owner peer) plus the * peer-token proxy. `timeoutMs` bounds the headers wait (SSE bodies ride past - * it), mirroring the hub/attached proxies. */ + * it), mirroring the hub/attached proxies. #1774: `localEngineUrl` (the local + * engine origin, read per request) arms the local-first probe for an + * unknown-owner read; absent → no verdict, the read falls through local. */ export function createObservationReadPlane( - opts: ObservationReadRouterOpts & { timeoutMs?: number }, + opts: ObservationReadRouterOpts & { timeoutMs?: number; localEngineUrl?: () => string | undefined }, ): ObservationReadPlane { const router = new ObservationReadRouter(opts); + const localEngineUrl = opts.localEngineUrl; return { resolve: (method, pathname) => router.resolve(method, pathname), + ...(localEngineUrl + ? { probeLocal: (req: http.IncomingMessage, sessionId: string) => probeLocalSession({ localUrl: localEngineUrl(), sessionId, req }) } + : {}), proxyToPeer(req: http.IncomingMessage, res: http.ServerResponse, target: ObservationReadPeerTarget): boolean { // A synthetic per-peer hub credential → HubProxy attaches // serverAuthHeader(target.token) === peerAuthHeader(target.token). diff --git a/packages/extension/src/amicode_service/observation_write_plane.ts b/packages/extension/src/amicode_service/observation_write_plane.ts index 2c8954380..f08ca7696 100644 --- a/packages/extension/src/amicode_service/observation_write_plane.ts +++ b/packages/extension/src/amicode_service/observation_write_plane.ts @@ -40,6 +40,7 @@ import { type ObservationWriteRouterOpts, type ObservationWritePeerTarget, } from "./session_multiplexer"; +import { probeLocalSession } from "./local_session_probe"; import type { ObservationWritePlane } from "./server"; /** Does the owning peer host a project whose worktree equals `dir`? Probes the @@ -69,14 +70,20 @@ async function peerHasProjectWorktree(peerUrl: string, token: string, dir: strin * ignored) that AUTHORIZES through the pure write gate, plus the peer proxy * presenting the credential the owner accepts (the peer reader token). * `timeoutMs` bounds the headers wait (SSE bodies ride past it), mirroring the - * read plane / hub / attached proxies. */ + * read plane / hub / attached proxies. #1774: `localEngineUrl` (the local + * engine origin, read per request) arms the local-first probe for an + * unknown-owner write; absent → no verdict, the write falls through local. */ export function createObservationWritePlane( - opts: ObservationWriteRouterOpts & { timeoutMs?: number }, + opts: ObservationWriteRouterOpts & { timeoutMs?: number; localEngineUrl?: () => string | undefined }, ): ObservationWritePlane { const router = new ObservationWriteRouter(opts); + const localEngineUrl = opts.localEngineUrl; return { resolve: (method, pathname) => router.resolve(method, pathname), resolveCreate: (method, pathname, headers) => router.resolveCreate(method, pathname, headers), + ...(localEngineUrl + ? { probeLocal: (req: http.IncomingMessage, sessionId: string) => probeLocalSession({ localUrl: localEngineUrl(), sessionId, req }) } + : {}), proxyToPeer(req: http.IncomingMessage, res: http.ServerResponse, target: ObservationWritePeerTarget): boolean { // A synthetic per-peer hub credential → HubProxy attaches // serverAuthHeader(target.token) === peerAuthHeader(target.token). The diff --git a/packages/extension/src/amicode_service/server.ts b/packages/extension/src/amicode_service/server.ts index 98368ecfc..8136b12c9 100644 --- a/packages/extension/src/amicode_service/server.ts +++ b/packages/extension/src/amicode_service/server.ts @@ -34,6 +34,8 @@ import type { ObservationWriteResolution, ObservationWritePeerTarget, } from "./session_multiplexer"; +import { OWNER_MAP_REFRESH_MS } from "./session_multiplexer"; +import type { LocalSessionPresence } from "./local_session_probe"; import type { EventFanInDriver } from "./sse_fanin_driver"; import { BOUND_NONCE_HEADER, BOUND_IDENTITY_HEADER } from "./fleet_bootstrap_headers"; @@ -183,6 +185,9 @@ export interface ObservationReadPlane { * true when it owns the response; false when no upstream is bound (→ the * caller answers a named 503, never local). */ proxyToPeer(req: http.IncomingMessage, res: http.ServerResponse, target: ObservationReadPeerTarget): boolean; + /** #1774: does the LOCAL engine hold this session? Consulted for an + * unknown-owner request (local first). Absent → no verdict (fall through). */ + probeLocal?(req: http.IncomingMessage, sessionId: string): Promise; } /** #1542 (Fleet Studio B2b, WRITE seam): the OBSERVATION-ONLY per-session @@ -224,6 +229,9 @@ export interface ObservationWritePlane { res: http.ServerResponse, target: ObservationWritePeerTarget, ): Promise; + /** #1774: does the LOCAL engine hold this session? Consulted for an + * unknown-owner write (local first). Absent → no verdict (fall through). */ + probeLocal?(req: http.IncomingMessage, sessionId: string): Promise; } /** #1261 (AC6): a client's own named hub-down state — distinct from the base @@ -241,6 +249,28 @@ export const FLEET_PEER_UNREACHABLE_POINTER = "the attached server is unreachable — check the peer's tunnel / service, " + "or detach to work locally"; +/** #1774: the NAMED retryable answer for a session-scoped request whose id has + * no known owner and that the LOCAL engine does not hold — the owner map may + * simply not have caught up (a just-created session, a peer whose entries + * expired), so it is a 503 with a Retry-After of one owner-map refresh, never a + * 404 or an empty result. */ +export const OWNER_UNKNOWN_ERROR = "session owner unknown"; +export const OWNER_UNKNOWN_RETRY_AFTER_S = Math.ceil(OWNER_MAP_REFRESH_MS / 1000); + +function ownerUnknown(sessionId: string): AmicodeHandlerResult { + return { + status: 503, + headers: { "Retry-After": String(OWNER_UNKNOWN_RETRY_AFTER_S) }, + body: JSON.stringify({ + ok: false, + error: OWNER_UNKNOWN_ERROR, + reason: "owner-unknown", + session_id: sessionId, + retry_after_s: OWNER_UNKNOWN_RETRY_AFTER_S, + }), + }; +} + /** #1448 (W1a): the session-multiplexer feature-flag env var. Default ON. */ export const FLEET_MULTIPLEX_FLAG = "AMICO_FLEET_MULTIPLEX"; @@ -581,7 +611,9 @@ export class AmicodeServiceServer { return; } // #1519 (W1c, ADR 0033 §D1): the SSE fan-in interception for the app's - // ONE global /event stream (ADR 0027 single-origin). AHEAD of every + // global /event stream. #1772: one fan-in connection PER LISTENER (windows, + // Chat Deck panes, split panes) — a second view no longer evicts the first; + // beyond the driver's cap a listener gets a retryable 503. AHEAD of every // downstream path (the fleet hub block, the engine proxy) so a machine // that owns peer sessions fans their events IN onto this one response. // A SINGLE flag-gated expression: with the flag OFF (default) the `&&` @@ -803,7 +835,15 @@ export class AmicodeServiceServer { // peer-unreachable 503, NEVER the local engine (the #1382 invariant). if (this.observeRead) { const decision = this.observeRead.resolve(req.method ?? "GET", url.pathname); - if (decision) { + if (decision?.kind === "unknown-owner") { + // #1774: no known owner → the LOCAL engine first; only its own 404 + // is "not here" → the retryable owner-unknown 503. Otherwise the read + // falls through to the engine proxy below, byte-identical. + if ((await this.observeRead.probeLocal?.(req, decision.sessionId)) === "absent") { + send(ownerUnknown(decision.sessionId)); + return; + } + } else if (decision) { if (decision.kind === "peer") { if (this.observeRead.proxyToPeer(req, res, decision)) return; } @@ -841,7 +881,16 @@ export class AmicodeServiceServer { // pathed write streams (proxyToPeer). const createDecision = this.observeWrite.resolveCreate(req.method ?? "GET", url.pathname, req.headers); const decision = createDecision ?? this.observeWrite.resolve(req.method ?? "GET", url.pathname); - if (decision) { + if (decision?.kind === "unknown-owner") { + // #1774: no known owner → the LOCAL engine first (an existence probe, + // never the write itself); only its own 404 is "not here" → the + // retryable owner-unknown 503, and the write runs nowhere. Otherwise + // it falls through to the engine proxy below, byte-identical. + if ((await this.observeWrite.probeLocal?.(req, decision.sessionId)) === "absent") { + send(ownerUnknown(decision.sessionId)); + return; + } + } else if (decision) { if (decision.kind === "peer") { const handled = createDecision ? await this.observeWrite.proxyCreateToPeer(req, res, decision) diff --git a/packages/extension/src/amicode_service/session_multiplexer.ts b/packages/extension/src/amicode_service/session_multiplexer.ts index 0abf3fd72..ede8cb22c 100644 --- a/packages/extension/src/amicode_service/session_multiplexer.ts +++ b/packages/extension/src/amicode_service/session_multiplexer.ts @@ -17,7 +17,7 @@ // it is a proxy, never a redirect. The app holds ONE base URL throughout. import * as http from "node:http"; -import type { SessionOwnerTag } from "./merged_projection"; +import type { SessionOwnerTag, OwnerRefreshPeer } from "./merged_projection"; import { evaluateRemoteWriteGate, type WriteGrantRead } from "./remote_write_gate"; // ── owner-routing header ───────────────────────────────────────────────────── @@ -77,6 +77,12 @@ export function extractSessionIdFromReadPath(pathname: string): string | undefin * never really created, or was deleted) and drops back to local. */ export const PENDING_OWNER_TTL_MS = 60_000; +/** #1774: how long a peer's last-good owner entries survive refreshes in which + * that peer's fetch FAILED (timeout / network error / 5xx). A peer's session set + * changes slowly relative to this; past it the entries expire. Configurable per + * map (`retentionMs`). */ +export const OWNER_RETENTION_MS = 60_000; + /** A session entry with owner tag (the fleet-wide projection's shape). */ export interface SessionEntry { id: string; @@ -84,6 +90,38 @@ export interface SessionEntry { [key: string]: unknown; } +/** #1774: does a failed peer refresh RETAIN the peer's last-good entries? Decided + * from #1771's per-peer outcome: a timeout, or a transport failure (a network + * error, a 5xx, the peer's upstream momentarily absent) retains. A credential or + * trust problem — unauthorized, untrusted, credential missing, identity + * conflict — does not: it is not a blip, and it surfaces as unavailable. */ +export function refreshRetains(peer: Pick): boolean { + if (peer.outcome === "timeout") return true; + if (peer.outcome !== "error") return false; + return peer.reason === "fetch-failed" || peer.reason === "no-upstream"; +} + +/** #1774: what one refresh did to ONE peer's owner entries (for the refresh log). + * - retained — the fetch failed within the window: `held` last-good entries + * kept, last confirmed `ageMs` ago; + * - expired — the window lapsed: `dropped` entries expired, the peer is now + * named unavailable; + * - dropped — a non-retaining failure (unauthorized, …): `dropped` entries + * removed at once, the peer is now named unavailable; + * - unavailable — the peer holds nothing and is still named unavailable. + * A successful refresh (or a failure with nothing ever held) has no record. */ +export type OwnerRetention = + | { kind: "retained"; held: number; ageMs: number } + | { kind: "expired"; dropped: number } + | { kind: "dropped"; dropped: number } + | { kind: "unavailable" }; + +/** One owned entry as the map holds it (the owner is the map key's peer). */ +interface HeldEntry { + id: string; + directory?: string; +} + /** The per-session owner binding — maps session IDs to their owner machine_id. * Populated from the fleet-wide projection's amicode_owner overlay (slice 2). * A miss returns undefined, which the multiplexer interprets as "local". @@ -92,7 +130,16 @@ export interface SessionEntry { * a peer is not yet in the pull-only projection, so a PENDING overlay bridges * the gap — `recordPendingOwner` binds the new id at create time, the binding * survives projection refreshes that don't yet include it, is superseded once - * the projection does, and expires after a TTL. */ + * the projection does, and expires after a TTL. + * + * #1774: when the projection carries #1771's PER-PEER refresh outcomes, the + * update is authoritative only for the peers that answered. A peer whose fetch + * failed (an empty slice, not an error) keeps its LAST-GOOD entries for + * `retentionMs` — its sessions keep routing to it and its fan-in arm stays open + * — then they expire and the peer is NAMED unavailable (still an owner in + * `ownerMachineIds()`, never a reachable arm — see `fanInReachable`) until a + * successful refresh. A non-retaining failure (unauthorized, …) drops the + * entries at once and names the peer unavailable the same way. */ export class SessionOwnerMap { private readonly owners = new Map(); /** #1766: each OWNED session's working directory (id → directory), recorded @@ -101,34 +148,106 @@ export class SessionOwnerMap { private readonly directories = new Map(); /** Create-time bindings awaiting projection confirmation (id → {owner, at}). */ private readonly pending = new Map(); + /** #1774: each peer's entries as of its last SUCCESSFUL refresh (peer → {at, entries}). */ + private readonly lastGood = new Map(); + /** #1774: peers whose entries expired or were dropped — named unavailable. */ + private readonly unavailablePeers = new Set(); private readonly now: () => number; + private readonly retentionMs: number; - constructor(opts: { now?: () => number } = {}) { + constructor(opts: { now?: () => number; retentionMs?: number } = {}) { this.now = opts.now ?? Date.now; + this.retentionMs = opts.retentionMs ?? OWNER_RETENTION_MS; } /** Rebuild the map from a fleet-wide projection's session list. The projection * is authoritative: a pending binding now present in it is superseded (dropped - * from the overlay); expired pending bindings are pruned. */ - update(sessions: SessionEntry[]): void { + * from the overlay); expired pending bindings are pruned. + * + * #1774: `peers` (the projection's per-peer refresh outcomes) scopes that + * authority — see the class comment. Returns what the update did to each + * failed peer's entries (absent for a peer that answered). Without `peers` + * every owner is replaced, exactly as before. */ + update( + sessions: SessionEntry[], + peers?: ReadonlyArray>, + ): Map { this.owners.clear(); this.directories.clear(); + const fresh = new Map(); for (const s of sessions) { if (typeof s.id === "string" && s.amicode_owner?.owner_machine_id) { - this.owners.set(s.id, s.amicode_owner.owner_machine_id); + const owner = s.amicode_owner.owner_machine_id; // #1766: the session record's own `directory` (the cross-project list // carries it verbatim); the owner tag mirrors the same value, so it is // the fallback when an entry carries no top-level field. const directory = typeof s.directory === "string" && s.directory !== "" ? s.directory : s.amicode_owner.directory; - if (typeof directory === "string" && directory !== "") this.directories.set(s.id, directory); + const held: HeldEntry = { id: s.id, ...(typeof directory === "string" && directory !== "" ? { directory } : {}) }; + this.hold(owner, held); + const list = fresh.get(owner); + if (list) list.push(held); + else fresh.set(owner, [held]); } } + const retention = peers === undefined ? this.forgetPeers() : this.applyPeerOutcomes(fresh, peers); // Prune the pending overlay: drop entries the authoritative projection now // carries (superseded), and any that have outlived the TTL. const cutoff = this.now() - PENDING_OWNER_TTL_MS; for (const [id, entry] of this.pending) { if (this.owners.has(id) || entry.at < cutoff) this.pending.delete(id); } + return retention; + } + + private hold(owner: string, e: HeldEntry): void { + this.owners.set(e.id, owner); + if (e.directory !== undefined) this.directories.set(e.id, e.directory); + } + + /** No per-peer outcomes: the projection is authoritative for every owner, so + * there is nothing to retain and no one to name unavailable. */ + private forgetPeers(): Map { + this.lastGood.clear(); + this.unavailablePeers.clear(); + return new Map(); + } + + /** #1774: per peer — an answer replaces, a retaining failure keeps the + * last-good entries inside the window, anything else drops them. */ + private applyPeerOutcomes( + fresh: Map, + peers: ReadonlyArray>, + ): Map { + const now = this.now(); + const out = new Map(); + const refreshed = new Set(peers.map((p) => p.machineId)); + // A peer no longer refreshed at all left the fleet's serving set: nothing + // of it is retained, and it is not unavailable — it is gone. + for (const id of [...this.lastGood.keys()]) if (!refreshed.has(id)) this.lastGood.delete(id); + for (const id of [...this.unavailablePeers]) if (!refreshed.has(id)) this.unavailablePeers.delete(id); + for (const peer of peers) { + const id = peer.machineId; + if (peer.outcome === "ok") { + const entries = fresh.get(id) ?? []; + if (entries.length > 0) this.lastGood.set(id, { at: now, entries }); + else this.lastGood.delete(id); + this.unavailablePeers.delete(id); + continue; + } + const last = this.lastGood.get(id); + if (last && refreshRetains(peer) && now - last.at <= this.retentionMs) { + // keep what this refresh did not re-confirm; never shadow a fresh entry + for (const e of last.entries) if (!this.owners.has(e.id)) this.hold(id, e); + out.set(id, { kind: "retained", held: last.entries.length, ageMs: now - last.at }); + } else if (last) { + this.lastGood.delete(id); + this.unavailablePeers.add(id); + out.set(id, refreshRetains(peer) ? { kind: "expired", dropped: last.entries.length } : { kind: "dropped", dropped: last.entries.length }); + } else if (this.unavailablePeers.has(id)) { + out.set(id, { kind: "unavailable" }); + } + } + return out; } /** Bind a just-created session to its owner peer BEFORE the projection catches @@ -165,9 +284,18 @@ export class SessionOwnerMap { /** The DISTINCT owner machine_ids currently holding ≥1 owned session (#1511, * §D1) — the fan-in membership's owner set. Order is unspecified; callers - * exclude `local` themselves. */ + * exclude `local` themselves. #1774: plus every peer NAMED unavailable (its + * entries expired or were dropped), so the fan-in keeps it as a member and + * announces it unavailable instead of dropping its arm in silence — the + * driver's `fanInReachable` keeps such a member from ever being an arm. */ ownerMachineIds(): string[] { - return [...new Set(this.owners.values())]; + return [...new Set([...this.owners.values(), ...this.unavailablePeers])]; + } + + /** #1774: is this peer NAMED unavailable — its retention window lapsed, or its + * refresh was refused — with no successful refresh since? */ + isUnavailable(machineId: string): boolean { + return this.unavailablePeers.has(machineId); } /** The number of tracked sessions (for diagnostics). */ @@ -178,6 +306,18 @@ export class SessionOwnerMap { // ── owner-map feed (#1449, W1b) ────────────────────────────────────────────── +/** #1774: the fan-in driver's per-owner reachability, composed over the owner + * map — an owner-peer the map NAMES unavailable is never an active arm, whatever + * its transport says, so the aggregator closes its arm with the existing named + * unavailable comment instead of dropping it in silence. Both fan-in drivers in + * index.ts are wired with this. */ +export function fanInReachable( + ownerMap: SessionOwnerMap, + transportUp: (machineId: string) => boolean, +): (machineId: string) => boolean { + return (machineId) => !ownerMap.isUnavailable(machineId) && transportUp(machineId); +} + /** The live loop that keeps a SessionOwnerMap populated (#1449, W1b, AC1). * * The fleet-wide projection is PULL-ONLY — it is rebuilt per request at the @@ -191,17 +331,48 @@ export class SessionOwnerMap { * Honesty: a projection BUILD failure (a peer fan-out that threw) leaves the * LAST-GOOD map intact — it never clears the map on error, so a transient blip * does not silently reroute owned sessions to local. Only a SUCCESSFUL - * projection re-populates the map (clear + repopulate, the SessionOwnerMap.update - * contract). */ + * projection re-populates the map (the SessionOwnerMap.update contract). + * #1774: the builder itself never throws — a failed PEER is an empty slice plus + * its #1771 outcome in `peers`, which the feed hands to `update` so that peer + * keeps its last-good entries for the retention window. */ export interface OwnerMapFeedOpts { ownerMap: SessionOwnerMap; /** Build the fleet-wide projection whose `.sessions` carry the amicode_owner - * overlay. Injected so the feed is unit-testable without a real fan-out. */ - buildProjection: () => Promise<{ sessions: SessionEntry[] }>; + * overlay. Injected so the feed is unit-testable without a real fan-out. + * #1771: `peers` (optional) carries each peer's refresh outcome — the feed + * logs one line per peer, so a failed peer refresh is visible even though + * neither the builder nor the feed ever throws. #1774: and scopes the map + * update per peer (retention); absent → every owner is replaced. */ + buildProjection: () => Promise<{ sessions: SessionEntry[]; peers?: OwnerRefreshPeer[] }>; /** The refresh cadence in ms. Default 5000. */ intervalMs?: number; /** Observe a build failure (diagnostics only — the feed never throws). */ onError?: (err: unknown) => void; + /** #1771: the service log for per-peer refresh outcomes. Absent → silent. */ + log?: (line: string) => void; +} + +/** #1771: one per-peer refresh line: outcome, duration, entry count, and for a + * failure its named reason and detail. #1774: plus what the owner map did with + * a failed peer's entries (`retention=`), so a retained peer reads as retained — + * `entries` stays this refresh's own count. Absent for an answered peer. */ +export function ownerRefreshLine(p: OwnerRefreshPeer, retention?: OwnerRetention): string { + const reason = p.reason !== undefined ? ` reason=${p.reason}` : ""; + const detail = p.detail !== undefined ? ` detail=${JSON.stringify(p.detail)}` : ""; + return `[owner-map] refresh peer=${p.machineId} outcome=${p.outcome} duration_ms=${p.durationMs} entries=${p.entries}${reason}${detail}${retentionFields(retention)}`; +} + +function retentionFields(r: OwnerRetention | undefined): string { + if (r === undefined) return ""; + switch (r.kind) { + case "retained": + return ` retention=retained held=${r.held} age_ms=${r.ageMs}`; + case "expired": + case "dropped": + return ` retention=${r.kind} dropped=${r.dropped}`; + case "unavailable": + return " retention=unavailable"; + } } /** The default owner-map refresh cadence. */ @@ -216,15 +387,27 @@ export class OwnerMapFeed { /** Rebuild the projection ONCE and update the owner-map. Never throws: a * build failure is reported via `onError` and the last-good map is kept. */ async refreshOnce(): Promise { - let projection: { sessions: SessionEntry[] }; + let projection: { sessions: SessionEntry[]; peers?: OwnerRefreshPeer[] }; try { projection = await this.opts.buildProjection(); } catch (err) { this.opts.onError?.(err); + this.log(`[owner-map] refresh failed error=${JSON.stringify(err instanceof Error ? err.message : String(err))}`); return; // last-good map intact — never clear on a transient failure } if (this.stopped) return; // a stop() raced the in-flight build - this.opts.ownerMap.update(projection.sessions ?? []); + // #1774: the per-peer outcomes scope the update — a failed peer keeps its + // last-good entries for the retention window instead of vanishing. + const retention = this.opts.ownerMap.update(projection.sessions ?? [], projection.peers); + for (const peer of projection.peers ?? []) this.log(ownerRefreshLine(peer, retention.get(peer.machineId))); + } + + private log(line: string): void { + try { + this.opts.log?.(line); + } catch { + /* a broken log sink never breaks the feed */ + } } /** Start the loop: fire an immediate refresh, then re-refresh on the @@ -264,16 +447,26 @@ export interface PeerTransport { /** A REACHABLE peer read target — proxy the read to `url` with `token`. */ export type ObservationReadPeerTarget = { kind: "peer"; machineId: string; url: string; token: string }; +/** #1774: a session-scoped request whose id has NO known owner (not in the owner + * map, no pending binding). The caller tries the LOCAL engine first; if the + * local engine does not hold the session it answers a retryable owner-unknown + * 503 — never a 404 or an empty result (the map may simply not have caught up). */ +export type ObservationUnknownOwner = { kind: "unknown-owner"; sessionId: string }; + /** The resolution of an observation-mode read: * - a REACHABLE peer target → proxy to the owner with its reader token; * - the honest DEGRADED variant (owner is a KNOWN peer, but its transport is * down — getUrl() undefined / no token) → the caller answers a named 503, * NEVER local-dressed-as-peer (the #1382 invariant); + * - #1774 the unknown-owner outcome → local first, then a retryable 503; * - `undefined` (NOT a target) → LOCAL, the fail-safe: a write, a keyless / - * non-session path, /amicode/*, a local-owned or unowned session, or an - * owner that is not a known peer. The read falls through BYTE-IDENTICAL to - * the local engine. */ -export type ObservationReadResolution = ObservationReadPeerTarget | { kind: "degraded"; machineId: string }; + * non-session path, /amicode/*, a local-owned session, or an owner that is + * not a known peer. The read falls through BYTE-IDENTICAL to the local + * engine. */ +export type ObservationReadResolution = + | ObservationReadPeerTarget + | { kind: "degraded"; machineId: string } + | ObservationUnknownOwner; /** Options for the ObservationReadRouter. */ export interface ObservationReadRouterOpts { @@ -314,7 +507,7 @@ export class ObservationReadRouter { * 0. NON-read (POST/PATCH/DELETE/…) → local (writes NEVER route) * 1. /amicode/* (the machine's own honesty + local surface) → local * 2. no session id extractable from the path → local - * 3. owner unknown (unowned) → local + * 3. owner unknown (unowned) → #1774 the unknown-owner outcome (local first) * 4. owner == localMachineId (local-owned) → local * 5. owner is not a known peer → local * 6. owner is a known peer, transport down (no url/token) → DEGRADED (503) @@ -328,7 +521,7 @@ export class ObservationReadRouter { const sessionId = extractSessionIdFromReadPath(pathname); // (2) if (!sessionId) return undefined; const owner = this.ownerMap.resolveOwner(sessionId); // (3) - if (!owner) return undefined; + if (!owner) return { kind: "unknown-owner", sessionId }; if (owner === this.localMachineId) return undefined; // (4) const peer = this.peer(owner); // (5) if (!peer) return undefined; @@ -358,11 +551,12 @@ export type ObservationWriteDenied = { kind: "denied"; machineId: string; reason * credential (the reader token); * - a NAMED deny → the caller answers the gate's honest deny (an auth denial, * or the peer-unreachable 503 for `transport-down`), NEVER local (#1382); + * - #1774 the unknown-owner outcome → local first, then a retryable 503; * - `undefined` (NOT a target) → LOCAL, the fail-safe: a GET/HEAD (the read * plane owns those — the INVERTED decision), a keyless / non-session path, - * /amicode/*, a local-owned or unowned session, or an owner that is not a - * serving peer. The write falls through BYTE-IDENTICAL to the local engine. */ -export type ObservationWriteResolution = ObservationWritePeerTarget | ObservationWriteDenied; + * /amicode/*, a local-owned session, or an owner that is not a serving + * peer. The write falls through BYTE-IDENTICAL to the local engine. */ +export type ObservationWriteResolution = ObservationWritePeerTarget | ObservationWriteDenied | ObservationUnknownOwner; /** Options for the ObservationWriteRouter. */ export interface ObservationWriteRouterOpts { @@ -425,7 +619,7 @@ export class ObservationWriteRouter { * 0. GET/HEAD → local (the read plane owns reads — writes NEVER touch GET) * 1. /amicode/* (the machine's own surface) → local * 2. no session id extractable from the path → local - * 3. owner unknown (unowned) → local + * 3. owner unknown (unowned) → #1774 the unknown-owner outcome (local first) * 4. owner == localMachineId (local-owned) → local * 5. owner is not a serving peer → local * 6. owner is a serving peer → AUTHORIZE via the pure write gate: @@ -439,7 +633,7 @@ export class ObservationWriteRouter { const sessionId = extractSessionIdFromReadPath(pathname); // (2) if (!sessionId) return undefined; const owner = this.ownerMap.resolveOwner(sessionId); // (3) - if (!owner) return undefined; + if (!owner) return { kind: "unknown-owner", sessionId }; if (owner === this.localMachineId) return undefined; // (4) const peer = this.peer(owner); // (5) if (!peer) return undefined; // owner is not a serving peer → local fall-through diff --git a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts index c5b1fa49f..5e6caa966 100644 --- a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts +++ b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts @@ -38,8 +38,8 @@ export { LOCAL_NAMESPACE, NS_SEP } from "./sse_composite_cursor"; * #1617 — `pause()` / `resume()` are OPTIONAL flow control the driver drives * when the shared downstream backpressures: pausing every live upstream while * the downstream is stalled is what keeps the bounded per-namespace buffers - * from overflow-dropping (the overflow is otherwise UNRECOVERABLE — the live - * global event route has no replay cursor). A source that cannot pause simply + * from overflow-dropping (an overflow drop is never replayed — the arm stays + * open — so it can only be announced, by the overflow gap). A source that cannot pause simply * omits them; the aggregator falls back to buffer-and-drain (today's behavior * for injected test sources) — additive, no existing caller breaks. */ export interface SseFrameSource { @@ -49,6 +49,11 @@ export interface SseFrameSource { pause?(): void; /** #1617 — resume pulling once the downstream has drained. */ resume?(): void; + /** #1771 — once `next()` has resolved null: did the upstream END cleanly or + * fail with an ERROR? The census's close reason (and, #1773, the reason a + * fleet-of-one listener is severed under); both still end the source the + * same way. Absent / undefined ⇒ a clean end. */ + endReason?(): "end" | "error" | undefined; } /** The downstream response, narrowed to what the relay needs. In production an @@ -161,6 +166,49 @@ export function frameRawId(rawFrame: string): string | undefined { return id === undefined || id === "" ? undefined : id; } +/** #1773 — does the frame carry a `data` field (an event the client dispatches)? + * This is what "delivered" means for the reconnect gap: ANY data frame, with or + * without an `id:` line. A comment-only frame (`: keep-alive`) is not data. */ +export function isDataFrame(rawFrame: string): boolean { + for (const line of rawFrame.split(/\r?\n/)) { + if (line === "data" || line.startsWith("data:")) return true; + } + return false; +} + +/** #1777 — the engine's per-connection control frames on `/global/event`. They + * carry the connection's position as their id but consume no sequence (#1776). */ +export type EngineControlType = "server.connected" | "server.heartbeat" | "server.cursor.unservable"; + +/** The engine writes a control frame as a payload-only event whose payload + * starts `{"id":…,"type":…}` (handlers/global.ts `controlEvent`). A bus event + * serializes `{"directory":…` first, and quoted text inside any event is + * JSON-escaped, so this anchored match never mistakes one for a control frame. */ +const ENGINE_CONTROL = /^data: ?\{"payload":\{"id":"[^"\\]*","type":"(server\.(?:connected|heartbeat|cursor\.unservable))"/m; +/** Control frames are tiny; a big frame is never one (and is never scanned). */ +const ENGINE_CONTROL_MAX_BYTES = 2048; + +/** #1777 — which engine control frame this is, or undefined for any other frame + * (a bus event, a comment, a relay's own control frame). */ +export function engineControlType(rawFrame: string): EngineControlType | undefined { + if (rawFrame.length > ENGINE_CONTROL_MAX_BYTES) return undefined; + return ENGINE_CONTROL.exec(rawFrame)?.[1] as EngineControlType | undefined; +} + +/** #1777 — the engine's stream position `:` (#1776), or undefined + * for any other id (an older relay's, a composite). */ +export function parseEngineId(id: string | undefined): { epoch: string; seq: number } | undefined { + if (id === undefined) return undefined; + const m = /^([A-Za-z0-9_-]+):(\d+)$/.exec(id.trim()); + return m ? { epoch: m[1], seq: Number(m[2]) } : undefined; +} + +/** The id-less, payload-less gap frame the client resyncs on (#1617 shape). */ +function gapFrame(namespaces: string[]): string { + const data = JSON.stringify({ type: "amicode.sync.gap", namespaces }); + return `event: amicode.sync.gap\ndata: ${data}\n\n`; +} + /** Rebuild a frame with its `id:` line set to `newId` (or removed when null), * preserving every other line (`event:`/`data:`/`retry:`/comments) verbatim. * The single canonical `id:` line is placed just before the trailing blank so @@ -210,6 +258,58 @@ export async function pipeToResponse( // ── the aggregator core ────────────────────────────────────────────────────── +/** #1771 — one arm's traffic counters on this connection (the census reads + * them; nothing about delivery depends on them). Counted per INGESTED frame — + * whether it is delivered, buffered, or dropped. */ +export interface ArmTraffic { + frames: number; + /** UTF-8 bytes of the upstream frames as received. */ + bytes: number; + largestFrame: number; + /** Clock time of the last frame (the aggregator's `now`); undefined if none. */ + lastFrameAt?: number; + /** Frames dropped on overflow, cumulative (never zeroed by a gap). */ + drops: number; + /** Gap frames emitted that named this arm. */ + gaps: number; +} + +/** #1771 — why a gap frame was emitted: an arm re-opened after delivering + * (#1638), frames were dropped on overflow (#1617), or (#1777) the upstream + * answered a resume with a reset — its replay could not serve the cursor. */ +export type GapCause = "reopen" | "overflow" | "reset"; + +/** #1773 — what a re-open of an arm must do (`reopenArm`'s verdict): + * - `open` — the arm never delivered a data frame (an initial open): nothing + * was lost, open it silently; + * - `resume` — #1777: the arm delivered engine-sequenced frames, so open it + * FROM ITS CURSOR (`cursorFor`) — the upstream replays what was + * missed. Nothing is written; whatever the replay cannot serve is + * announced when the upstream answers (a reset → one gap); + * - `gap` — composite mode, an upstream with no ids to resume from (an older + * engine): the gap frame naming it was written; open it; + * - `sever` — fleet-of-one, same upstream: a synthetic frame is not allowed in + * the verbatim stream (#1264), so the caller must END THE LISTENER + * WITH A TRANSPORT ERROR instead of re-opening — the client + * registers the disconnect and resyncs. Nothing was written. */ +export type ReopenVerdict = "open" | "resume" | "gap" | "sever"; + +/** #1777 — what the caller of `ingest` must do with the arm: carry on, or (a + * fleet-of-one resume the upstream could not honor — it answered without ids) + * sever the listener, since the verbatim stream may not carry a gap frame. */ +export type IngestVerdict = "ok" | "sever"; + +/** #1771 — the aggregator's transitions, for the census. Observation only: + * callbacks run after the aggregator decided, and a throwing observer is + * swallowed. */ +export interface SseFanInObserver { + /** The FIRST overflow drop of an episode for `namespace` (an episode ends + * when the overflow gap naming it is emitted). `pending` is its live depth. */ + onOverflowDrop?(namespace: string, pending: number): void; + /** A gap frame was emitted naming `namespaces`. */ + onGap?(namespaces: string[], cause: GapCause): void; +} + export interface SseFanInOptions { /** The one downstream `/event` response every arm fans into. */ sink: SseSink; @@ -226,6 +326,10 @@ export interface SseFanInOptions { * buffers cannot overflow-drop. Absent ⇒ buffer-and-drain only (today's * behavior for injected test sources). */ onBackpressure?: () => void; + /** #1771 — the clock stamping each arm's last-frame time. Default Date.now. */ + now?: () => number; + /** #1771 — the census's view of drops and gaps. Absent ⇒ unobserved. */ + observer?: SseFanInObserver; } export class SseFanInAggregator { @@ -233,6 +337,10 @@ export class SseFanInAggregator { private readonly bufferBound: number; private readonly focusProvider?: () => FocusSnapshot | undefined; private readonly onBackpressure?: () => void; + private readonly now: () => number; + private readonly observer?: SseFanInObserver; + /** #1771 — per-arm traffic counters (census only). */ + private readonly trafficByArm = new Map(); /** namespace → last delivered upstream id (the composite cursor state). */ private readonly composite = new Map(); @@ -261,8 +369,22 @@ export class SseFanInAggregator { /** #1638 — namespaces whose arm has EVER delivered a frame this connection. * A re-open of an arm not in this set is the INITIAL open (no gap); a re-open * of one already here is a NON-INITIAL re-open (the #1601 path → gap in - * composite mode). */ + * composite mode, a severed listener in fleet-of-one). #1773: ANY data frame + * counts (an older engine sends no `id:` lines at all), counted as the arm + * hands it over — whether it is then delivered, buffered, or dropped. */ private readonly everDelivered = new Set(); + /** #1777 — namespace → the engine id its arm RESUMES from: the last id'd frame + * this connection ACCEPTED from it (written, or queued behind backpressure — + * never one dropped on overflow), so a re-open's replay starts right after + * everything already on its way to the listener and never re-sends it. An + * upstream reset re-bases it in the new epoch. */ + private readonly resumeAt = new Map(); + /** #1777 — arms re-opened FROM A CURSOR whose upstream has not answered yet. + * The first data frame decides: the engine's `server.connected` confirms the + * replay (and is the upstream's handshake, not an event — it is not + * forwarded in composite mode); a reset frame becomes one gap; an id-less + * frame means an upstream that cannot resume at all (the #1773 fallback). */ + private readonly resuming = new Set(); /** #1617 (fleet-of-one) — frames stranded on the zero-peer VERBATIM path while * the downstream is backpressured. Kept SEPARATE from the composite `buffers` * because these flush byte-for-byte (raw `sink.write`, no composite-id rewrite) @@ -275,6 +397,8 @@ export class SseFanInAggregator { this.bufferBound = opts.bufferBound ?? 256; this.focusProvider = opts.focusSnapshot; this.onBackpressure = opts.onBackpressure; + this.now = opts.now ?? Date.now; + this.observer = opts.observer; } /** Begin (or reconnect): parse the client's opaque composite cursor into the @@ -285,7 +409,11 @@ export class SseFanInAggregator { connect(cursor?: string): Map { const resume = parseCompositeCursor(cursor); this.composite.clear(); - for (const [k, v] of resume) this.composite.set(k, v); + this.resumeAt.clear(); + for (const [k, v] of resume) { + this.composite.set(k, v); + this.resumeAt.set(k, v); + } if (this.focusProvider) this.emitFocusFrame(); return new Map(resume); } @@ -332,18 +460,57 @@ export class SseFanInAggregator { } /** Ingest one raw SSE frame from an arm. `namespace === "local"` for the local - * arm; a machineId for a peer arm. */ - ingest(namespace: string, rawFrame: string): void { + * arm; a machineId for a peer arm. #1777: returns `sever` when the listener + * must be ended instead (see `IngestVerdict`); `ok` otherwise. */ + ingest(namespace: string, rawFrame: string): IngestVerdict { + this.countFrame(namespace, rawFrame); // #1771 census — observation only + const data = isDataFrame(rawFrame); + if (data) this.everDelivered.add(namespace); // #1773 — any data frame const rawId = frameRawId(rawFrame); const fleetOfOne = this.peerArms.size === 0; + const verbatim = namespace === LOCAL_NAMESPACE && fleetOfOne; + const control = data ? engineControlType(rawFrame) : undefined; + + // #1777 — the FIRST data frame of an arm re-opened from its cursor says + // whether the upstream replayed. An id-less one: an engine that cannot + // resume (older than #1776) ignored the cursor, so the stretch since the + // drop is lost — the #1773 fallback announces it (a gap; in fleet-of-one, + // where no synthetic frame may enter the verbatim stream, a sever). The + // engine's `server.connected` confirms the replay; in composite mode it is + // the upstream connection's handshake, not an event of this listener's + // stream, so it is not forwarded (the client would refetch on it). + if (data && this.resuming.delete(namespace)) { + if (rawId === undefined) { + if (verbatim) return "sever"; + this.writeControl(gapFrame([namespace])); + this.noteGap([namespace], "reopen"); + } else if (control === "server.connected" && !verbatim) { + this.advanceResume(namespace, rawId); + return "ok"; + } + } + + // #1777 — an upstream RESET: the engine could not serve the cursor the arm + // asked for (its process restarted, or the replay window moved past it). In + // composite mode that is exactly one gap naming this namespace — the reset + // frame itself is not forwarded (one signal, not two) — and the arm's resume + // point re-bases in the new epoch. In fleet-of-one the reset frame passes + // verbatim below: it IS the engine's own signal, and the client resyncs on + // it (#1264 — nothing synthetic enters the solo stream). + if (control === "server.cursor.unservable" && !verbatim) { + if (rawId !== undefined) this.advanceResume(namespace, rawId); + this.writeControl(gapFrame([namespace])); + this.noteGap([namespace], "reset"); + return "ok"; + } - if (namespace === LOCAL_NAMESPACE && fleetOfOne) { + if (verbatim) { // §D4 / AC1 byte-identity: with no peer arms the local frame is written // VERBATIM (bare id, every line untouched). The composite still tracks the // local position silently, so a peer that joins mid-stream inherits it. if (rawId !== undefined) { this.composite.set(LOCAL_NAMESPACE, rawId); - this.everDelivered.add(LOCAL_NAMESPACE); + this.advanceResume(LOCAL_NAMESPACE, rawId); } // #1617 (fleet-of-one) — honor backpressure on the verbatim path too. When // the downstream is already backpressured, hold the frame VERBATIM in the @@ -354,7 +521,7 @@ export class SseFanInAggregator { // changes only TIMING — not a byte — so the #1264 guard holds. if (!this.flowing) { this.verbatimBuffer.push(rawFrame); - return; + return "ok"; } const accepted = this.sink.write(rawFrame); this.sink.flush?.(); @@ -362,7 +529,7 @@ export class SseFanInAggregator { this.flowing = false; this.onBackpressure?.(); } - return; + return "ok"; } // Composite mode (≥1 peer arm active): every arm's frames carry the full @@ -375,6 +542,7 @@ export class SseFanInAggregator { } else { this.enqueue(namespace, rawFrame); } + return "ok"; } /** Drain the ordered pending stream after the downstream drains (decision B / @@ -451,21 +619,35 @@ export class SseFanInAggregator { } } - /** #1638 — a peer/local arm is being RE-OPENED (the #1601 reconcile path). In - * composite mode (≥1 peer arm) a NON-INITIAL re-open forces a client refetch - * via exactly one id-less `amicode.sync.gap` naming that namespace, because - * the upstream `/global/event` route does NOT replay on `?lastEventID` (Step 0 - * = no replay) so the cursor-advance alone cannot recover the reconnect gap. - * In FLEET-OF-ONE (zero peers) NO gap is emitted — a synthetic frame in the - * verbatim stream would violate the #1264 byte-identity guard; the solo-mode - * arm re-opens silently (its verbatim stream stays frame-for-frame identical). - * The INITIAL open of an arm (never delivered a frame) emits no gap either — - * only a re-open of an arm that has already delivered. */ - reopenArm(namespace: string): void { - if (this.peerArms.size === 0) return; // fleet-of-one → NO synthetic frame (#1264) - if (!this.everDelivered.has(namespace)) return; // initial open → no gap - const data = JSON.stringify({ type: "amicode.sync.gap", namespaces: [namespace] }); - this.writeControl(`event: amicode.sync.gap\ndata: ${data}\n\n`); + /** #1638 — a peer/local arm is being RE-OPENED (the #1601 reconcile path). + * + * #1777 — an arm that delivered engine-sequenced frames RESUMES: the caller + * re-opens it from `cursorFor(namespace)` and the upstream replays the frames + * the listener missed, so nothing is written and nothing is lost — no gap, no + * sever, in either mode. What the replay cannot serve is announced when the + * upstream answers (`ingest`: a reset → one gap; an id-less answer → the + * fallback below). + * + * #1773 — the FALLBACK, for an upstream with no ids to resume from (an engine + * older than #1776): every loss is loud. "Non-initial" means the arm + * delivered ANY data frame (id'd or not). In composite mode the re-open + * forces a client refetch via exactly one id-less `amicode.sync.gap` naming + * the namespace. In FLEET-OF-ONE (zero peers) a synthetic frame in the + * verbatim stream would violate the #1264 byte-identity guard, so the verdict + * is `sever`: the caller ends the listener with a transport error, and the + * client's disconnect is the gap signal. Nothing is written for `sever`. The + * INITIAL open of an arm (never delivered a data frame) is `open` in both + * modes — nothing was lost. A gap never advances any cursor. */ + reopenArm(namespace: string): ReopenVerdict { + if (!this.everDelivered.has(namespace)) return "open"; // initial open → nothing lost + if (parseEngineId(this.resumeAt.get(namespace)) !== undefined) { + this.resuming.add(namespace); // the upstream's first answer confirms or refuses the replay + return "resume"; + } + if (this.peerArms.size === 0) return "sever"; // fleet-of-one → NO synthetic frame (#1264) + this.writeControl(gapFrame([namespace])); + this.noteGap([namespace], "reopen"); + return "gap"; } /** #1617 — the overflow defense-in-depth. When resume() observes a nonzero @@ -480,9 +662,9 @@ export class SseFanInAggregator { const dropped: string[] = []; for (const [ns, n] of this.droppedCount) if (n > 0) dropped.push(ns); if (dropped.length === 0) return; - const data = JSON.stringify({ type: "amicode.sync.gap", namespaces: dropped }); - this.writeControl(`event: amicode.sync.gap\ndata: ${data}\n\n`); + this.writeControl(gapFrame(dropped)); for (const ns of dropped) this.droppedCount.set(ns, 0); + this.noteGap(dropped, "overflow"); } /** The number of frames DROPPED for a namespace on buffer overflow (decision @@ -496,18 +678,72 @@ export class SseFanInAggregator { return formatCompositeCursor(this.composite); } - /** #1638 — the live per-namespace last-DELIVERED id (the composite map's - * entry for `namespace`), or undefined if the arm has delivered no id'd frame - * yet. The driver reads this at re-open time to resume the arm from its - * last-delivered position rather than the stale connect-time cursor — honoring - * #1617's own cursor invariant. Advances per delivered frame on BOTH the - * composite and the verbatim path; an id-less frame never advances it. */ + /** #1638 — the live per-namespace resume position the driver re-opens an arm + * from (rather than the stale connect-time cursor — honoring #1617's own + * cursor invariant), or undefined if the arm has delivered no id'd frame and + * the client's cursor named none. Advances per frame on BOTH the composite + * and the verbatim path; an id-less frame never advances it. + * + * #1777 — it holds the engine's own `:` and counts a frame queued + * behind backpressure as delivered (it is on its way to the listener), so the + * upstream's replay after a re-open starts right after it and never re-sends + * a frame the listener will already get; a frame dropped on overflow does + * not count (that stretch is the overflow gap's, #1617). */ cursorFor(namespace: string): string | undefined { - return this.composite.get(namespace); + return this.resumeAt.get(namespace); + } + + /** #1771 — one arm's traffic counters on this connection (a copy; zeros for + * an arm that has ingested nothing). */ + traffic(namespace: string): ArmTraffic { + const t = this.trafficByArm.get(namespace); + return t ? { ...t } : { frames: 0, bytes: 0, largestFrame: 0, drops: 0, gaps: 0 }; + } + + /** #1771 — how many of this arm's frames wait undelivered right now: its data + * entries on the ordered pending stream, plus (local) the fleet-of-one + * verbatim buffer. */ + pendingDepth(namespace: string): number { + let n = 0; + for (const entry of this.pending) if (entry.kind === "data" && entry.namespace === namespace) n++; + if (namespace === LOCAL_NAMESPACE) n += this.verbatimBuffer.length; + return n; } // ── internals ──────────────────────────────────────────────────────────── + private trafficOf(namespace: string): ArmTraffic { + let t = this.trafficByArm.get(namespace); + if (!t) { + t = { frames: 0, bytes: 0, largestFrame: 0, drops: 0, gaps: 0 }; + this.trafficByArm.set(namespace, t); + } + return t; + } + + private countFrame(namespace: string, rawFrame: string): void { + const t = this.trafficOf(namespace); + const bytes = Buffer.byteLength(rawFrame, "utf8"); + t.frames += 1; + t.bytes += bytes; + if (bytes > t.largestFrame) t.largestFrame = bytes; + t.lastFrameAt = this.now(); + } + + private noteGap(namespaces: string[], cause: GapCause): void { + for (const ns of namespaces) this.trafficOf(ns).gaps += 1; + this.observe((o) => o.onGap?.([...namespaces], cause)); + } + + private observe(fn: (o: SseFanInObserver) => void): void { + if (!this.observer) return; + try { + fn(this.observer); + } catch { + /* the census must never break delivery */ + } + } + /** Write one frame downstream in composite mode: advance the namespace's * cursor from the frame's id, stamp the full composite as the wire `id:`, * write + flush. A `write()` returning false backpressures the shared @@ -516,7 +752,7 @@ export class SseFanInAggregator { const rawId = frameRawId(rawFrame); if (rawId !== undefined) { this.composite.set(namespace, rawId); - this.everDelivered.add(namespace); + this.advanceResume(namespace, rawId); } const wire = rewriteFrameId(rawFrame, formatCompositeCursor(this.composite)); const accepted = this.sink.write(wire); @@ -543,10 +779,26 @@ export class SseFanInAggregator { if (entry.kind === "data" && entry.namespace === namespace) live++; } if (live >= this.bufferBound) { - this.droppedCount.set(namespace, (this.droppedCount.get(namespace) ?? 0) + 1); + const before = this.droppedCount.get(namespace) ?? 0; + this.droppedCount.set(namespace, before + 1); + this.trafficOf(namespace).drops += 1; // #1771 census + if (before === 0) this.observe((o) => o.onOverflowDrop?.(namespace, live)); return; } this.pending.push({ kind: "data", namespace, frame: rawFrame }); + const rawId = frameRawId(rawFrame); + if (rawId !== undefined) this.advanceResume(namespace, rawId); // #1777 — queued = on its way + } + + /** #1777 — move a namespace's resume point to `rawId`: never backwards within + * one engine epoch (a resume point only ever follows what was accepted); a + * new epoch (the upstream restarted and reset) or an id that is not an + * engine's replaces it. */ + private advanceResume(namespace: string, rawId: string): void { + const next = parseEngineId(rawId); + const prev = parseEngineId(this.resumeAt.get(namespace)); + if (next && prev && next.epoch === prev.epoch && next.seq < prev.seq) return; + this.resumeAt.set(namespace, rawId); } private emitFocusFrame(): void { diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index 2ef85910f..52f851853 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -5,8 +5,11 @@ // #1511 built the synchronous frame-routing CORE (sse_fanin_aggregator.ts) and // unit-tested all 7 ACs against a deterministic sink. What was deliberately NOT // wired was the driver that opens the REAL upstream SSE connections and pumps -// their frames onto the ONE downstream `res` the app reads (ADR 0027 single- -// origin). This module is that driver. +// their frames onto the downstream `res` the app reads. This module is that +// driver. #1772: ADR 0027's single-origin assumption (the app opens exactly one +// `/event`) no longer holds — multiple windows, Chat Deck panes and split panes +// each open their own — so every listener gets its own connection (aggregator, +// arms, reconcile) up to a cap; none is ever evicted to make room for another. // // On the observation path (#1543/#1565), `handle()` ALWAYS accepts (returns // true) — zero non-local owners starts in fleet-of-one mode where the @@ -30,6 +33,7 @@ import { } from "./sse_fanin_aggregator"; import type { SessionOwnerMap } from "./session_multiplexer"; import type { PeerTokenRead } from "./fleet_peer_store"; +import { FanInCensus, emptyCloses, type FanInCloseReason, type CensusConnection } from "./fanin_census"; /** The one thing dispatch needs from the driver (server.ts imports this type and * exposes it on FleetPlane.eventFanIn). `handle` always returns true on the @@ -77,26 +81,102 @@ export interface SseFanInDriverDeps { /** The live reconcile cadence per connection (ms). Default 1000. Tests set a * large value and call `reconcile()` directly for determinism. */ reconcileMs?: number; + /** #1772 — the cap on CONCURRENT listeners (one FanInConnection each). A + * listener beyond it is refused with a retryable 503 + Retry-After; no live + * listener is ever closed to make room. Default DEFAULT_FANIN_MAX_LISTENERS. */ + maxListeners?: number; + /** #1771 — this service's census (counters + log lines). The caller owns its + * cadence (`start()`) and its sink. Absent ⇒ a silent census (counted, + * logged nowhere). */ + census?: FanInCensus; } +/** #1772 — default concurrent-listener cap per fan-in service. A service rarely + * has more than a handful of views (windows, Chat Deck panes, split panes). */ +export const DEFAULT_FANIN_MAX_LISTENERS = 16; +/** #1772 — the Retry-After hint (seconds) on a beyond-cap refusal. */ +export const FANIN_LISTENER_CAP_RETRY_AFTER_S = 5; +/** #1772 — the named error on a beyond-cap refusal. */ +export const FANIN_LISTENER_CAP_ERROR = "fan-in listener cap reached"; + +/** The relay reader's frame splitter: decoded upstream text in, WHOLE frames out + * (delimiter-inclusive, verbatim). #1779: LINEAR in the text received. The old + * reader appended every chunk to the whole partial buffer and searched it from + * the start, so a multi-MB frame arriving in 64 KB chunks cost quadratic time. + * Now only the NEW chunk is scanned; a delimiter split across two chunks is + * caught by looking back into the partial frame (at most two characters), + * which is held as pieces and joined ONCE, when it completes. + * + * A frame ends at a blank line — a line feed whose line is empty: `\n\n`, or + * with CRLF line endings `\r\n\r\n` (any LF/CRLF mix: `\n\r\n`, `\r\n\n`). The + * frame is relayed exactly as received, CRs and all — never normalized (#1264 + * byte identity); the aggregator reads either line ending. */ +export class SseFrameSplitter { + /** The partial frame, as received (never re-copied while it grows). */ + private parts: string[] = []; + /** The partial frame's last ≤2 characters — the lookback for a split delimiter. */ + private tail = ""; + + /** Feed one decoded chunk; returns the frames it completed, in order. */ + push(chunk: string): string[] { + const frames: string[] = []; + if (chunk === "") return frames; + const tail = this.tail; + let start = 0; // where the partial frame resumes in this chunk + let floor = -tail.length; // the current frame's first position (negative = in the tail) + /** The character at chunk position `i` (negative reaches into the tail), or + * NaN before the current frame — a lookback never crosses a frame boundary. */ + const at = (i: number): number => + i < floor ? NaN : i >= 0 ? chunk.charCodeAt(i) : tail.charCodeAt(tail.length + i); + let from = 0; + let lf: number; + while ((lf = chunk.indexOf("\n", from)) !== -1) { + from = lf + 1; + // the line this line feed ends is empty: it follows another line ending + const prev = at(lf - 1); + if (prev !== LF_CODE && !(prev === CR_CODE && at(lf - 2) === LF_CODE)) continue; + this.parts.push(chunk.slice(start, from)); + frames.push(this.parts.length === 1 ? this.parts[0] : this.parts.join("")); // whole frame, delimiter-inclusive + this.parts = []; + start = floor = from; + } + if (start < chunk.length) { + const rest = start === 0 ? chunk : chunk.slice(start); + this.parts.push(rest); + this.tail = rest.length >= 2 ? rest.slice(-2) : (start === 0 ? tail + rest : rest).slice(-2); + } else { + this.tail = ""; // a frame ended exactly at the chunk's end + } + return frames; + } +} +const LF_CODE = 10; +const CR_CODE = 13; + /** The production upstream reader: a frame-preserving http SSE source. Splits the * byte stream on the SSE blank-line delimiter into WHOLE frames (delimiter- * inclusive) so `event:`/`data:`/`retry:`/`id:` lines survive intact for the * aggregator's §D2 relay. Never throws — a connection error just ends the - * source (the aggregator names the gap; D3 replays it on reconnect). */ + * source (the aggregator names the gap; D3 replays it on reconnect). #1771: + * the source remembers WHICH ended it — a clean upstream end or an error — for + * the census's close reason; both still end it identically. #1773: a non-200 + * answer (an auth failure, an overloaded peer, a beyond-cap 503) is an ERROR — + * a loss, not a clean end — and its body is never parsed as frames. */ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { const queue: string[] = []; const waiters: Array<(v: string | null) => void> = []; let ended = false; + let reason: "end" | "error" | undefined; let clientReq: http.ClientRequest | undefined; const pushFrame = (f: string): void => { const w = waiters.shift(); if (w) w(f); else queue.push(f); }; - const end = (): void => { + const end = (why: "end" | "error"): void => { if (ended) return; ended = true; + reason = why; let w: ((v: string | null) => void) | undefined; while ((w = waiters.shift())) w(null); }; @@ -106,23 +186,26 @@ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { const headers: Record = {}; if (r.authHeader) headers.authorization = r.authHeader; clientReq = http.get(u, { headers }, (res) => { - let buf = ""; + if (res.statusCode !== 200) { + // #1773 — not a stream: an upstream ERROR. Discard the body (never read + // it as SSE frames) and let the socket go. + res.on("error", () => undefined); // a reset mid-drain must not throw + res.resume(); + end("error"); + return; + } + const splitter = new SseFrameSplitter(); res.setEncoding("utf8"); res.on("data", (chunk: string) => { if (ended) return; - buf += chunk; - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - pushFrame(buf.slice(0, idx + 2)); // whole frame, delimiter-inclusive - buf = buf.slice(idx + 2); - } + for (const frame of splitter.push(chunk)) pushFrame(frame); }); - res.on("end", () => end()); - res.on("error", () => end()); + res.on("end", () => end("end")); + res.on("error", () => end("error")); }); - clientReq.on("error", () => end()); + clientReq.on("error", () => end("error")); } catch { - end(); + end("error"); } return { next(): Promise { @@ -133,14 +216,21 @@ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { }, close(): void { clientReq?.destroy(); - end(); + end("end"); }, + endReason: () => reason, }; } -/** One live `/event` fan-in connection: the aggregator on this response, the - * open upstream sources keyed by namespace, and the reconcile timer. */ -class FanInConnection { +/** One live `/event` fan-in connection — ONE PER LISTENER (#1772): the + * aggregator on this response, the open upstream sources keyed by namespace, + * and this connection's own reconcile timer. Closing it closes only its arms. + * #1771: it is also the census's view of this listener — per-arm opens and + * closes-by-reason live here, traffic/pending/drops/gaps in the aggregator. + * #1773: it never ends its listener silently — every close except a graceful + * service shutdown is an ERROR close (the listener sees a transport error and + * resyncs), and a fleet-of-one local-arm loss severs the listener that way. */ +class FanInConnection implements CensusConnection { private readonly agg: SseFanInAggregator; private readonly sources = new Map(); private resume = new Map(); @@ -148,11 +238,23 @@ class FanInConnection { private readonly authHeader?: string; private timer?: ReturnType; private closed = false; + /** #1771 — per arm: how often it opened, and why it closed. */ + private readonly armLife = new Map }>(); + /** #1773 — per arm: the reason of its most recent close (a severed + * listener's close is named after the loss that caused it). */ + private readonly lastClose = new Map(); + /** #1771 — sources THIS connection closed (their reason is already recorded; + * the pump's exit must not re-count them as an upstream end). */ + private readonly closedByUs = new WeakSet(); constructor( + readonly id: string, private readonly deps: SseFanInDriverDeps, + private readonly census: FanInCensus, private readonly req: http.IncomingMessage, private readonly res: http.ServerResponse, + /** Invoked exactly once, when this connection closes (for any reason). */ + private readonly onClosed: (reason: FanInCloseReason) => void, ) { this.open = deps.openUpstream ?? openHttpSseSource; const a = req.headers.authorization; @@ -167,14 +269,61 @@ class FanInConnection { ...(deps.focusSnapshot ? { focusSnapshot: deps.focusSnapshot } : {}), // #1617 — re-derive the transparent proxy's backpressure by hand. When the // shared downstream `write()` returns false, pause EVERY live upstream so - // the bounded per-namespace buffers cannot overflow-drop (the overflow is - // otherwise unrecoverable — the live global event route has no replay - // cursor). The `drain` edge (wired in start()) resumes them and calls + // the bounded per-namespace buffers cannot overflow-drop (an overflow drop + // is not replayed — the arm stays open — so it can only be announced, by + // the overflow gap). The `drain` edge (wired in start()) resumes them and calls // agg.resume() to flush the buffers in order and re-arm `flowing`. onBackpressure: () => this.pauseUpstreams(), + // #1771 — the census clock and its view of drops and gaps. + now: census.now, + observer: { + onOverflowDrop: (ns, pending) => census.overflowDrop(this.id, ns, pending), + onGap: (nss, cause) => census.gapEmitted(this.id, nss, cause), + }, }); } + /** #1771 — this listener's arms as the census reports them (read at census + * time, never pushed per frame). */ + arms(): ReturnType { + return [...this.armLife.entries()].map(([arm, life]) => ({ + arm, + traffic: this.agg.traffic(arm), + pending: this.agg.pendingDepth(arm), + opens: life.opens, + closes: { ...life.closes }, + })); + } + + private lifeOf(namespace: string): { opens: number; closes: Record } { + let life = this.armLife.get(namespace); + if (!life) { + life = { opens: 0, closes: emptyCloses() }; + this.armLife.set(namespace, life); + } + return life; + } + + /** #1771 — record (and log) an arm's close, once. */ + private noteArmClosed(namespace: string, reason: FanInCloseReason): void { + this.lifeOf(namespace).closes[reason] += 1; + this.lastClose.set(namespace, reason); + this.census.armClosed(this.id, namespace, reason); + } + + /** Close an upstream THIS connection decided to close (membership change, + * listener closed, shutdown) — recorded with its reason before the source's + * own end reaches the pump. */ + private closeSource(namespace: string, source: SseFrameSource, reason: FanInCloseReason): void { + this.closedByUs.add(source); + this.noteArmClosed(namespace, reason); + try { + source.close(); + } catch { + /* already gone */ + } + } + /** #1617 — pause every live upstream source that supports it. A source * without pause()/resume() (an injected test source, or a transport that * cannot pause) simply keeps flowing into the aggregator's bounded buffer. */ @@ -261,11 +410,12 @@ class FanInConnection { // Open a real upstream for each newly-active peer arm. for (const machineId of active) { if (this.sources.has(machineId)) continue; - // #1638 — resume from the aggregator's LIVE last-delivered id for this - // namespace (its composite cursor), NOT the stale connect-time id. A - // re-open of an arm that already delivered is a NON-INITIAL re-open — the - // aggregator emits one composite-mode gap (reopenArm is a no-op for a - // never-delivered arm and in fleet-of-one, so calling it here is safe). + // #1638 — resume from the aggregator's LIVE resume position for this + // namespace, NOT the stale connect-time id. #1777: a re-open of an arm + // that delivered engine ids RESUMES there and the peer replays; one with + // no ids (an older engine) gets the composite-mode gap instead + // (reopenArm is a no-op for a never-delivered arm; a peer arm is only + // active in composite mode, so its verdict is never `sever`). this.reopenAndTrack(machineId, this.deps.peerBaseUrl(machineId), this.peerAuth(machineId)); } // Close the real upstream for any peer arm no longer active (dark peer, or @@ -273,11 +423,7 @@ class FanInConnection { for (const machineId of [...this.sources.keys()]) { if (machineId === LOCAL_NAMESPACE) continue; if (!active.has(machineId)) { - try { - this.sources.get(machineId)?.close(); - } catch { - /* already gone */ - } + this.closeSource(machineId, this.sources.get(machineId)!, "membership-change"); this.sources.delete(machineId); } } @@ -291,21 +437,34 @@ class FanInConnection { // until reload"). Guard at the call site so a HEALTHY local arm is never // re-opened or duplicated (openArm early-returns on a present source too); // resume from LOCAL's live cursor for lossless replay — mirrors start(). + // #1773: in fleet-of-one a local arm that delivered is not re-opened at all — + // reopenAndTrack severs the listener (an error close) so the client resyncs. if (!this.sources.has(LOCAL_NAMESPACE)) { this.reopenAndTrack(LOCAL_NAMESPACE, this.deps.localEventUrl(), this.authHeader); } } - /** #1638 — re-open an arm from the aggregator's LIVE last-delivered cursor (not - * the stale connect-time id) and, on a NON-INITIAL re-open, force a client - * refetch via the aggregator's composite-mode gap. `agg.reopenArm` is a no-op - * for an arm that has never delivered (an initial open) and in fleet-of-one - * (where a synthetic frame would break the #1264 byte-identity guard), so it - * is safe to call at every reconcile-time open. Falls back to the connect-time - * resume id when the arm has delivered nothing yet. */ + /** #1638 — re-open an arm from the aggregator's LIVE resume position (not the + * stale connect-time id). `agg.reopenArm` is a no-op for an arm that has + * never delivered (an initial open), so it is safe to call at every + * reconcile-time open. Falls back to the connect-time resume id when the arm + * has delivered nothing yet. + * + * #1777 — an arm that delivered engine ids RESUMES: it re-opens from its last + * delivered id and the upstream replays the stretch it missed — no gap, no + * sever. An upstream reset is announced as the upstream answers. + * + * #1773 — the fallback for an upstream without ids: in composite mode one gap; + * in fleet-of-one the verdict is `sever` (a synthetic gap frame may not enter + * the verbatim stream, #1264), so instead of re-opening silently (losing the + * stretch unannounced) the listener is ended with a transport error, named + * after the loss that caused it. */ private reopenAndTrack(namespace: string, url: string | undefined, authHeader: string | undefined): void { if (this.sources.has(namespace)) return; - this.agg.reopenArm(namespace); + if (this.agg.reopenArm(namespace) === "sever") { + this.close(this.lastClose.get(namespace) ?? "upstream-end"); + return; + } const lastEventId = this.agg.cursorFor(namespace) ?? this.resume.get(namespace); this.openArm(namespace, url, authHeader, lastEventId); } @@ -334,6 +493,9 @@ class FanInConnection { ...(lastEventId ? { lastEventId } : {}), }); this.sources.set(namespace, source); + const life = this.lifeOf(namespace); + life.opens += 1; + this.census.armOpened(this.id, namespace, life.opens); void this.pump(namespace, source); } @@ -342,70 +504,143 @@ class FanInConnection { * the client tracks — NOT the bare `pipeToResponse` relay, which would emit * the NS_SEP form the composite cursor cannot round-trip). */ private async pump(namespace: string, source: SseFrameSource): Promise { + let threw = false; for (;;) { let frame: string | null; try { frame = await source.next(); } catch { + threw = true; break; } if (frame === null || this.closed) break; - this.agg.ingest(namespace, frame); + // #1777 — a fleet-of-one arm resumed from its cursor whose upstream + // answered without ids (an engine that cannot replay) lost the stretch + // since the drop; the verbatim stream may carry no gap frame, so the + // listener is severed under the loss that caused the re-open (#1773). + if (this.agg.ingest(namespace, frame) === "sever") { + this.close(this.lastClose.get(namespace) ?? "upstream-end"); + return; + } + } + // #1771 — the upstream's OWN end (not one this connection caused): name it. + if (!this.closed && !this.closedByUs.has(source)) { + this.noteArmClosed(namespace, threw || source.endReason?.() === "error" ? "upstream-error" : "upstream-end"); } // The upstream ended or errored — remove the dead source so the next // reconcile() tick can re-open the arm (#1566). this.sources.delete(namespace); } - close(): void { + /** Close this listener's connection: stop its reconcile, close its arms, and + * end its response. #1773 — only a graceful service SHUTDOWN ends the stream + * cleanly; every other reason ERRORS it (`res.destroy(err)`), so the listener + * — or the relay in front of it — sees a transport error, never a clean end + * that reads as "nothing was lost". Idempotent: the first reason wins. */ + close(reason: FanInCloseReason = "listener-closed"): void { if (this.closed) return; this.closed = true; if (this.timer) { clearInterval(this.timer); this.timer = undefined; } - for (const s of this.sources.values()) { - try { - s.close(); - } catch { - /* already gone */ - } - } + for (const [namespace, s] of this.sources) this.closeSource(namespace, s, reason); this.sources.clear(); try { - this.res.end(); + if (reason === "shutdown") this.res.end(); + else this.res.destroy(new Error(`amicode fan-in: listener stream closed (${reason}); resync required`)); } catch { /* already torn down */ } + this.onClosed(reason); + } +} + +/** #1772 — the explicit, retryable beyond-cap refusal: 503 + Retry-After and a + * named JSON body (the `{ ok: false, error, reason }` shape of the service's + * other 503s). The app's stream loop treats any non-200 as a disconnect and + * retries; nothing about this response is an eviction of anyone else. */ +function refuseBeyondCap(res: http.ServerResponse, cap: number): void { + try { + res.writeHead(503, { + "content-type": "application/json", + "cache-control": "no-cache", + "retry-after": String(FANIN_LISTENER_CAP_RETRY_AFTER_S), + }); + res.end( + JSON.stringify({ + ok: false, + error: FANIN_LISTENER_CAP_ERROR, + reason: "listener-cap", + max_listeners: cap, + retry_after_s: FANIN_LISTENER_CAP_RETRY_AFTER_S, + }), + ); + } catch { + /* the refused socket is already gone — nothing to answer */ } } -/** The `/event` fan-in driver (ADR 0033 §D1). One live connection at a time - * (ADR 0027 single-origin: the app opens exactly one `/event`); a second - * `handle` tears the prior connection down first. */ +/** The `/event` fan-in driver (ADR 0033 §D1). #1772 — ONE FanInConnection PER + * LISTENER: the single-origin assumption (ADR 0027: the app opens exactly one + * `/event`) no longer holds with multiple windows, Chat Deck panes and split + * panes onto the same hub. Evicting the prior listener made two views re-open + * against each other once per second, indefinitely, silently dropping whatever + * the evicted view would have received. Each listener now owns its aggregator, + * upstream arms and reconcile timer; the driver only tracks the live set, + * bounded by `maxListeners` (refusal beyond it, never an eviction). */ export class SseFanInDriver implements EventFanInDriver { - private active?: FanInConnection; + private readonly connections = new Set(); + /** #1771 — this service's census (a silent one when none was injected). */ + readonly census: FanInCensus; + private nextConnection = 0; + private shuttingDown = false; - constructor(private readonly deps: SseFanInDriverDeps) {} + constructor(private readonly deps: SseFanInDriverDeps) { + this.census = deps.census ?? new FanInCensus(); + } handle(req: http.IncomingMessage, res: http.ServerResponse): boolean { + // #1772: beyond the cap the listener is REFUSED — answered here (so dispatch + // never falls through to an un-fanned engine stream) with a retryable 503 + // and a Retry-After hint. No existing listener is closed to make room. + const cap = this.deps.maxListeners ?? DEFAULT_FANIN_MAX_LISTENERS; + if (this.connections.size >= cap) { + refuseBeyondCap(res, cap); + this.census.listenerRefused(cap); + return true; + } // #1565: always accept on the observation path. Zero non-local owners starts // in fleet-of-one mode (the aggregator's §D4 byte-identity relay); reconcile - // opens peer arms as the OwnerMapFeed discovers them. - this.active?.close(); - const conn = new FanInConnection(this.deps, req, res); - this.active = conn; - conn.start(); - res.on("close", () => { - if (this.active === conn) this.active = undefined; - conn.close(); + // opens peer arms as the OwnerMapFeed discovers them. #1773: a connection + // can also close ITSELF (a fleet-of-one local-arm loss severs the listener); + // either way it leaves the live set and the census once, under its reason. + const conn: FanInConnection = new FanInConnection(`c${++this.nextConnection}`, this.deps, this.census, req, res, (reason) => { + this.connections.delete(conn); + this.census.listenerClosed(conn.id, reason); }); + this.connections.add(conn); + this.census.listenerAccepted(conn); + conn.start(); + // A departing listener closes ONLY its own connection (and so only its own + // upstream arms); every other listener keeps streaming. #1771: a close the + // service's own teardown caused is attributed to `shutdown`. + res.on("close", () => conn.close(this.shuttingDown ? "shutdown" : "listener-closed")); return true; } - /** Reconcile the live connection's membership NOW (the per-connection timer - * target; also called directly by the arm-lifecycle tests for determinism). */ + /** Reconcile EVERY live connection's membership NOW (each connection's own + * timer is the cadence; also called directly by the arm-lifecycle tests for + * determinism). */ reconcile(): void { - this.active?.reconcile(); + for (const conn of this.connections) conn.reconcile(); + } + + /** #1771 — the service is stopping: the listener and arm closes its socket + * teardown triggers next are attributed `shutdown` (not `listener-closed`), + * and the census cadence stops. Closes nothing itself — delivery unchanged. */ + shutdown(): void { + this.shuttingDown = true; + this.census.stop(); } } diff --git a/packages/extension/src/amicode_service_runner.ts b/packages/extension/src/amicode_service_runner.ts index 5575a35fa..b3edfc416 100644 --- a/packages/extension/src/amicode_service_runner.ts +++ b/packages/extension/src/amicode_service_runner.ts @@ -555,6 +555,8 @@ export async function bootAmicodeServiceRunner(opts: AmicodeServiceRunnerOptions engine: { password: unarmed ? undefined : password, getUrl: () => engineUrl }, shelf: { distRoot: opts.appDistRoot }, ...(hubFleet ? { fleet: hubFleet } : {}), + // #1771: the fan-in census + owner-map refresh outcomes → the runner's log. + log, }); const servicePort = opts.servicePort ?? 4095; let url: URL; diff --git a/packages/extension/src/amicode_service_wiring.ts b/packages/extension/src/amicode_service_wiring.ts index 77193ee00..d60e86a90 100644 --- a/packages/extension/src/amicode_service_wiring.ts +++ b/packages/extension/src/amicode_service_wiring.ts @@ -331,6 +331,9 @@ export async function startAmicodeService( shelf: opts.appDistRoot !== undefined ? { distRoot: opts.appDistRoot } : undefined, fleet: fleet ?? opts.fleet, modelRouting: opts.modelRouting, + // #1771: the fan-in census + owner-map refresh outcomes → the Amicode + // output channel. + log: (line) => log.appendLine(line), }); let url: URL; if (opts.port && opts.port > 0) { diff --git a/packages/extension/src/extension.ts b/packages/extension/src/extension.ts index 01bc9a92c..2e2a664e8 100644 --- a/packages/extension/src/extension.ts +++ b/packages/extension/src/extension.ts @@ -2477,6 +2477,8 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { configuredBinary: vscode.workspace.getConfiguration("amicode").get("opencodeBinary", ""), configuredPort: vscode.workspace.getConfiguration("amicode").get("opencodePort", 0), plistContent, + // #1775: every per-peer SSH forward agent, enumerated by label convention. + launchAgentsDir: path.join(os.homedir(), "Library", "LaunchAgents"), }); for (const c of fleetChecks) { results.push({ name: c.name, ok: c.ok, detail: c.ok ? c.detail : `${c.detail} → ${c.fix ?? ""}`.trim() }); @@ -2878,6 +2880,8 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { configuredBinary: vscode.workspace.getConfiguration("amicode").get("opencodeBinary", ""), configuredPort: vscode.workspace.getConfiguration("amicode").get("opencodePort", 0), plistContent, + // #1775: per-peer forward agents (checkPeerForwards self-skips off darwin). + launchAgentsDir: path.join(os.homedir(), "Library", "LaunchAgents"), }); const failed = checks.filter((c) => !c.ok); if (failed.length === 0) return; diff --git a/packages/extension/src/fleet_health.ts b/packages/extension/src/fleet_health.ts index ca1d8d334..463d07246 100644 --- a/packages/extension/src/fleet_health.ts +++ b/packages/extension/src/fleet_health.ts @@ -20,6 +20,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { homedir } from "node:os"; import { readFleetTopology, type FleetTopologyState } from "./fleet_topology"; +import { listPeerBaseUrls } from "./amicode_service/fleet_peer_store"; export const FLEET_GUARD_REL = "tools/fleet/amico-opencode-fleet-guard"; export const FLEET_GUARD_INSTALL = path.join(homedir(), ".local", "bin", "amico-opencode-fleet-guard"); @@ -162,6 +163,155 @@ export function checkFleetTunnel( return { name: "Fleet tunnel", ok: true, detail: `ServerAlive 15/2 + TCPKeepAlive, ${port} forward` }; } +// ── #1775: per-peer SSH forward agents ───────────────────────────────────────── +// The installer generates one launchd agent per attached peer (tools/fleet/ +// co.harmoniqs.amico-peer-forward.plist); this check enumerates every forward +// agent in the LaunchAgents dir BY LABEL CONVENTION and holds each to the hub +// tunnel's keepalive standard. The attached peers come from the reader +// peer-token store: a loopback base_url's port IS the forward's local end. + +/** The managed per-peer forward label — deterministic from the peer's machine id. */ +export function peerForwardLabel(machineId: string): string { + return `co.harmoniqs.amico-peer-${machineId}-forward`; +} + +/** Any `co.harmoniqs.amico--forward` agent file — managed or hand-installed. */ +const FORWARD_AGENT_FILE = /^co\.harmoniqs\.amico-(.+)-forward\.plist$/; +/** The managed shape inside it: `co.harmoniqs.amico-peer--forward`. */ +const MANAGED_FORWARD_FILE = /^co\.harmoniqs\.amico-peer-(.+)-forward\.plist$/; + +/** A peer's ssh-forwarded local port: only a loopback http base URL with an + * explicit port rides an SSH forward (https = tailscale/direct reach → none). */ +function loopbackForwardPort(baseUrl: string): number | null { + let u: URL; + try { + u = new URL(baseUrl); + } catch { + return null; + } + if (u.protocol !== "http:" || (u.hostname !== "127.0.0.1" && u.hostname !== "localhost")) return null; + const port = Number(u.port); + return Number.isInteger(port) && port >= 1 && port <= 65535 ? port : null; +} + +/** The keepalive standard (fleet_transport sshForwardArgs), value-exact. */ +function forwardKeepaliveIssues(content: string): string[] { + const issues: string[] = []; + const interval = /ServerAliveInterval=(\d+)/.exec(content)?.[1]; + if (interval === undefined) issues.push("ServerAliveInterval 15 missing"); + else if (interval !== "15") issues.push(`ServerAliveInterval is ${interval} (stale, should be 15)`); + const count = /ServerAliveCountMax=(\d+)/.exec(content)?.[1]; + if (count === undefined) issues.push("ServerAliveCountMax 2 missing"); + else if (count !== "2") issues.push(`ServerAliveCountMax is ${count} (stale, should be 2)`); + if (!content.includes("TCPKeepAlive=yes")) issues.push("TCPKeepAlive yes missing"); + if (!content.includes("ExitOnForwardFailure=yes")) issues.push("ExitOnForwardFailure yes missing"); + return issues; +} + +const PEER_FORWARD_REGENERATE = "bash tools/fleet/install.sh (Fleet — Repair regenerates it: 15/2 + TCPKeepAlive)"; +const PEER_FORWARD_REPLACE = "bash tools/fleet/install.sh (Fleet — Repair replaces it with the managed agent and keeps a backup)"; + +/** Per-peer forward check: one FleetCheck per attached loopback peer (its + * managed agent present, 15/2 + TCPKeepAlive + ExitOnForwardFailure, the + * `127.0.0.1::127.0.0.1:` forward) plus one per OTHER + * forward agent found by label convention (a hand-installed legacy agent, or + * a managed agent for a peer no longer attached). darwin only; [] when there + * is nothing to report. */ +export function checkPeerForwards(opts: { + launchAgentsDir: string; + /** The reader peer-token store (default: fleet_peer_store's resolution). */ + peerStoreFile?: string; + platform?: string; + topology?: FleetTopologyState; + listDir?: (dir: string) => string[]; + read?: (p: string) => string; +}): FleetCheck[] { + if ((opts.platform ?? process.platform) !== "darwin") return []; + const read = opts.read ?? ((p: string) => fs.readFileSync(p, "utf8")); + const listDir = opts.listDir ?? ((d: string) => fs.readdirSync(d)); + let files: string[] = []; + try { + files = listDir(opts.launchAgentsDir).filter((f) => FORWARD_AGENT_FILE.test(f)).sort(); + } catch { + files = []; + } + const attached = listPeerBaseUrls({ storeFile: opts.peerStoreFile }) + .map((p) => ({ machineId: p.machineId, port: loopbackForwardPort(p.baseUrl) })) + .filter((p): p is { machineId: string; port: number } => p.port !== null); + const remotePort = fleetPort(opts.topology ?? readFleetTopology()); + + const checks: FleetCheck[] = []; + const claimed = new Set(); + for (const peer of attached) { + const label = peerForwardLabel(peer.machineId); + const file = `${label}.plist`; + claimed.add(file); + const name = `Fleet peer forward (${peer.machineId})`; + const portForward = `127.0.0.1:${peer.port}:127.0.0.1:${remotePort}`; + const content = files.includes(file) ? readOrNull(path.join(opts.launchAgentsDir, file), read) : null; + if (content == null) { + checks.push({ + name, + ok: false, + detail: `launchd forward agent missing (${path.join(opts.launchAgentsDir, file)})`, + fix: "bash tools/fleet/install.sh (Fleet — Repair generates it)", + }); + continue; + } + const issues = forwardKeepaliveIssues(content); + if (!content.includes(portForward)) issues.push(`LocalForward ${portForward} missing`); + checks.push( + issues.length > 0 + ? { name, ok: false, detail: issues.join("; "), fix: PEER_FORWARD_REGENERATE } + : { name, ok: true, detail: `ServerAlive 15/2 + TCPKeepAlive, ${portForward} forward` }, + ); + } + + for (const file of files) { + if (claimed.has(file)) continue; + const content = readOrNull(path.join(opts.launchAgentsDir, file), read) ?? ""; + const issues = forwardKeepaliveIssues(content); + const managed = MANAGED_FORWARD_FILE.exec(file); + if (managed !== null) { + // A managed agent whose peer is no longer in the token store. + const name = `Fleet peer forward (${managed[1]})`; + const note = "no attached peer in the token store"; + checks.push( + issues.length > 0 + ? { name, ok: false, detail: `${issues.join("; ")} (${note})`, fix: PEER_FORWARD_REGENERATE } + : { name, ok: true, detail: `ServerAlive 15/2 + TCPKeepAlive (${note})` }, + ); + continue; + } + // A hand-installed `co.harmoniqs.amico--forward`. It belongs to an + // attached peer when it binds that peer's local port or names its machine + // id as the ssh destination — the installer replaces exactly those. + const label = file.replace(/\.plist$/, ""); + const name = `Fleet peer forward (hand-installed ${label})`; + const owner = attached.find( + (p) => new RegExp(`((127\\.0\\.0\\.1|localhost):)?${p.port}:`).test(content) || content.includes(`${p.machineId}`), + ); + if (owner !== undefined) { + checks.push({ + name, + ok: false, + detail: `hand-installed forward for peer ${owner.machineId}, not managed by the installer${issues.length > 0 ? ` — ${issues.join("; ")}` : ""}`, + fix: PEER_FORWARD_REPLACE, + }); + } else if (issues.length > 0) { + checks.push({ + name, + ok: false, + detail: `${issues.join("; ")} (no attached peer matches it, so the installer leaves it alone)`, + fix: `retune it to 15/2 + TCPKeepAlive, or attach its peer and run bash tools/fleet/install.sh (Fleet — Repair then replaces it and keeps a backup)`, + }); + } else { + checks.push({ name, ok: true, detail: "ServerAlive 15/2 + TCPKeepAlive (hand-installed; no attached peer matches it)" }); + } + } + return checks; +} + /** Fleet role check — surfaces the current mode from the projection topology. * Every state renders honestly (#1106): ok → the role + canonical target + * the D1 freshness verdict when present; absent → the base-default @@ -209,10 +359,13 @@ export function checkFleetRole( return { name: "Fleet role", ok: false, detail: `unknown role: ${role}`, fix: "refresh the projection: `amico fleet status --projection`" }; } -/** Aggregate helper — returns all fleet checks (role + guard + settings + tunnel). +/** Aggregate helper — returns all fleet checks (role + guard + settings + tunnel + * + per-peer forwards). * When role is standalone (or the projection is absent — the same base * default), guard/settings/tunnel checks are skipped (not relevant); a - * broken projection surfaces ONLY the role check's rendered fail. */ + * broken projection surfaces ONLY the role check's rendered fail. The + * per-peer forward checks (#1775) run on EVERY role — a peer may be + * standalone, server, or client — whenever a LaunchAgents dir is supplied. */ export function fleetHealthReport(args: { repoGuardPath: string; installedGuardPath?: string; @@ -224,9 +377,24 @@ export function fleetHealthReport(args: { platform?: string; /** #1106: the projection topology state (injectable; default: readFleetTopology()). */ topology?: FleetTopologyState; + /** #1775: the LaunchAgents dir the per-peer forward agents live in + * (production: ~/Library/LaunchAgents). Absent → no peer forward checks. */ + launchAgentsDir?: string; + /** #1775: the reader peer-token store (default: fleet_peer_store's resolution). */ + peerStoreFile?: string; }): FleetCheck[] { const topology = args.topology ?? readFleetTopology(); const role = topology.kind === "ok" ? topology.role : "standalone"; + // The agents are read from the injected dir itself — `read` above is the + // guard-file injection and never stands in for a plist read. + const peerForwards = (): FleetCheck[] => + args.launchAgentsDir === undefined + ? [] + : checkPeerForwards({ launchAgentsDir: args.launchAgentsDir, peerStoreFile: args.peerStoreFile, platform: args.platform, topology }); + + if (topology.kind === "broken") { + return [checkFleetRole({ topology, platform: args.platform })]; + } // Guard/settings/tunnel checks are only relevant for CLIENT machines — they // verify the engine-blocking guard, the binary redirect, and the SSH tunnel @@ -237,6 +405,7 @@ export function fleetHealthReport(args: { if (role !== "client" || topology.kind !== "ok") { return [ checkFleetRole({ topology, platform: args.platform }), + ...peerForwards(), ]; } @@ -245,5 +414,6 @@ export function fleetHealthReport(args: { checkFleetGuard(args.repoGuardPath, args.installedGuardPath, { read: args.read, isExecutable: args.isExecutable, platform: args.platform }), checkFleetSettings(args.configuredBinary, args.configuredPort, { platform: args.platform, topology }), checkFleetTunnel(args.plistContent, { platform: args.platform, topology }), + ...peerForwards(), ]; } diff --git a/packages/extension/test/amicode_service_engine_proxy.test.ts b/packages/extension/test/amicode_service_engine_proxy.test.ts index 8ac22cc3b..8957598f8 100644 --- a/packages/extension/test/amicode_service_engine_proxy.test.ts +++ b/packages/extension/test/amicode_service_engine_proxy.test.ts @@ -13,6 +13,10 @@ import * as http from "node:http"; import { AddressInfo } from "node:net"; import { createAmicodeService } from "../src/amicode_service"; import { serverAuthHeader } from "../src/server_auth"; +import { EngineProxy } from "../src/amicode_service/engine_proxy"; +import { HubProxy } from "../src/amicode_service/hub_proxy"; +import type { HubCredentialRead } from "../src/amicode_service/hub_credential"; +import { fakeEngine, startEngineSseServer, LOCAL_EPOCH } from "./fixtures/engine_stream/engine_frames"; interface RecordedHit { method: string; @@ -187,3 +191,102 @@ describe("amicode service — engine proxy (transparent passthrough to the spawn expect(last - first, `first@${first}ms last@${last}ms — buffered, not streamed`).toBeGreaterThanOrEqual(180); }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1773 AC3 — a proxied stream lost AFTER its response headers were sent is an +// ERROR close, never a clean end. The proxies used to answer an upstream failure +// mid-stream with `res.end()`, which a listener (or the next relay) reads as +// "the stream ended normally, nothing was lost". Both the engine proxy and the +// hub proxy now destroy the downstream with the error, so it sees a transport +// error. A clean upstream end still passes through as a clean end. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1773 AC3 — engine and hub proxies error-close a stream lost after headers", () => { + /** Serve `handler` on 127.0.0.1:0 and dial it at its own address. */ + async function front(handler: (req: http.IncomingMessage, res: http.ServerResponse) => void) { + const server = http.createServer(handler); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + stop: () => + new Promise((r) => { + server.close(() => r()); + server.closeAllConnections?.(); + }), + }; + } + + /** Read a whole response: its status, body, and whether it ENDED cleanly or ERRORED. */ + function read(url: string): Promise<{ status: number; body: string; how: "end" | "error" }> { + return new Promise((resolve, reject) => { + const req = http.get(url, (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (c: string) => (body += c)); + res.on("end", () => resolve({ status: res.statusCode ?? 0, body, how: "end" })); + res.on("error", () => resolve({ status: res.statusCode ?? 0, body, how: "error" })); + }); + req.on("error", reject); + }); + } + + const hubCredential = (): HubCredentialRead => ({ + ok: true, + mint: "hub", + credential: { baseUrl: "http://hub.invalid", token: "hub-tok" }, + }); + + const proxies: Array<{ name: string; handler: (upstreamUrl: string) => (req: http.IncomingMessage, res: http.ServerResponse) => void }> = [ + { + name: "engine proxy", + handler: (upstreamUrl) => { + const proxy = new EngineProxy({ getUrl: () => upstreamUrl }); + return (req, res) => void proxy.handle(req, res); + }, + }, + { + name: "hub proxy", + handler: (upstreamUrl) => { + const proxy = new HubProxy({ getUrl: () => upstreamUrl, credential: hubCredential }); + return (req, res) => void proxy.handle(req, res); + }, + }, + ]; + + for (const { name, handler } of proxies) { + it(`${name}: the upstream dies mid-stream → the listener's response ERRORS after the frames it got (no clean end)`, async () => { + const engine = fakeEngine({ legacy: true }); + const sent = [engine.connected(), engine.event("session.status", { sessionID: "ses_1", status: { type: "busy" } })]; + const upstream = await startEngineSseServer((res) => { + for (const f of sent) res.write(f); + setTimeout(() => res.destroy(), 20); // the link dies after the headers and two frames + }); + const relay = await front(handler(upstream.url)); + try { + const { status, body, how } = await read(`${relay.url}/global/event`); + expect(status).toBe(200); // the headers were already relayed … + expect(body).toBe(sent.join("")); // … with every frame the upstream sent … + expect(how).toBe("error"); // … so the loss must surface as a transport error + } finally { + await relay.stop(); + await upstream.stop(); + } + }); + + it(`${name}: a clean upstream end still relays as a clean end (pass-through unchanged)`, async () => { + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const sent = [engine.connected(), engine.heartbeat()]; + const upstream = await startEngineSseServer((res) => { + for (const f of sent) res.write(f); + res.end(); + }); + const relay = await front(handler(upstream.url)); + try { + expect(await read(`${relay.url}/global/event`)).toEqual({ status: 200, body: sent.join(""), how: "end" }); + } finally { + await relay.stop(); + await upstream.stop(); + } + }); + } +}); diff --git a/packages/extension/test/amicode_service_fleet_session_status_route.test.ts b/packages/extension/test/amicode_service_fleet_session_status_route.test.ts index 9bbb641a8..a4b86851b 100644 --- a/packages/extension/test/amicode_service_fleet_session_status_route.test.ts +++ b/packages/extension/test/amicode_service_fleet_session_status_route.test.ts @@ -11,7 +11,7 @@ // - served: it reads the LIVE owner map of whichever boot branch built one // (observation-only base peer AND the premium staged plane), fanning out to // the owner in the session's exact directory with the owner's own credential. -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll, vi } from "vitest"; import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -313,6 +313,41 @@ describe("#1766 — GET /amicode/fleet/session-status (route-level)", () => { } }); + // ── #1771: both boot branches log the owner-map refresh + the fan-in census ── + it.each([ + { name: "observation-only base peer", premium: false, plane: "observe" }, + { name: "premium staged plane", premium: true, plane: "premium" }, + ])("#1771 $name: the service log gets each refresh's per-peer outcome and the fan-in census", async ({ premium, plane }) => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + const lines: string[] = []; + let notify: (() => void) | undefined; + const svc = createAmicodeService({ + password: "service-own-mint", + engine: engine(), + fleet: { + hub: { getUrl: () => undefined }, + fleetPeers: servingPeerProvider(), + ...(premium ? { entitlements: ["amicissimo"], overlaySource } : { observationOnly: true }), + }, + log: (l) => { + lines.push(l); + notify?.(); + }, + }); + try { + // the feed's immediate refresh reaches the peer: ONE line for the peer + const isPeerLine = (l: string) => l.startsWith(`[owner-map] refresh peer=${PEER_ID} `); + if (!lines.some(isPeerLine)) await new Promise((r) => (notify = () => lines.some(isPeerLine) && r())); + expect(lines.find(isPeerLine)).toMatch(new RegExp(`^\\[owner-map\\] refresh peer=${PEER_ID} outcome=ok duration_ms=\\d+ entries=1$`)); + expect(lines.some((l) => l.includes(`peer=${LOCAL_ID} `))).toBe(false); // the local engine is not a peer + vi.advanceTimersByTime(30_000); + expect(lines.filter((l) => l.startsWith(`[fanin] census plane=${plane} listeners=0 `))).toHaveLength(1); + } finally { + await svc.stop(); + vi.useRealTimers(); + } + }); + it("an empty / missing ids parameter is an empty 200 map with no upstream status call", async () => { const svc = createAmicodeService({ password: "service-own-mint", diff --git a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts index 73c0f36e9..458204e73 100644 --- a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts +++ b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts @@ -26,8 +26,14 @@ import { SessionOwnerMap, SessionMultiplexProxy, PENDING_OWNER_TTL_MS, + OWNER_RETENTION_MS, + OWNER_MAP_REFRESH_MS, + fanInReachable, type SessionEntry, } from "../src/amicode_service/session_multiplexer"; +import { createAmicodeService } from "../src/amicode_service"; +import { SseFanInDriver, type FanInUpstreamRequest } from "../src/amicode_service/sse_fanin_driver"; +import { honestSourceComment, type SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; import { AmicodeServiceServer, fleetMultiplexEnabled, @@ -40,6 +46,7 @@ import { EngineProxy } from "../src/amicode_service/engine_proxy"; import { writeAttachmentPointerFile } from "../src/amicode_service/attachment_pointer"; import { writeKeeperPointerFile } from "../src/amicode_service/keeper_pointer"; import { writeHubCredential, readHubCredential } from "../src/amicode_service/hub_credential"; +import { buildOwnerRoutingProjection, type OwnerRefreshPeer } from "../src/amicode_service/merged_projection"; import { serverAuthHeader } from "../src/server_auth"; // ── a marker stub: returns a distinct marker for every request; records paths ── @@ -148,6 +155,572 @@ describe("#1449 AC1 — SessionOwnerMap fed by a live projection loop (not a tes }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1771 — every ownership refresh logs, PER PEER, its outcome (ok / timeout / +// error / unauthorized), duration and entry count. The projection builder never +// throws and the feed never throws, so before this a peer whose refresh failed +// was silently missing from the map (its event arm closed, its sessions routed +// local) with nothing in any log. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1771 — each ownership refresh logs a per-peer outcome record", () => { + it("one line per peer: outcome, duration, entries — a failed peer refresh is visible though nothing throws", async () => { + const ownerMap = new SessionOwnerMap(); + const lines: string[] = []; + const feed = new OwnerMapFeed({ + ownerMap, + log: (l) => lines.push(l), + buildProjection: async () => ({ + sessions: [{ id: "ses-A", amicode_owner: { owner_machine_id: "studio", owner_name: "Studio", is_local: false } }], + peers: [ + { machineId: "studio", outcome: "ok", durationMs: 120, entries: 1 }, + { machineId: "mini", outcome: "timeout", durationMs: 10_004, entries: 0, reason: "fetch-failed", detail: "The operation was aborted due to timeout" }, + ], + }), + }); + await expect(feed.refreshOnce()).resolves.toBeUndefined(); + expect(ownerMap.resolveOwner("ses-A")).toBe("studio"); + expect(lines).toEqual([ + "[owner-map] refresh peer=studio outcome=ok duration_ms=120 entries=1", + '[owner-map] refresh peer=mini outcome=timeout duration_ms=10004 entries=0 reason=fetch-failed detail="The operation was aborted due to timeout"', + ]); + }); + + it("through the REAL routing projection: an unauthorized peer is a named outcome; the healthy peer still lands", async () => { + const ownerMap = new SessionOwnerMap(); + const lines: string[] = []; + const fetchImpl = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.startsWith("http://peer-b")) return new Response("no", { status: 401 }); + if (url.endsWith("/global/health")) return new Response(JSON.stringify({ version: "t" }), { status: 200 }); + if (url.startsWith("http://peer-a")) return new Response(JSON.stringify([{ id: "ses-a1" }, { id: "ses-a2" }]), { status: 200 }); + return new Response(JSON.stringify([]), { status: 200 }); // local + }) as unknown as typeof fetch; + const feed = new OwnerMapFeed({ + ownerMap, + log: (l) => lines.push(l), + buildProjection: async () => { + const p = await buildOwnerRoutingProjection({ + localMachineId: "macbook", + local: { getUrl: () => "http://local" }, + peers: [ + { machineId: "peer-a", getUrl: () => "http://peer-a", token: "tok-a" }, + { machineId: "peer-b", getUrl: () => "http://peer-b", token: "tok-b" }, + ], + rosterLookup: () => undefined, + fetchImpl, + }); + return { sessions: p.sessions as SessionEntry[], peers: p.peers }; + }, + }); + await expect(feed.refreshOnce()).resolves.toBeUndefined(); // never throws + expect(ownerMap.resolveOwner("ses-a1")).toBe("peer-a"); + const a = lines.find((l) => l.includes("peer=peer-a "))!; + const b = lines.find((l) => l.includes("peer=peer-b "))!; + expect(a).toMatch(/^\[owner-map\] refresh peer=peer-a outcome=ok duration_ms=\d+ entries=2$/); + expect(b).toMatch(/^\[owner-map\] refresh peer=peer-b outcome=unauthorized duration_ms=\d+ entries=0 reason=unauthorized$/); + expect(lines).toHaveLength(2); // per PEER — the local engine is not a peer + }); + + it("a projection BUILD failure is logged too (and still never throws)", async () => { + const lines: string[] = []; + const feed = new OwnerMapFeed({ + ownerMap: new SessionOwnerMap(), + log: (l) => lines.push(l), + buildProjection: async () => { + throw new Error("peer fan-out failed"); + }, + }); + await expect(feed.refreshOnce()).resolves.toBeUndefined(); + expect(lines).toEqual(['[owner-map] refresh failed error="peer fan-out failed"']); + }); + + it("#1774: a failed peer's line says what the map did with its entries — retained, expired, dropped, still unavailable", async () => { + let t = 0; + const ownerMap = new SessionOwnerMap({ now: () => t }); + const lines: string[] = []; + let next: { sessions: SessionEntry[]; peers: OwnerRefreshPeer[] } = { + sessions: [ + { id: "ses-s1", amicode_owner: { owner_machine_id: "studio", owner_name: "Studio", is_local: false } }, + { id: "ses-s2", amicode_owner: { owner_machine_id: "studio", owner_name: "Studio", is_local: false } }, + { id: "ses-m1", amicode_owner: { owner_machine_id: "mini", owner_name: "Mini", is_local: false } }, + ], + peers: [ + { machineId: "studio", outcome: "ok", durationMs: 40, entries: 2 }, + { machineId: "mini", outcome: "ok", durationMs: 30, entries: 1 }, + ], + }; + const feed = new OwnerMapFeed({ ownerMap, log: (l) => lines.push(l), buildProjection: async () => next }); + await feed.refreshOnce(); + expect(lines.splice(0)).toEqual([ + "[owner-map] refresh peer=studio outcome=ok duration_ms=40 entries=2", + "[owner-map] refresh peer=mini outcome=ok duration_ms=30 entries=1", + ]); + + t = 15_000; + next = { + sessions: [], + peers: [ + { machineId: "studio", outcome: "timeout", durationMs: 10_004, entries: 0, reason: "fetch-failed", detail: "The operation was aborted due to timeout" }, + { machineId: "mini", outcome: "unauthorized", durationMs: 9, entries: 0, reason: "unauthorized" }, + ], + }; + await feed.refreshOnce(); + expect(lines.splice(0)).toEqual([ + '[owner-map] refresh peer=studio outcome=timeout duration_ms=10004 entries=0 reason=fetch-failed detail="The operation was aborted due to timeout" retention=retained held=2 age_ms=15000', + "[owner-map] refresh peer=mini outcome=unauthorized duration_ms=9 entries=0 reason=unauthorized retention=dropped dropped=1", + ]); + + t = OWNER_RETENTION_MS + 1; + next = { sessions: [], peers: [{ machineId: "studio", outcome: "error", durationMs: 12, entries: 0, reason: "fetch-failed", detail: "HTTP 502" }] }; + await feed.refreshOnce(); + expect(lines.splice(0)).toEqual([ + '[owner-map] refresh peer=studio outcome=error duration_ms=12 entries=0 reason=fetch-failed detail="HTTP 502" retention=expired dropped=2', + ]); + + t += 5_000; + await feed.refreshOnce(); + expect(lines.splice(0)).toEqual([ + '[owner-map] refresh peer=studio outcome=error duration_ms=12 entries=0 reason=fetch-failed detail="HTTP 502" retention=unavailable', + ]); + }); +}); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 — the ownership map keeps a peer's sessions across a failed refresh. A +// failed peer fetch is an EMPTY slice in the projection (the builder never +// throws), so before this the peer's sessions vanished from the map on one blip: +// its event arm closed without notice and its sessions routed local (404/empty). +// Retention is decided PER PEER from #1771's refresh outcome — timeouts, network +// errors and 5xx retain; an unauthorized answer does not (a credential problem). +// ══════════════════════════════════════════════════════════════════════════════ +const ownedBy = (id: string, owner: string, directory?: string): SessionEntry => ({ + id, + ...(directory !== undefined ? { directory } : {}), + amicode_owner: { owner_machine_id: owner, owner_name: owner, is_local: false }, +}); +const peerOk = (machineId: string, entries = 1): OwnerRefreshPeer => ({ machineId, outcome: "ok", durationMs: 40, entries }); +const peerTimeout = (machineId: string): OwnerRefreshPeer => ({ + machineId, + outcome: "timeout", + durationMs: 10_004, + entries: 0, + reason: "fetch-failed", + detail: "The operation was aborted due to timeout", +}); +const peer5xx = (machineId: string): OwnerRefreshPeer => ({ machineId, outcome: "error", durationMs: 12, entries: 0, reason: "fetch-failed", detail: "HTTP 502" }); +const peerNetErr = (machineId: string): OwnerRefreshPeer => ({ machineId, outcome: "error", durationMs: 3, entries: 0, reason: "fetch-failed", detail: "fetch failed" }); +const peerUnauthorized = (machineId: string): OwnerRefreshPeer => ({ machineId, outcome: "unauthorized", durationMs: 9, entries: 0, reason: "unauthorized" }); + +describe("#1774 AC1 — a refresh in which one peer times out or errors keeps that peer's previous owner entries", () => { + for (const [label, failed] of [ + ["a timeout", peerTimeout("studio")], + ["a 5xx answer", peer5xx("studio")], + ["a network error", peerNetErr("studio")], + ] as const) { + it(`${label}: the peer's sessions still resolve to it (owner + directory); the healthy peer is replaced as usual`, async () => { + const ownerMap = new SessionOwnerMap(); + let next: { sessions: SessionEntry[]; peers: OwnerRefreshPeer[] } = { + sessions: [ownedBy("ses-s1", "studio", "/srv/a"), ownedBy("ses-s2", "studio"), ownedBy("ses-m1", "mini")], + peers: [peerOk("studio", 2), peerOk("mini")], + }; + const feed = new OwnerMapFeed({ ownerMap, buildProjection: async () => next }); + await feed.refreshOnce(); + // studio's fetch fails: its slice is EMPTY in this projection; mini answers. + next = { sessions: [ownedBy("ses-m2", "mini")], peers: [failed, peerOk("mini")] }; + await feed.refreshOnce(); + expect(ownerMap.resolveOwner("ses-s1")).toBe("studio"); + expect(ownerMap.resolveOwner("ses-s2")).toBe("studio"); + expect(ownerMap.resolveDirectory("ses-s1")).toBe("/srv/a"); + expect(ownerMap.ownerMachineIds().sort()).toEqual(["mini", "studio"]); + // the healthy peer's refresh is still authoritative + expect(ownerMap.resolveOwner("ses-m1")).toBeUndefined(); + expect(ownerMap.resolveOwner("ses-m2")).toBe("mini"); + }); + } + + it("through the REAL routing projection: a peer whose session list answers 502 keeps its owner entries", async () => { + let studioDown = false; + const fetchImpl = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.startsWith("http://studio") && studioDown) return new Response("bad gateway", { status: 502 }); + if (url.endsWith("/global/health")) return new Response(JSON.stringify({ version: "t" }), { status: 200 }); + if (url.startsWith("http://studio")) return new Response(JSON.stringify(/archived=true/.test(url) ? [] : [{ id: "ses-s1" }]), { status: 200 }); + return new Response(JSON.stringify([]), { status: 200 }); // local + }) as unknown as typeof fetch; + const ownerMap = new SessionOwnerMap(); + const feed = new OwnerMapFeed({ + ownerMap, + buildProjection: async () => { + const p = await buildOwnerRoutingProjection({ + localMachineId: "macbook", + local: { getUrl: () => "http://local" }, + peers: [{ machineId: "studio", getUrl: () => "http://studio", token: "tok" }], + rosterLookup: () => undefined, + fetchImpl, + }); + return { sessions: p.sessions as SessionEntry[], peers: p.peers }; + }, + }); + await feed.refreshOnce(); + expect(ownerMap.resolveOwner("ses-s1")).toBe("studio"); + studioDown = true; + await feed.refreshOnce(); + expect(ownerMap.resolveOwner("ses-s1")).toBe("studio"); // retained, not dropped to local + }); + + it("its event arm stays OPEN across the failed refresh — no close, no unavailable comment", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + const origins = armOrigins(); + const driver = retentionDriver(origins, ownerMap); + const res = sseRes(); + driver.handle(sseReq(), res); + await settle(); + expect(origins.opens("studio")).toBe(1); + + ownerMap.update([], [peerTimeout("studio")]); // studio's slice is empty this refresh + driver.reconcile(); + await settle(); + expect(origins.closes("studio")).toBe(0); + expect(origins.opens("studio")).toBe(1); // the same arm, never re-opened + expect(res.text()).not.toContain("unavailable"); + res.fireClose(); + }); +}); + +describe("#1774 AC2 — past the retention window the entries expire, the arm closes, and listeners get the named unavailable comment", () => { + it(`the default window is ${OWNER_RETENTION_MS} ms since the peer's last GOOD refresh: retained through it, expired past it`, () => { + let t = 1_000_000; + const m = new SessionOwnerMap({ now: () => t }); + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + t += 30_000; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-s1")).toBe("studio"); + t += 30_000; // exactly the window since the last good refresh + m.update([], [peer5xx("studio")]); + expect(m.resolveOwner("ses-s1")).toBe("studio"); + t += 1; // past it + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-s1")).toBeUndefined(); + expect(m.size).toBe(0); + expect(m.isUnavailable("studio")).toBe(true); + // still a NAMED owner of the fan-in membership (so the stream says so) … + expect(m.ownerMachineIds()).toEqual(["studio"]); + // … and never an active arm, whatever its transport says + expect(fanInReachable(m, () => true)("studio")).toBe(false); + }); + + it("the window is configurable per map (retentionMs)", () => { + let t = 0; + const m = new SessionOwnerMap({ now: () => t, retentionMs: 5_000 }); + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + t = 5_000; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-s1")).toBe("studio"); + t = 5_001; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-s1")).toBeUndefined(); + }); + + it("expiry closes the peer's arm and the listener receives the named unavailable comment — not silence", async () => { + let t = 0; + const ownerMap = new SessionOwnerMap({ now: () => t }); + ownerMap.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + const origins = armOrigins(); + const driver = retentionDriver(origins, ownerMap); + const res = sseRes(); + driver.handle(sseReq(), res); + await settle(); + expect(origins.opens("studio")).toBe(1); + + t = OWNER_RETENTION_MS + 1; + ownerMap.update([], [peerTimeout("studio")]); + driver.reconcile(); + await settle(); + expect(origins.closes("studio")).toBe(1); + expect(res.text()).toContain(honestSourceComment("studio", "peer-unreachable")); + // announced once, not on every reconcile + driver.reconcile(); + await settle(); + expect(res.text().split("source studio unavailable").length - 1).toBe(1); + res.fireClose(); + }); + + it("an UNAUTHORIZED refresh does not retain: the entries drop at once and the peer is named unavailable (Key Decision)", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + const origins = armOrigins(); + const driver = retentionDriver(origins, ownerMap); + const res = sseRes(); + driver.handle(sseReq(), res); + await settle(); + + ownerMap.update([], [peerUnauthorized("studio")]); // well inside the window + expect(ownerMap.resolveOwner("ses-s1")).toBeUndefined(); + driver.reconcile(); + await settle(); + expect(origins.closes("studio")).toBe(1); + expect(res.text()).toContain(honestSourceComment("studio", "peer-unreachable")); + res.fireClose(); + }); + + it("a trust or credential problem drops like unauthorized; a transport-down peer retains like a timeout", () => { + const m = new SessionOwnerMap(); + for (const reason of ["untrusted", "credential-missing", "identity-conflict"] as const) { + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + m.update([], [{ machineId: "studio", outcome: "error", reason }]); + expect(m.resolveOwner("ses-s1")).toBeUndefined(); + } + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + m.update([], [{ machineId: "studio", outcome: "error", reason: "no-upstream" }]); + expect(m.resolveOwner("ses-s1")).toBe("studio"); + }); + + it("a peer that LEAVES the refreshed set is gone, not retained and not named unavailable", () => { + const m = new SessionOwnerMap(); + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + m.update([], []); // studio is no longer a serving peer + expect(m.resolveOwner("ses-s1")).toBeUndefined(); + expect(m.isUnavailable("studio")).toBe(false); + expect(m.ownerMachineIds()).toEqual([]); + }); + + it("production wiring (observation boot): a peer whose refresh turns unauthorized is announced unavailable on the live /event", async () => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); // the feed + reconcile cadences; HTTP stays real + let studioDenies = false; + const local = await startFleetStub(() => false, []); + const studio = await startFleetStub(() => studioDenies, [{ id: "ses-s1" }]); + const root = mkdtempSync(join(tmpdir(), "amicode-1774-wire-")); + const savedHub = process.env.AMICO_FLEET_HUB_FILE; + process.env.AMICO_FLEET_HUB_FILE = join(root, "hub-cred-absent.json"); + const svc = createAmicodeService({ + password: "svc-1774", + engine: { password: "engine-1774", getUrl: () => local.url }, + fleet: { + hub: { getUrl: () => undefined }, + observationOnly: true, + fleetPeers: { + localMachineId: "macbook", + getServingPeers: () => [{ machineId: "studio" }], + readPeerToken: (id) => (id === "studio" ? { ok: true, credential: { baseUrl: studio.url, token: "tok-studio" } } : { ok: false }), + rosterLookup: (id) => ({ name: id }), + }, + }, + }); + const origin = (await svc.start()).toString().replace(/\/$/, ""); + const stream = new AbortController(); + try { + expect(await until(() => studio.listReads() >= 2)).toBe(true); // the boot refresh (active + archived) + await settle(); + const res = await fetch(`${origin}/event`, { headers: { Authorization: serverAuthHeader("engine-1774") }, signal: stream.signal }); + const text = streamText(res); + expect(await until(() => studio.eventOpens() === 1)).toBe(true); // the peer arm is open + + studioDenies = true; + const reads = studio.listReads(); + await vi.advanceTimersByTimeAsync(OWNER_MAP_REFRESH_MS); // the next owner-map refresh + expect(await until(() => studio.listReads() >= reads + 2)).toBe(true); + await settle(); + await vi.advanceTimersByTimeAsync(1_000); // the listener's reconcile tick + expect(await until(() => text().includes(honestSourceComment("studio", "peer-unreachable")))).toBe(true); + } finally { + stream.abort(); + await svc.stop(); + vi.useRealTimers(); + await Promise.all([local.stop(), studio.stop()]); + if (savedHub === undefined) delete process.env.AMICO_FLEET_HUB_FILE; + else process.env.AMICO_FLEET_HUB_FILE = savedHub; + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe("#1774 AC3 — a successful refresh replaces the peer's entries authoritatively", () => { + it("after a retained failure, the next good refresh drops the sessions that ended and adds the new ones", () => { + let t = 0; + const m = new SessionOwnerMap({ now: () => t }); + m.update([ownedBy("ses-ended", "studio"), ownedBy("ses-kept", "studio", "/old")], [peerOk("studio", 2)]); + t = 10_000; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-ended")).toBe("studio"); // retained through the blip + t = 20_000; + m.update([ownedBy("ses-kept", "studio", "/new"), ownedBy("ses-born", "studio")], [peerOk("studio", 2)]); + expect(m.resolveOwner("ses-ended")).toBeUndefined(); // it ended — gone + expect(m.resolveOwner("ses-kept")).toBe("studio"); + expect(m.resolveDirectory("ses-kept")).toBe("/new"); + expect(m.resolveOwner("ses-born")).toBe("studio"); + expect(m.size).toBe(2); + // and the good refresh restarts the window: a failure 59 s later still retains + t = 20_000 + OWNER_RETENTION_MS - 1_000; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-born")).toBe("studio"); + expect(m.resolveOwner("ses-ended")).toBeUndefined(); // never resurrected from the stale snapshot + }); + + it("a good refresh after expiry ends the unavailability: the peer is an ordinary owner again and its arm re-opens", async () => { + let t = 0; + const ownerMap = new SessionOwnerMap({ now: () => t }); + ownerMap.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + const origins = armOrigins(); + const driver = retentionDriver(origins, ownerMap); + const res = sseRes(); + driver.handle(sseReq(), res); + await settle(); + t = OWNER_RETENTION_MS + 1; + ownerMap.update([], [peerTimeout("studio")]); + driver.reconcile(); + await settle(); + expect(origins.closes("studio")).toBe(1); + + t += 5_000; + ownerMap.update([ownedBy("ses-s2", "studio")], [peerOk("studio")]); + expect(ownerMap.isUnavailable("studio")).toBe(false); + expect(ownerMap.resolveOwner("ses-s2")).toBe("studio"); + expect(ownerMap.resolveOwner("ses-s1")).toBeUndefined(); + driver.reconcile(); + await settle(); + expect(origins.opens("studio")).toBe(2); // the arm is back + res.fireClose(); + }); +}); + +/** A fleet engine stub: its cross-project session list (401 while `denies()`), + * its health, and a /global/event stream that stays open. */ +async function startFleetStub(denies: () => boolean, sessions: Array>) { + let listReads = 0; + let eventOpens = 0; + const server = http.createServer((req, res) => { + const u = new URL(req.url ?? "/", "http://stub"); + if (u.pathname === "/experimental/session") { + listReads++; + if (denies()) return void res.writeHead(401).end("no"); + res.writeHead(200, { "content-type": "application/json" }); + return void res.end(JSON.stringify(u.searchParams.get("archived") === "true" ? [] : sessions)); + } + if (u.pathname === "/global/health") { + res.writeHead(200, { "content-type": "application/json" }); + return void res.end(JSON.stringify({ version: "stub-1774" })); + } + if (u.pathname === "/global/event") { + eventOpens++; + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + return void res.write(": open\n\n"); + } + res.writeHead(404).end(); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + return { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + listReads: () => listReads, + eventOpens: () => eventOpens, + stop: () => + new Promise((r) => { + server.closeAllConnections(); + server.close(() => r()); + }), + }; +} + +/** Accumulate a streaming response's text in the background. */ +function streamText(res: Response): () => string { + let text = ""; + const reader = res.body!.getReader(); + const decoder = new TextDecoder(); + void (async () => { + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) return; + text += decoder.decode(value, { stream: true }); + } + } catch { + /* aborted at teardown */ + } + })(); + return () => text; +} + +/** Poll (real timers) until `cond` holds or the deadline passes. */ +async function until(cond: () => boolean, timeoutMs = 4_000): Promise { + const start = Date.now(); + while (!cond() && Date.now() - start < timeoutMs) await new Promise((r) => setTimeout(r, 10)); + return cond(); +} + +// ── a deterministic fan-in harness (the driver + aggregator are used as-is) ─── +interface SseRes extends http.ServerResponse { + text(): string; + fireClose(): void; +} +function sseRes(): SseRes { + const chunks: string[] = []; + const closers: Array<() => void> = []; + const res = { + headersSent: false, + write(c: string) { + chunks.push(c); + return true; + }, + writeHead() { + res.headersSent = true; + return res; + }, + flushHeaders() {}, + on(event: string, cb: () => void) { + if (event === "close") closers.push(cb); + return res; + }, + end() {}, + text: () => chunks.join(""), + fireClose: () => closers.forEach((cb) => cb()), + }; + return res as unknown as SseRes; +} +const sseReq = (): http.IncomingMessage => ({ url: "/global/event", headers: {} }) as unknown as http.IncomingMessage; +/** Let the arm pumps drain (microtasks + one macrotask tick). */ +const settle = () => new Promise((r) => setTimeout(r, 20)); + +/** Upstream arms that stay open until closed, counted per namespace. */ +function armOrigins() { + const opens = new Map(); + const closes = new Map(); + const bump = (m: Map, k: string) => m.set(k, (m.get(k) ?? 0) + 1); + const openUpstream = (r: FanInUpstreamRequest): SseFrameSource => { + bump(opens, r.namespace); + const waiters: Array<(v: string | null) => void> = []; + let ended = false; + return { + next: () => (ended ? Promise.resolve(null) : new Promise((resolve) => waiters.push(resolve))), + close: () => { + if (ended) return; + ended = true; + bump(closes, r.namespace); + waiters.splice(0).forEach((w) => w(null)); + }, + }; + }; + return { openUpstream, opens: (ns: string) => opens.get(ns) ?? 0, closes: (ns: string) => closes.get(ns) ?? 0 }; +} + +/** The fan-in driver as index.ts wires it: reachability composed over the owner + * map (`fanInReachable`), so a peer the map names unavailable is never an arm. */ +function retentionDriver(origins: ReturnType, ownerMap: SessionOwnerMap): SseFanInDriver { + const peerBaseUrl = (id: string) => (id === "studio" ? "http://studio.invalid" : undefined); + return new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => "http://local.invalid", + peerBaseUrl, + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: "http://studio.invalid", token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reachable: fanInReachable(ownerMap, (id) => peerBaseUrl(id) !== undefined), + reconcileMs: 999_999, + openUpstream: origins.openUpstream, + }); +} + // ══════════════════════════════════════════════════════════════════════════════ // FREE-TIER REMOTE CREATE (feature/free-tier-fleet) — pending create-time owner // bindings. A session just created on a peer is not yet in the pull-only fleet diff --git a/packages/extension/test/amicode_service_observe_event_routing.test.ts b/packages/extension/test/amicode_service_observe_event_routing.test.ts index 4334ca60a..4ca15ca07 100644 --- a/packages/extension/test/amicode_service_observe_event_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_event_routing.test.ts @@ -10,7 +10,7 @@ // The route dispatch → the observation `/event` interception → the #1511 // aggregator → the real downstream `res` is exercised deterministically via an // INJECTED upstream opener; the live cross-machine delivery is the opt-in E2E. -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll, vi } from "vitest"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -358,6 +358,30 @@ describe("#1543 — production wiring (createAmicodeService observation-only pat await svc.stop(); } }); + + it("#1771: the observation fan-in's census goes to the service log every 30 s, and stops with the service", async () => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + const lines: string[] = []; + try { + const svc = createAmicodeService({ + password: PW, + engine: { password: "engine-mint", getUrl: () => engineStub.url }, + fleet: { hub: { getUrl: () => undefined }, observationOnly: true, fleetPeers: servingPeerProvider() }, + log: (line) => lines.push(line), + }); + const census = () => lines.filter((l) => l.startsWith("[fanin] census ")); + expect(census()).toHaveLength(0); + vi.advanceTimersByTime(30_000); + expect(census()).toHaveLength(1); + expect(census()[0]).toContain("plane=observe"); + expect(census()[0]).toContain("listeners=0 accepted=0 closed=0 evictions=0"); + await svc.stop(); + vi.advanceTimersByTime(90_000); + expect(census()).toHaveLength(1); // the cadence is torn down with the service + } finally { + vi.useRealTimers(); + } + }); }); // ══════════════════════════════════════════════════════════════════════════════ @@ -482,3 +506,116 @@ describe("#1580 — observation fan-in serves /global/event (v1 global-instance expect(joined).toContain('"from":"studio"'); }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1772 — the observation fan-in serves EVERY listener through the REAL dispatch +// path. Two concurrent `/global/event` readers (two windows, a Chat Deck pane, a +// split pane onto the same hub) both stay connected and both receive every +// frame; a listener beyond the cap is refused with a retryable 503 + Retry-After +// and no existing listener is closed to make room. The upstream origins are +// modeled as BROADCAST buses: each arm open is a new subscription, and a frame +// reaches every OPEN subscription — so an evicted listener visibly misses it. +// ══════════════════════════════════════════════════════════════════════════════ +interface BroadcastOrigins { + openUpstream: (r: { namespace: string }) => SseFrameSource; + broadcast(namespace: string, frame: string): void; + opens(namespace: string): number; + live(namespace: string): number; +} +function broadcastOrigins(): BroadcastOrigins { + const subs = new Map>(); + const openCount = new Map(); + return { + openUpstream(r) { + const ctl = controllableSource(); + const set = subs.get(r.namespace) ?? new Set(); + subs.set(r.namespace, set); + set.add(ctl); + openCount.set(r.namespace, (openCount.get(r.namespace) ?? 0) + 1); + return { + next: () => ctl.source.next(), + close: () => { + set.delete(ctl); + ctl.end(); + }, + }; + }, + broadcast(namespace, frame) { + for (const ctl of subs.get(namespace) ?? []) ctl.push(frame); + }, + opens: (ns) => openCount.get(ns) ?? 0, + live: (ns) => subs.get(ns)?.size ?? 0, + }; +} + +async function until(cond: () => boolean, timeoutMs = 2000): Promise { + const start = Date.now(); + while (!cond() && Date.now() - start < timeoutMs) await new Promise((r) => setTimeout(r, 10)); +} + +function observeDriver(origins: BroadcastOrigins, extra: Partial[0]> = {}): SseFanInDriver { + const ownerMap = new SessionOwnerMap(); + ownerMap.update([{ id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }]); + return new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => "http://local.invalid", + peerBaseUrl: (id) => (id === "studio" ? "http://studio.invalid" : undefined), + peerToken: (id) => (id === "studio" ? { ok: true, credential: { baseUrl: "http://studio.invalid", token: "tok-studio" } } : { ok: false, reason: "absent" }), + reconcileMs: 999999, + openUpstream: origins.openUpstream, + ...extra, + }); +} + +describe("#1772 — the real dispatch path accepts concurrent global-stream listeners", () => { + it("AC1 — two concurrent /global/event readers both receive the local AND the peer frame", async () => { + const origins = broadcastOrigins(); + const svc = new AmicodeServiceServer({ password: PW }); + svc.attachObservationEventPlane(observeDriver(origins)); + const origin = (await svc.start()).toString().replace(/\/$/, ""); + try { + const readA = readSseFrames(`${origin}/global/event`, authed, { maxFrames: 2, timeoutMs: 3000 }); + await until(() => origins.opens("local") >= 1); + const readB = readSseFrames(`${origin}/global/event`, authed, { maxFrames: 2, timeoutMs: 3000 }); + await until(() => origins.opens("local") >= 2); + origins.broadcast("local", sseFrame("event: message", 'data: {"from":"macbook"}', "id: l1")); + origins.broadcast("studio", sseFrame("event: message", 'data: {"from":"studio"}', "id: p1")); + const [a, b] = await Promise.all([readA, readB]); + for (const frames of [a, b]) { + const joined = frames.join(""); + expect(joined).toContain('"from":"macbook"'); + expect(joined).toContain('"from":"studio"'); + } + } finally { + await svc.stop(); + } + }); +}); + +describe("#1772 AC4 — beyond the (configured) cap, the real dispatch path refuses — retryable, no eviction", () => { + it("maxListeners: 1 → a second /global/event gets 503 + Retry-After; the first listener keeps streaming", async () => { + const origins = broadcastOrigins(); + const svc = new AmicodeServiceServer({ password: PW }); + svc.attachObservationEventPlane(observeDriver(origins, { maxListeners: 1 })); + const origin = (await svc.start()).toString().replace(/\/$/, ""); + try { + const readA = readSseFrames(`${origin}/global/event`, authed, { maxFrames: 1, timeoutMs: 3000 }); + await until(() => origins.opens("local") >= 1); + + const refused = await fetch(`${origin}/global/event`, { headers: authed, signal: AbortSignal.timeout(3000) }); + expect(refused.status).toBe(503); + expect(Number(refused.headers.get("retry-after"))).toBeGreaterThan(0); + const body = (await refused.json()) as { ok: boolean; reason: string; max_listeners: number }; + expect(body).toMatchObject({ ok: false, reason: "listener-cap", max_listeners: 1 }); + + // The refusal opened no arm and closed none: the first listener is intact. + expect(origins.opens("local")).toBe(1); + expect(origins.live("local")).toBe(1); + origins.broadcast("local", sseFrame("event: message", 'data: {"n":"after-refusal"}', "id: l9")); + expect((await readA).join("")).toContain('"n":"after-refusal"'); + } finally { + await svc.stop(); + } + }); +}); diff --git a/packages/extension/test/amicode_service_observe_read_routing.test.ts b/packages/extension/test/amicode_service_observe_read_routing.test.ts index 795c25362..92eae6ac9 100644 --- a/packages/extension/test/amicode_service_observe_read_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_read_routing.test.ts @@ -21,7 +21,7 @@ import { join } from "node:path"; import * as http from "node:http"; import { AddressInfo } from "node:net"; -import { AmicodeServiceServer } from "../src/amicode_service/server"; +import { AmicodeServiceServer, OWNER_UNKNOWN_ERROR, OWNER_UNKNOWN_RETRY_AFTER_S } from "../src/amicode_service/server"; import { createAmicodeService } from "../src/amicode_service"; import { EngineProxy } from "../src/amicode_service/engine_proxy"; import { @@ -148,11 +148,14 @@ describe("#1537 — ObservationReadRouter resolution order", () => { expect(r.resolve("PATCH", "/session/ses-studio")).toBeUndefined(); expect(r.resolve("DELETE", "/api/session/ses-studio")).toBeUndefined(); }); - it("local-owned / unowned / not-a-known-peer → undefined (local)", () => { + it("local-owned / not-a-known-peer → undefined (local)", () => { expect(r.resolve("GET", "/session/ses-local")).toBeUndefined(); // local-owned - expect(r.resolve("GET", "/api/session/ses-unknown")).toBeUndefined(); // unowned expect(r.resolve("GET", "/api/session/ses-stranger")).toBeUndefined(); // owner not a known peer }); + it("#1774: an UNOWNED session (no known owner) → the unknown-owner outcome: the caller tries local first", () => { + expect(r.resolve("GET", "/api/session/ses-unknown")).toEqual({ kind: "unknown-owner", sessionId: "ses-unknown" }); + expect(r.resolve("HEAD", "/session/ses-unknown/message")).toEqual({ kind: "unknown-owner", sessionId: "ses-unknown" }); + }); it("/amicode/* (own honesty + local surface) → undefined (never proxied)", () => { expect(r.resolve("GET", "/amicode/fleet/status")).toBeUndefined(); expect(r.resolve("GET", "/amicode/fleet/sessions")).toBeUndefined(); @@ -279,6 +282,135 @@ describe("#1537 AC5 — a known-but-unreachable owner answers honestly, never lo }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC1 — a refresh in which the owner peer timed out keeps its sessions +// routed to it: the read is proxied to the peer, never to the local engine. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1774 AC1 — a peer-owned read keeps routing to the owner across a failed refresh", () => { + it("GET for a session whose owner's last refresh timed out → still the PEER; local NOT dialed", async () => { + const ownerMap = new SessionOwnerMap(); + const studioOwned = { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }; + ownerMap.update([studioOwned], [{ machineId: "studio", outcome: "ok" }]); + ownerMap.update([], [{ machineId: "studio", outcome: "timeout", reason: "fetch-failed" }]); // the slice is empty + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => localStub.url })); + server.attachObservationReadPlane( + createObservationReadPlane({ + ownerMap, + localMachineId: "macbook", + peer: (id) => (id === "studio" ? { getUrl: () => peerStub.url, token: "tok-studio" } : undefined), + }), + ); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const localBefore = localStub.requests.length; + const peerBefore = peerStub.requests.length; + try { + const res = await fetch(`${origin}/api/session/ses-studio`, { headers: authed }); + expect(res.status).toBe(200); + expect(((await res.json()) as { marker: string }).marker).toBe(peerStub.marker); + expect(peerStub.requests.length).toBe(peerBefore + 1); + expect(localStub.requests.length).toBe(localBefore); + } finally { + await server.stop(); + } + }); +}); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC4 — a session-scoped read for an id with NO known owner is tried on the +// LOCAL engine first; if the local engine does not have it (its own 404), the +// answer is a RETRYABLE 503 + Retry-After naming `owner-unknown` — not a 404 and +// not an empty result (the owner map may simply not have caught up). +// ══════════════════════════════════════════════════════════════════════════════ + +/** An engine-like local stub: `GET /session/{id}` (the existence probe the + * service makes) answers 200 for a session it holds and the engine's own 404 + * otherwise; every other session path answers a marker for a held session and + * an EMPTY list for a missing one (the "empty result" a miss must not become). */ +function startEngineLikeStub(held: Set): Promise { + const requests: Stub["requests"] = []; + const server = http.createServer((req, res) => { + const u = new URL(req.url ?? "/", "http://stub"); + requests.push({ method: req.method ?? "GET", path: u.pathname, auth: typeof req.headers.authorization === "string" ? req.headers.authorization : undefined }); + const id = extractSessionIdFromReadPath(u.pathname); + res.setHeader("content-type", "application/json"); + if (id !== undefined && !held.has(id)) { + if (/^\/(api\/)?session\/[^/]+$/.test(u.pathname)) return void res.writeHead(404).end(JSON.stringify({ name: "NotFoundError", data: { message: `Session not found: ${id}` } })); + return void res.writeHead(200).end("[]"); + } + res.writeHead(200).end(JSON.stringify({ ok: true, marker: "ENGINE-LIKE" })); + }); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const port = (server.address() as AddressInfo).port; + resolve({ url: `http://127.0.0.1:${port}`, marker: "ENGINE-LIKE", requests, stop: () => new Promise((r) => server.close(() => r())) }); + }); + }); +} + +describe("#1774 AC4 — an unknown-owner read is tried locally; a local miss is a retryable owner-unknown 503", () => { + let engine: Stub; + beforeAll(async () => { + engine = await startEngineLikeStub(new Set(["ses-here"])); + }); + afterAll(async () => { + await engine?.stop(); + }); + const noPeers = (): PeerTransport | undefined => undefined; + + function boot(): AmicodeServiceServer { + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => engine.url })); + server.attachObservationReadPlane( + createObservationReadPlane({ ownerMap: new SessionOwnerMap(), localMachineId: "macbook", peer: noPeers, localEngineUrl: () => engine.url }), + ); + return server; + } + + for (const path of ["/api/session/ses-ghost", "/session/ses-ghost/message", "/api/session/ses-ghost/message?limit=50"]) { + it(`GET ${path} (not on the local engine) → 503 owner-unknown + Retry-After, never the engine's 404 or an empty list`, async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + try { + const res = await fetch(`${origin}${path}`, { headers: authed }); + expect(res.status).toBe(503); + expect(res.headers.get("retry-after")).toBe(String(OWNER_UNKNOWN_RETRY_AFTER_S)); + const body = (await res.json()) as { ok: boolean; error: string; reason: string; session_id: string; retry_after_s: number }; + expect(body).toEqual({ ok: false, error: OWNER_UNKNOWN_ERROR, reason: "owner-unknown", session_id: "ses-ghost", retry_after_s: OWNER_UNKNOWN_RETRY_AFTER_S }); + } finally { + await server.stop(); + } + }); + } + + it("the local engine is probed with the request's OWN credential and directory scope", async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const before = engine.requests.length; + try { + await fetch(`${origin}/session/ses-ghost/message?directory=${encodeURIComponent("/srv/proj")}`, { headers: authed }); + const probe = engine.requests.slice(before); + expect(probe).toEqual([{ method: "GET", path: "/session/ses-ghost", auth: authed.Authorization }]); + } finally { + await server.stop(); + } + }); + + it("an unowned id the LOCAL engine holds → served by the local engine (local first)", async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const before = engine.requests.length; + try { + const res = await fetch(`${origin}/api/session/ses-here/message`, { headers: authed }); + expect(res.status).toBe(200); + expect(((await res.json()) as { marker: string }).marker).toBe("ENGINE-LIKE"); + expect(engine.requests.slice(before).map((r) => r.path)).toEqual(["/session/ses-here", "/api/session/ses-here/message"]); + } finally { + await server.stop(); + } + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // AC6 — production wiring: createAmicodeService on the OBSERVATION-ONLY path // wires the seam (a peer-owned read routes to the peer); the armed path stays @@ -351,4 +483,23 @@ describe("#1537 AC6 — production wiring (createAmicodeService observation-only await svc.stop(); } }); + + it("#1774: the observation boot wires the local-first probe — an id no one owns and the local engine lacks → 503 owner-unknown", async () => { + const engine = await startEngineLikeStub(new Set()); + const svc = createAmicodeService({ + password: PW, + engine: { password: "engine-mint", getUrl: () => engine.url }, + fleet: { hub: { getUrl: () => undefined }, observationOnly: true, fleetPeers: servingPeerProvider() }, + }); + const origin = (await svc.start()).toString().replace(/\/$/, ""); + try { + const res = await fetch(`${origin}/api/session/ses-ghost`, { headers: { Authorization: serverAuthHeader("engine-mint") } }); + expect(res.status).toBe(503); + expect(res.headers.get("retry-after")).toBe(String(OWNER_UNKNOWN_RETRY_AFTER_S)); + expect(((await res.json()) as { reason: string }).reason).toBe("owner-unknown"); + } finally { + await svc.stop(); + await engine.stop(); + } + }); }); diff --git a/packages/extension/test/amicode_service_observe_write_routing.test.ts b/packages/extension/test/amicode_service_observe_write_routing.test.ts index ecd481666..de59b6cc3 100644 --- a/packages/extension/test/amicode_service_observe_write_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_write_routing.test.ts @@ -28,7 +28,7 @@ import { join } from "node:path"; import * as http from "node:http"; import { AddressInfo } from "node:net"; -import { AmicodeServiceServer } from "../src/amicode_service/server"; +import { AmicodeServiceServer, OWNER_UNKNOWN_ERROR, OWNER_UNKNOWN_RETRY_AFTER_S } from "../src/amicode_service/server"; import { createAmicodeService } from "../src/amicode_service"; import { EngineProxy } from "../src/amicode_service/engine_proxy"; import { @@ -147,12 +147,16 @@ describe("#1542 — ObservationWriteRouter resolution order (GET/non-GET INVERTE expect(r.resolve("HEAD", "/api/session/ses-studio")).toBeUndefined(); }); - it("local-owned / unowned / not-a-known-peer → undefined (BYTE-IDENTICAL local fall-through)", () => { + it("local-owned / not-a-known-peer → undefined (BYTE-IDENTICAL local fall-through)", () => { expect(r.resolve("POST", "/session/ses-local")).toBeUndefined(); // local-owned - expect(r.resolve("POST", "/api/session/ses-unknown/message")).toBeUndefined(); // unowned expect(r.resolve("POST", "/api/session/ses-stranger/message")).toBeUndefined(); // owner not a serving peer }); + it("#1774: an UNOWNED session (no known owner) → the unknown-owner outcome: the caller tries local first", () => { + expect(r.resolve("POST", "/api/session/ses-unknown/message")).toEqual({ kind: "unknown-owner", sessionId: "ses-unknown" }); + expect(r.resolve("DELETE", "/session/ses-unknown")).toEqual({ kind: "unknown-owner", sessionId: "ses-unknown" }); + }); + it("/amicode/* and non-session paths → undefined (never a write target)", () => { expect(r.resolve("POST", "/amicode/fleet/status")).toBeUndefined(); expect(r.resolve("POST", "/amicode/profile")).toBeUndefined(); @@ -747,6 +751,247 @@ describe("#1542 AC4 — local-owned / unowned writes fall through byte-identical }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC1 — a refresh in which the owner peer errored (5xx) keeps its sessions +// routed to it: the authorized write is proxied to the peer, never run locally. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1774 AC1 — a peer-owned write keeps routing to the owner across a failed refresh", () => { + it("POST for a session whose owner's last refresh answered 502 → still the PEER; local NOT dialed", async () => { + const ownerMap = new SessionOwnerMap(); + const studioOwned = { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }; + ownerMap.update([studioOwned], [{ machineId: "studio", outcome: "ok" }]); + ownerMap.update([], [{ machineId: "studio", outcome: "error", reason: "fetch-failed" }]); // HTTP 502: an empty slice + const peer = (id: string): PeerTransport | undefined => (id === "studio" ? { getUrl: () => peerStub.url, token: "tok-studio" } : undefined); + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => localStub.url })); + server.attachObservationReadPlane(createObservationReadPlane({ ownerMap, localMachineId: "macbook", peer })); + server.attachObservationWritePlane( + createObservationWritePlane({ ownerMap, localMachineId: "macbook", peer, grantReader: (o) => (o === "studio" ? { scope: "control", state: "active" } : undefined) }), + ); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const localBefore = localStub.requests.length; + const peerBefore = peerStub.requests.length; + try { + const res = await fetch(`${origin}/api/session/ses-studio/message`, { + method: "POST", + headers: { ...authed, "content-type": "application/json" }, + body: JSON.stringify({ text: "hi" }), + }); + expect(((await res.json()) as { marker: string }).marker).toBe(peerStub.marker); + expect(peerStub.requests.slice(peerBefore).some((r) => r.method === "POST" && r.path === "/api/session/ses-studio/message")).toBe(true); + expect(localStub.requests.length).toBe(localBefore); + } finally { + await server.stop(); + } + }); +}); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC4 — a session-scoped WRITE for an id with NO known owner is tried on the +// LOCAL engine first (an existence probe — never the write itself); if the local +// engine does not have it, the answer is a retryable owner-unknown 503 and the +// write is NOT executed anywhere. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1774 AC4 — an unknown-owner write is tried locally; a local miss is a retryable owner-unknown 503", () => { + let engine: Stub; + beforeAll(async () => { + // engine-like: GET /session/{id} → 200 for a held session, the engine's 404 otherwise + const requests: Stub["requests"] = []; + const server = http.createServer((req, res) => { + const u = new URL(req.url ?? "/", "http://stub"); + requests.push({ method: req.method ?? "GET", path: u.pathname }); + res.setHeader("content-type", "application/json"); + const held = /^\/(api\/)?session\/ses-here(\/|$)/.test(u.pathname); + if (!held) return void res.writeHead(404).end('{"name":"NotFoundError"}'); + res.writeHead(200).end(JSON.stringify({ ok: true, marker: "ENGINE-LIKE" })); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + engine = { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + marker: "ENGINE-LIKE", + requests, + stop: () => new Promise((r) => server.close(() => r())), + }; + }); + afterAll(async () => { + await engine?.stop(); + }); + + function boot(): AmicodeServiceServer { + const ownerMap = new SessionOwnerMap(); + const noPeers = (): PeerTransport | undefined => undefined; + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => engine.url })); + server.attachObservationReadPlane(createObservationReadPlane({ ownerMap, localMachineId: "macbook", peer: noPeers, localEngineUrl: () => engine.url })); + server.attachObservationWritePlane( + createObservationWritePlane({ ownerMap, localMachineId: "macbook", peer: noPeers, grantReader: () => undefined, localEngineUrl: () => engine.url }), + ); + return server; + } + + for (const [method, path] of [ + ["POST", "/api/session/ses-ghost/message"], + ["DELETE", "/session/ses-ghost"], + ["PATCH", "/api/session/ses-ghost"], + ] as const) { + it(`${method} ${path} (not on the local engine) → 503 owner-unknown + Retry-After; the write is never executed`, async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const before = engine.requests.length; + try { + const res = await fetch(`${origin}${path}`, { + method, + headers: { ...authed, "content-type": "application/json" }, + ...(method === "DELETE" ? {} : { body: JSON.stringify({ text: "hi" }) }), + }); + expect(res.status).toBe(503); + expect(res.headers.get("retry-after")).toBe(String(OWNER_UNKNOWN_RETRY_AFTER_S)); + expect(await res.json()).toEqual({ ok: false, error: OWNER_UNKNOWN_ERROR, reason: "owner-unknown", session_id: "ses-ghost", retry_after_s: OWNER_UNKNOWN_RETRY_AFTER_S }); + // only the existence probe reached the local engine — never the write + expect(engine.requests.slice(before)).toEqual([{ method: "GET", path: "/session/ses-ghost" }]); + } finally { + await server.stop(); + } + }); + } + + it("an unowned id the LOCAL engine holds → the write runs on the local engine (local first)", async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const before = engine.requests.length; + try { + const res = await fetch(`${origin}/api/session/ses-here/message`, { + method: "POST", + headers: { ...authed, "content-type": "application/json" }, + body: JSON.stringify({ text: "hi" }), + }); + expect(res.status).toBe(200); + expect(((await res.json()) as { marker: string }).marker).toBe("ENGINE-LIKE"); + expect(engine.requests.slice(before)).toEqual([ + { method: "GET", path: "/session/ses-here" }, + { method: "POST", path: "/api/session/ses-here/message" }, + ]); + } finally { + await server.stop(); + } + }); +}); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC5 — the two "just created" bridges keep working under the unknown-owner +// outcome: a just-created LOCAL session (not yet in the owner map) is found by the +// local-first probe and served locally; a just-created REMOTE session's pending +// binding keeps routing to its peer — even across a refresh in which that peer +// failed — and never consults the local engine. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1774 AC5 — a just-created local session routes locally; the pending remote-create binding keeps working", () => { + /** An engine that CREATES sessions: POST /session mints ses-made-N and holds it. */ + async function startCreatingEngine() { + const held = new Set(); + const requests: Array<{ method: string; path: string }> = []; + let n = 0; + const server = http.createServer((req, res) => { + const u = new URL(req.url ?? "/", "http://stub"); + requests.push({ method: req.method ?? "GET", path: u.pathname }); + res.setHeader("content-type", "application/json"); + if (u.pathname === "/session" && req.method === "POST") { + const id = `ses-made-${++n}`; + held.add(id); + return void res.writeHead(200).end(JSON.stringify({ id })); + } + const id = /^\/(?:api\/)?session\/([^/]+)/.exec(u.pathname)?.[1]; + if (id !== undefined && !held.has(id)) return void res.writeHead(404).end('{"name":"NotFoundError"}'); + res.writeHead(200).end(JSON.stringify({ ok: true, marker: "LOCAL-CREATING", id })); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + return { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + requests, + stop: () => new Promise((r) => server.close(() => r())), + }; + } + + it("create locally, then read and write it at once (the owner map has not refreshed) → both served by the LOCAL engine", async () => { + const engine = await startCreatingEngine(); + const ownerMap = new SessionOwnerMap(); // the map knows nothing of the new session + const noPeers = (): PeerTransport | undefined => undefined; + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => engine.url })); + server.attachObservationReadPlane(createObservationReadPlane({ ownerMap, localMachineId: "macbook", peer: noPeers, localEngineUrl: () => engine.url })); + server.attachObservationWritePlane( + createObservationWritePlane({ ownerMap, localMachineId: "macbook", peer: noPeers, grantReader: () => undefined, localEngineUrl: () => engine.url }), + ); + const origin = (await server.start()).toString().replace(/\/$/, ""); + try { + const created = await fetch(`${origin}/session`, { method: "POST", headers: { ...authed, "content-type": "application/json" }, body: "{}" }); + const { id } = (await created.json()) as { id: string }; + expect(ownerMap.resolveOwner(id)).toBeUndefined(); + + const read = await fetch(`${origin}/session/${id}`, { headers: authed }); + expect(read.status).toBe(200); + expect(((await read.json()) as { marker: string }).marker).toBe("LOCAL-CREATING"); + + const write = await fetch(`${origin}/session/${id}/message`, { + method: "POST", + headers: { ...authed, "content-type": "application/json" }, + body: JSON.stringify({ text: "hi" }), + }); + expect(write.status).toBe(200); + expect(((await write.json()) as { marker: string }).marker).toBe("LOCAL-CREATING"); + expect(engine.requests.some((r) => r.method === "POST" && r.path === `/session/${id}/message`)).toBe(true); + } finally { + await server.stop(); + await engine.stop(); + } + }); + + it("a remote create's pending binding survives a refresh in which its peer FAILED: the follow-up read reaches the peer, local never consulted", async () => { + const engine = await startCreatingEngine(); + const peerSrv = http.createServer((req, res) => { + res.setHeader("content-type", "application/json"); + if (req.method === "POST") return void res.writeHead(200).end(JSON.stringify({ id: "ses-remote-pending" })); + res.writeHead(200).end(JSON.stringify({ ok: true, marker: "PEER-PENDING" })); + }); + await new Promise((r) => peerSrv.listen(0, "127.0.0.1", r)); + const peerUrl = `http://127.0.0.1:${(peerSrv.address() as AddressInfo).port}`; + const ownerMap = new SessionOwnerMap(); + const peer = (id: string): PeerTransport | undefined => (id === "studio" ? { getUrl: () => peerUrl, token: "tok-studio" } : undefined); + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => engine.url })); + server.attachObservationReadPlane(createObservationReadPlane({ ownerMap, localMachineId: "macbook", peer, localEngineUrl: () => engine.url })); + server.attachObservationWritePlane( + createObservationWritePlane({ + ownerMap, + localMachineId: "macbook", + peer, + grantReader: (o) => (o === "studio" ? { scope: "control", state: "active" } : undefined), + localEngineUrl: () => engine.url, + }), + ); + const origin = (await server.start()).toString().replace(/\/$/, ""); + try { + const created = await fetch(`${origin}/session`, { + method: "POST", + headers: { ...authed, [OWNER_ROUTING_HEADER]: "studio", "content-type": "application/json" }, + body: "{}", + }); + expect(((await created.json()) as { id: string }).id).toBe("ses-remote-pending"); + // the next owner-map refresh: studio's fetch timed out (an empty slice) + ownerMap.update([], [{ machineId: "studio", outcome: "timeout", reason: "fetch-failed" }]); + expect(ownerMap.resolveOwner("ses-remote-pending")).toBe("studio"); + + const read = await fetch(`${origin}/session/ses-remote-pending`, { headers: authed }); + expect(read.status).toBe(200); + expect(((await read.json()) as { marker: string }).marker).toBe("PEER-PENDING"); + expect(engine.requests).toEqual([]); // the local engine was never asked + } finally { + await server.stop(); + await new Promise((r) => peerSrv.close(() => r())); + await engine.stop(); + } + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // Byte-identity when UNATTACHED — a server with NO write plane consults nothing: // a POST for a peer-owned session falls through to the local engine (the diff --git a/packages/extension/test/engine_stream_fixtures.test.ts b/packages/extension/test/engine_stream_fixtures.test.ts new file mode 100644 index 000000000..f0c7f1d1f --- /dev/null +++ b/packages/extension/test/engine_stream_fixtures.test.ts @@ -0,0 +1,118 @@ +// engine_stream_fixtures.test.ts — #1771: the engine-shaped global-stream +// fixtures (test/fixtures/engine_stream) are SHARED test surface — the fan-in +// suites use them, and the loud-gap (#1773) and linear-splitting (#1779) slices +// build on them. This pins them to the wire the engine's `/global/event` route +// actually writes (overlay handlers/global.ts + bus/global.ts + the event-v2 +// bridge), so a fixture can never drift into a frame production never sends. +import { describe, it, expect } from "vitest"; +import { + fakeEngine, + engineFrame, + busEvent, + syncCopyOf, + sequencedEvent, + legacyEvent, + largeSummaryFrame, + chunkBytes, + CHUNK_64K, + TEN_MB, + LOCAL_EPOCH, + ENGINE_DIRECTORY, +} from "./fixtures/engine_stream/engine_frames"; + +/** Split one SSE frame into its field lines (the trailing blank line dropped). */ +function fields(frame: string): Array<[string, string]> { + expect(frame.endsWith("\n\n")).toBe(true); + return frame + .slice(0, -2) + .split("\n") + .map((l) => { + const i = l.indexOf(":"); + return [l.slice(0, i), l.slice(i + 1).replace(/^ /, "")] as [string, string]; + }); +} +const idOf = (frame: string) => fields(frame).find(([k]) => k === "id")?.[1]; +const dataOf = (frame: string) => JSON.parse(fields(frame).find(([k]) => k === "data")![1]); + +describe("#1771 — engine-shaped fixtures match the engine's global-stream wire", () => { + it("a frame is the SSE encoder's: optional `id:`, NO `event:` line (the name is \"message\"), one `data:` line, blank-line terminated", () => { + const f = engineFrame(busEvent({ id: "evt_1", type: "session.status", properties: { a: 1 } }), { id: `${LOCAL_EPOCH}:3` }); + expect(fields(f).map(([k]) => k)).toEqual(["id", "data"]); + expect(f).not.toContain("event:"); + // JSON.stringify(event) — the bridge's key order, payload {id, type, properties} + expect(fields(f)[1][1]).toBe( + `{"directory":"${ENGINE_DIRECTORY}","project":"prj_0fixture","payload":{"id":"evt_1","type":"session.status","properties":{"a":1}}}`, + ); + }); + + it("LEGACY (pre-#1776) engine: every frame is id-less — events, twins, heartbeats, connected", () => { + const e = fakeEngine({ legacy: true }); + const frames = [e.connected(), e.event("session.status", { s: 1 }), ...e.stored("message.updated", { m: 1 }, { aggregateID: "ses_1" }), e.heartbeat()]; + for (const f of frames) { + expect(idOf(f)).toBeUndefined(); + expect(fields(f).map(([k]) => k)).toEqual(["data"]); + } + expect(dataOf(frames.at(-1)!).payload).toMatchObject({ type: "server.heartbeat", properties: {} }); + expect(legacyEvent({ x: 1 })).not.toContain("id:"); + }); + + it("SEQUENCED (#1776) engine: bus events take strictly increasing `:`; control frames carry the CURRENT seq and consume none", () => { + let now = 1_700_000_000_000; + const e = fakeEngine({ epoch: LOCAL_EPOCH, now: () => now }); + const connected = e.connected(); + const a = e.event("session.status", { s: 1 }); + now += 10_000; + const hb = e.heartbeat(); + const b = e.event("session.status", { s: 2 }); + expect([idOf(connected), idOf(a), idOf(hb), idOf(b)]).toEqual([`${LOCAL_EPOCH}:0`, `${LOCAL_EPOCH}:1`, `${LOCAL_EPOCH}:1`, `${LOCAL_EPOCH}:2`]); + expect(dataOf(hb).payload).toMatchObject({ type: "server.heartbeat", properties: { time: 1_700_000_010_000, seq: 1 } }); + expect(dataOf(connected).payload).toMatchObject({ type: "server.connected", properties: {} }); + expect(e.seq).toBe(2); + expect(sequencedEvent(LOCAL_EPOCH, 9, { n: 9 })).toMatch(new RegExp(`^id: ${LOCAL_EPOCH}:9\\n`)); + }); + + it("a stored event is the plain + sync TWIN: the copy mirrors its source and shares its id (one seq)", () => { + const e = fakeEngine({ epoch: LOCAL_EPOCH }); + const [plain, sync] = e.stored("message.part.updated", { part: { id: "prt_1" } }, { aggregateID: "ses_9", durableSeq: 41 }); + expect(idOf(plain)).toBe(`${LOCAL_EPOCH}:1`); + expect(idOf(sync)).toBe(idOf(plain)); // filtering sync copies leaves no hole + const p = dataOf(plain); + const s = dataOf(sync); + expect(p.payload).toEqual({ id: p.payload.id, type: "message.part.updated", properties: { part: { id: "prt_1" } } }); + expect(s.directory).toBe(p.directory); + expect(s.payload).toEqual({ + type: "sync", + syncEvent: { id: p.payload.id, type: "message.part.updated.1", seq: 41, aggregateID: "ses_9", data: { part: { id: "prt_1" } } }, + id: p.payload.id, + }); + // the bus assigns the copy's payload.id at emit → it serializes AFTER syncEvent + expect(fields(sync)[1][1].indexOf('"syncEvent"')).toBeLessThan(fields(sync)[1][1].lastIndexOf('"id"')); + expect(syncCopyOf(busEvent({ id: "evt_x", type: "t", properties: {} }), { seq: 1, aggregateID: "a" }).payload.type).toBe("sync"); + // on a legacy engine the twin is two id-less frames + const [lp, ls] = fakeEngine({ legacy: true }).stored("message.updated", {}, { aggregateID: "ses_1" }); + expect([idOf(lp), idOf(ls)]).toEqual([undefined, undefined]); + }); + + it("the large frame: ≥ 10 MB of message.updated summary patch text, delivered in exact 64 KB byte chunks", () => { + const big = largeSummaryFrame(); + const bytes = Buffer.byteLength(big); + expect(bytes).toBeGreaterThanOrEqual(TEN_MB); + expect(bytes).toBeLessThan(TEN_MB + 1024 * 1024); + expect(big.length).toBeLessThan(bytes); // non-ASCII: bytes ≠ string length + const data = dataOf(big); + expect(data.payload.type).toBe("message.updated"); + expect(data.payload.properties.info.summary.diffs[0]).toEqual( + expect.objectContaining({ file: expect.any(String), patch: expect.any(String), additions: 1400, deletions: 0, status: "modified" }), + ); + expect(idOf(big)).toBeUndefined(); // legacy by default … + expect(idOf(largeSummaryFrame({ epoch: LOCAL_EPOCH, seq: 7 }))).toBe(`${LOCAL_EPOCH}:7`); // … or sequenced + + const chunks = chunkBytes(big); + expect(chunks.slice(0, -1).every((c) => c.length === CHUNK_64K)).toBe(true); + expect(chunks.at(-1)!.length).toBeGreaterThan(0); + expect(chunks.at(-1)!.length).toBeLessThanOrEqual(CHUNK_64K); + expect(Buffer.concat(chunks).equals(Buffer.from(big, "utf8"))).toBe(true); + // at least one boundary falls inside a multi-byte code point, as on the wire + expect(chunks.some((c) => (c[0] & 0xc0) === 0x80)).toBe(true); + }); +}); diff --git a/packages/extension/test/fixtures/engine_stream/engine_frames.ts b/packages/extension/test/fixtures/engine_stream/engine_frames.ts new file mode 100644 index 000000000..4fc39b93e --- /dev/null +++ b/packages/extension/test/fixtures/engine_stream/engine_frames.ts @@ -0,0 +1,426 @@ +// ============================================================================ +// #1771 — ENGINE-SHAPED global-stream fixtures (shared test surface; #1773 and +// #1779 build on these). +// +// The fan-in suites used to feed synthetic `event: message` / `id: 1` frames — +// a wire the engine never sends. These builders produce the frames the engine's +// `/global/event` route ACTUALLY writes, so the suites exercise production's +// wire format: +// +// - The route serializes each global event once with `JSON.stringify(event)` +// (`{directory, project, workspace?, payload}`, payload `{id, type, +// properties}`) and writes it through effect's SSE encoder: an `id:` line +// when the event has one, NO `event:` line (the event name is "message", +// which the encoder omits), one `data:` line, then the blank line. +// - LEGACY (pre-#1776) engines send NO `id:` line at all — every frame is +// id-less. This is what a peer running an old engine sends (#1773 needs it). +// - SEQUENCED (#1776) engines stamp `id: :`. Bus events take a +// strictly increasing seq; the per-connection control frames +// (`server.connected`, `server.heartbeat`) consume NO seq and carry the +// CURRENT one, so ids on one connection never decrease. +// - A stored (durable) event is emitted TWICE by the event-v2 bridge: the +// plain event, then its `sync` copy (`payload.type === "sync"`, the source +// under `syncEvent`). The copy shares its source's seq, so on a sequenced +// engine both frames carry the SAME id. +// - A per-turn summary (`message.updated` with `info.summary.diffs[].patch`) +// can be ~10 MB in ONE frame (measured 12.3 MB on the Studio), which the +// network delivers in 64 KB chunks. +// +// Pure builders + one stateful `fakeEngine` that sequences like the engine. +// Deterministic by construction: no clock or randomness unless injected. +// ============================================================================ +import * as http from "node:http"; +import type { AddressInfo } from "node:net"; + +/** A representative engine process epoch (`randomBytes(6).toString("base64url")`). */ +export const LOCAL_EPOCH = "bG9jYWwx"; +/** A second process epoch — a peer's engine. */ +export const PEER_EPOCH = "c3R1ZGlv"; + +export const ENGINE_DIRECTORY = "/Users/jj/harmoniqs/amicode"; +export const ENGINE_PROJECT = "prj_0fixture"; + +/** The payload of one global event (what the engine's bus carries). */ +export interface EnginePayload { + id?: string; + type: string; + properties?: Record; + syncEvent?: Record; + [key: string]: unknown; +} + +/** One global event as the engine serializes it (`JSON.stringify(event)`). */ +export interface EngineGlobalEvent { + directory?: string; + project?: string; + workspace?: string; + payload: EnginePayload; +} + +/** The #1776 global-stream frame id: `:`. */ +export function engineId(epoch: string, seq: number): string { + return `${epoch}:${seq}`; +} + +/** Serialize one global event as the engine's SSE encoder writes it: an `id:` + * line only when `id` is given (sequenced engines), no `event:` line, one + * `data:` line, the blank-line terminator. */ +export function engineFrame(event: EngineGlobalEvent, opts: { id?: string } = {}): string { + const idLine = opts.id !== undefined ? `id: ${opts.id}\n` : ""; + return `${idLine}data: ${JSON.stringify(event)}\n\n`; +} + +/** A bus event as the event-v2 bridge emits it (`{directory, project, + * workspace?, payload: {id, type, properties}}` — key order preserved). */ +export function busEvent(opts: { + id: string; + type: string; + properties: Record; + directory?: string; + project?: string; + workspace?: string; +}): EngineGlobalEvent { + return { + directory: opts.directory ?? ENGINE_DIRECTORY, + project: opts.project ?? ENGINE_PROJECT, + ...(opts.workspace !== undefined ? { workspace: opts.workspace } : {}), + payload: { id: opts.id, type: opts.type, properties: opts.properties }, + }; +} + +/** The `sync` copy the bridge emits right after a durable event. The bus + * assigns the copy's `payload.id` from `syncEvent.id` at emit (so the key lands + * AFTER `syncEvent`, exactly as the engine serializes it). */ +export function syncCopyOf( + source: EngineGlobalEvent, + durable: { seq: number; aggregateID: string; version?: number }, +): EngineGlobalEvent { + const id = source.payload.id; + return { + ...(source.directory !== undefined ? { directory: source.directory } : {}), + ...(source.project !== undefined ? { project: source.project } : {}), + ...(source.workspace !== undefined ? { workspace: source.workspace } : {}), + payload: { + type: "sync", + syncEvent: { + id, + type: `${source.payload.type}.${durable.version ?? 1}`, + seq: durable.seq, + aggregateID: durable.aggregateID, + data: source.payload.properties ?? {}, + }, + id, + }, + }; +} + +/** A per-connection control event (`server.connected` / `server.heartbeat` / + * `server.cursor.unservable`): payload only, no directory. */ +export function controlEvent(id: string, type: string, properties: Record = {}): EngineGlobalEvent { + return { payload: { id, type, properties } }; +} + +// ── a stateful engine that sequences like the real one ────────────────────── + +export interface FakeEngine { + /** The process epoch, or undefined for a LEGACY (id-less) engine. */ + readonly epoch: string | undefined; + /** The current sequence (the last bus event's), 0 before any. */ + readonly seq: number; + /** The SSE id a sequenced engine stamps for `seq`; undefined when legacy. */ + idAt(seq: number): string | undefined; + /** The `server.connected` frame (first frame of every connection). */ + connected(): string; + /** A heartbeat frame (legacy: empty properties; sequenced: `{time, seq}`). */ + heartbeat(): string; + /** One non-durable bus event frame (consumes one seq). */ + event(type: string, properties: Record): string; + /** A stored (durable) event: the plain frame then its `sync` twin — one seq, + * shared (on a sequenced engine both frames carry the same id). */ + stored(type: string, properties: Record, durable: { aggregateID: string; durableSeq?: number }): [plain: string, sync: string]; +} + +/** An engine stream: `{ legacy: true }` (pre-#1776, NO `id:` lines) or + * `{ epoch }` (#1776, `id: :`). `now` stamps heartbeat time. */ +export function fakeEngine( + opts: { legacy: true; now?: () => number; directory?: string } | { epoch: string; now?: () => number; directory?: string }, +): FakeEngine { + const epoch = "epoch" in opts ? opts.epoch : undefined; + const now = opts.now ?? (() => 0); + const directory = opts.directory ?? ENGINE_DIRECTORY; + let seq = 0; + let evt = 0; + const nextPayloadId = (): string => `evt_${String(++evt).padStart(6, "0")}`; + const idAt = (s: number): string | undefined => (epoch === undefined ? undefined : engineId(epoch, s)); + const frame = (event: EngineGlobalEvent, s: number): string => { + const id = idAt(s); + return engineFrame(event, id !== undefined ? { id } : {}); + }; + let durableSeq = 0; + return { + epoch, + get seq() { + return seq; + }, + idAt, + connected: () => frame(controlEvent(nextPayloadId(), "server.connected"), seq), + heartbeat: () => + frame(controlEvent(nextPayloadId(), "server.heartbeat", epoch === undefined ? {} : { time: now(), seq }), seq), + event: (type, properties) => { + const s = ++seq; + return frame(busEvent({ id: nextPayloadId(), type, properties, directory }), s); + }, + stored: (type, properties, durable) => { + const s = ++seq; + const source = busEvent({ id: nextPayloadId(), type, properties, directory }); + const twin = syncCopyOf(source, { seq: durable.durableSeq ?? ++durableSeq, aggregateID: durable.aggregateID }); + return [frame(source, s), frame(twin, s)]; + }, + }; +} + +// ── #1777: an engine that REPLAYS (the #1776 subscribe contract) ──────────── + +/** One subscription to a replaying engine's `/global/event`: the arm source the + * fan-in pumps, plus the test's handle to end it (a link drop, an engine exit). */ +export interface EngineSubscription { + /** The cursor this subscription asked for (`undefined` = live only). */ + readonly cursor: string | undefined; + /** Whether the engine answered the cursor with a reset. */ + readonly reset: boolean; + source: { + next(): Promise; + close(): void; + endReason(): "end" | "error" | undefined; + }; + /** End the subscription: `"end"` = a clean upstream end, `"error"` = a failure. */ + end(why?: "end" | "error"): void; +} + +export interface ReplayingEngine { + /** The process epoch. */ + readonly epoch: string; + /** The current sequence (the last bus event's), 0 before any. */ + readonly seq: number; + /** The SSE id this engine stamps for `seq`. */ + idAt(seq: number): string; + /** Emit one bus event: framed ONCE (`id: :`), retained for replay, + * and delivered to every open subscription. Returns the frame. */ + publish(type: string, properties: Record): string; + /** A heartbeat on every open subscription (`{time, seq}`, the CURRENT seq). */ + heartbeat(): void; + /** Subscribe like `GET /global/event[?lastEventID=…]`: a servable cursor gets + * `server.connected` AT the cursor, then every retained frame after it, then + * live frames; an unservable one (wrong epoch, evicted, ahead, malformed) gets + * the `server.cursor.unservable` reset frame first, then `server.connected` at + * the current seq, then live. No cursor = `server.connected` + live. */ + subscribe(cursor?: string): EngineSubscription; + /** Drop retained frames at or below `seq` (the replay window moved past them). */ + evictThrough(seq: number): void; + /** Every subscription opened so far, in order. */ + readonly subscriptions: readonly EngineSubscription[]; + /** The subscriptions still open. */ + live(): EngineSubscription[]; +} + +/** A #1776 engine with a replay buffer, in-process: the fan-in suites drive a + * re-open against it and see exactly what a real engine would send. */ +export function replayingEngine(opts: { epoch: string; now?: () => number; directory?: string }): ReplayingEngine { + const epoch = opts.epoch; + const now = opts.now ?? (() => 0); + const directory = opts.directory ?? ENGINE_DIRECTORY; + let seq = 0; + let floor = 0; // the highest seq no longer retained + let evt = 0; + const retained: Array<{ seq: number; frame: string }> = []; + const subscriptions: Array = []; + const nextPayloadId = (): string => `evt_${epoch}_${String(++evt).padStart(6, "0")}`; + const control = (at: number, type: string, properties: Record = {}): string => + engineFrame(controlEvent(nextPayloadId(), type, properties), { id: engineId(epoch, at) }); + + const open = (cursor: string | undefined) => { + const queue: string[] = []; + const waiters: Array<(v: string | null) => void> = []; + let ended = false; + let reason: "end" | "error" | undefined; + const end = (why: "end" | "error" = "end"): void => { + if (ended) return; + ended = true; + reason = why; + sub.open = false; + let w: ((v: string | null) => void) | undefined; + while ((w = waiters.shift())) w(null); + }; + const push = (frame: string): void => { + if (ended) return; + const w = waiters.shift(); + if (w) w(frame); + else queue.push(frame); + }; + const parsed = cursor === undefined ? undefined : /^([A-Za-z0-9_-]+):(\d+)$/.exec(cursor.trim()); + const servable = + cursor === undefined || + (parsed !== null && parsed !== undefined && parsed[1] === epoch && Number(parsed[2]) <= seq && Number(parsed[2]) >= floor); + const position = cursor !== undefined && servable && parsed ? Number(parsed[2]) : seq; + const sub = { + cursor, + reset: !servable, + open: true, + push, + end, + source: { + next(): Promise { + const f = queue.shift(); + if (f !== undefined) return Promise.resolve(f); + if (ended) return Promise.resolve(null); + return new Promise((resolve) => waiters.push(resolve)); + }, + close: () => end("end"), + endReason: () => reason, + }, + }; + if (!servable) push(control(seq, "server.cursor.unservable", { epoch, seq })); + push(control(position, "server.connected")); + if (servable) for (const r of retained) if (r.seq > position) push(r.frame); + subscriptions.push(sub); + return sub; + }; + + return { + epoch, + get seq() { + return seq; + }, + idAt: (s) => engineId(epoch, s), + publish(type, properties) { + const s = ++seq; + const frame = engineFrame(busEvent({ id: nextPayloadId(), type, properties, directory }), { id: engineId(epoch, s) }); + retained.push({ seq: s, frame }); + for (const sub of subscriptions) if (sub.open) sub.push(frame); + return frame; + }, + heartbeat() { + for (const sub of subscriptions) if (sub.open) sub.push(control(seq, "server.heartbeat", { time: now(), seq })); + }, + subscribe: (cursor) => open(cursor), + evictThrough(s) { + while (retained.length > 0 && retained[0].seq <= s) retained.shift(); + floor = Math.max(floor, s); + }, + get subscriptions() { + return subscriptions; + }, + live: () => subscriptions.filter((s) => s.open), + }; +} + +/** A one-off sequenced bus-event frame at an explicit `seq` — for suites that + * script many namespaces' positions directly. `properties` ride verbatim. */ +export function sequencedEvent(epoch: string, seq: number, properties: Record, type = "message.part.updated"): string { + return engineFrame(busEvent({ id: `evt_${epoch}_${seq}`, type, properties }), { id: engineId(epoch, seq) }); +} + +/** A one-off LEGACY (id-less) bus-event frame. */ +export function legacyEvent(properties: Record, type = "message.part.updated", payloadId = "evt_legacy"): string { + return engineFrame(busEvent({ id: payloadId, type, properties })); +} + +// ── the large frame + its chunked delivery ────────────────────────────────── + +export const TEN_MB = 10 * 1024 * 1024; +export const CHUNK_64K = 64 * 1024; + +/** A `message.updated` frame whose `info.summary.diffs[].patch` text brings the + * frame to at least `targetBytes` UTF-8 bytes (default 10 MB) — the shape of + * the measured multi-MB summary events. The patch text carries non-ASCII + * characters, so bytes ≠ string length and a 64 KB byte-chunk boundary can + * fall inside a code point (as it does on the wire). */ +export function largeSummaryFrame(opts: { targetBytes?: number; epoch?: string; seq?: number } = {}): string { + const target = opts.targetBytes ?? TEN_MB; + const lines: string[] = []; + for (let i = 0; i < 1400; i++) lines.push(`+ const row${i} = "résumé → ${"x".repeat(60)}";`); + const patch = `@@ -1,0 +1,1400 @@\n${lines.join("\n")}\n`; + // one diff entry's serialized size (patch JSON-escapes its newlines) + const perDiff = Buffer.byteLength(JSON.stringify(patch)) + 96; + const count = Math.max(1, Math.ceil(target / perDiff)); + const diffs = Array.from({ length: count }, (_, i) => ({ + file: `src/module_${i}.ts`, + patch, + additions: 1400, + deletions: 0, + status: "modified", + })); + const event = busEvent({ + id: "evt_summary", + type: "message.updated", + properties: { info: { id: "msg_summary", sessionID: "ses_large", role: "user", summary: { diffs } } }, + }); + const id = opts.epoch !== undefined ? engineId(opts.epoch, opts.seq ?? 1) : undefined; + return engineFrame(event, id !== undefined ? { id } : {}); +} + +/** Split text's UTF-8 BYTES into `chunkSize` pieces (the last may be short) — + * how the network delivers a large frame. Concatenating the chunks yields the + * exact original bytes. */ +export function chunkBytes(text: string, chunkSize = CHUNK_64K): Buffer[] { + const bytes = Buffer.from(text, "utf8"); + const out: Buffer[] = []; + for (let off = 0; off < bytes.length; off += chunkSize) out.push(bytes.subarray(off, off + chunkSize)); + return out; +} + +// ── a scripted engine `/global/event` server (127.0.0.1, port 0) ──────────── + +export interface EngineSseServer { + /** `http://127.0.0.1:` — dial THIS address (never a wildcard bind). */ + url: string; + /** Every request's path + query, in arrival order. */ + requests: string[]; + stop(): Promise; +} + +/** Serve `/global/event` from a script: `script(res)` writes the stream (whole + * frames, or raw chunks) and decides how it ends (`res.end()` = a clean + * upstream end; `res.destroy()` = an upstream error). Bound to 127.0.0.1 on an + * ephemeral port and dialed at its own address, so no foreign listener can + * share it. #1773: `opts.status` answers a non-200 instead (an auth failure, an + * overloaded peer) with a JSON content type; the script writes its body. */ +export function startEngineSseServer( + script: (res: http.ServerResponse, req: http.IncomingMessage) => void | Promise, + opts: { status?: number } = {}, +): Promise { + const requests: string[] = []; + const status = opts.status ?? 200; + const server = http.createServer((req, res) => { + requests.push(req.url ?? "/"); + res.writeHead( + status, + status === 200 + ? { "content-type": "text/event-stream", "cache-control": "no-cache, no-transform" } + : { "content-type": "application/json" }, + ); + res.flushHeaders(); + void script(res, req); + }); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + resolve({ + url: `http://127.0.0.1:${port}`, + requests, + stop: () => + new Promise((r) => { + server.close(() => r()); + server.closeAllConnections?.(); + }), + }); + }); + }); +} + +/** Write chunks one at a time, honoring backpressure (wait for `drain`). */ +export async function writeChunks(res: http.ServerResponse, chunks: Buffer[]): Promise { + for (const c of chunks) { + if (!res.write(c)) await new Promise((r) => res.once("drain", () => r())); + } +} diff --git a/packages/extension/test/fleet_health.test.ts b/packages/extension/test/fleet_health.test.ts index 4a0f6d8c8..f87320847 100644 --- a/packages/extension/test/fleet_health.test.ts +++ b/packages/extension/test/fleet_health.test.ts @@ -1,5 +1,8 @@ -import { describe, it, expect } from "vitest"; -import { checkFleetGuard, checkFleetSettings, checkFleetTunnel, checkFleetRole, fleetHealthReport, FLEET_GUARD_INSTALL } from "../src/fleet_health"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { checkFleetGuard, checkFleetSettings, checkFleetTunnel, checkFleetRole, checkPeerForwards, fleetHealthReport, FLEET_GUARD_INSTALL } from "../src/fleet_health"; import type { FleetTopologyState } from "../src/fleet_topology"; const REPO = "/repo/tools/fleet/amico-opencode-fleet-guard"; @@ -313,3 +316,144 @@ describe("fleet_health on linux — the guard backstop is wired cross-platform ( expect(r[0].detail).toMatch(/standalone/); }); }); + +// ── #1775: every per-peer SSH forward agent is checked the way the hub tunnel is ── +// One launchd agent per attached peer (co.harmoniqs.amico-peer-- +// forward), enumerated by label convention from an INJECTED LaunchAgents dir; +// the attached peers come from the reader peer-token store (loopback base_url +// = the forward's local end). Missing or stale keepalive values fail with the +// install.sh fix line — the tunnel check's exact shape. +describe("#1775 peer forward agents — every per-peer forward is checked like the hub tunnel", () => { + const TEMPLATE = readFileSync(join(__dirname, "..", "..", "..", "tools", "fleet", "co.harmoniqs.amico-peer-forward.plist"), "utf8"); + let root: string; + let launchAgentsDir: string; + let peerStoreFile: string; + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "fleet-health-peers-")); + launchAgentsDir = join(root, "LaunchAgents"); + mkdirSync(launchAgentsDir, { recursive: true }); + peerStoreFile = join(root, "fleet-peer-tokens-reader.json"); + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + /** The agent exactly as the installer renders it from the repo template. */ + const rendered = (id: string, localPort: number, remotePort = 4096, alias = id): string => + TEMPLATE.replace(/PEER_MACHINE_ID/g, id) + .replace(/PEER_LOCAL_PORT/g, String(localPort)) + .replace(/PEER_REMOTE_PORT/g, String(remotePort)) + .replace(/PEER_SSH_ALIAS/g, alias); + const placeAgent = (file: string, content: string): void => writeFileSync(join(launchAgentsDir, file), content); + const managedFile = (id: string): string => `co.harmoniqs.amico-peer-${id}-forward.plist`; + const attach = (peers: Record): void => + writeFileSync(peerStoreFile, JSON.stringify({ store_version: 1, peers: Object.fromEntries(Object.entries(peers).map(([id, u]) => [id, { base_url: u, token: "t" }])) })); + const run = (topology: FleetTopologyState = okServer) => + checkPeerForwards({ launchAgentsDir, peerStoreFile, platform: "darwin", topology }); + + it("one check per attached peer: an installer-rendered agent passes with the tunnel's ok shape (15/2 + TCPKeepAlive, the forward)", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + placeAgent(managedFile("jjs-mac-studio"), rendered("jjs-mac-studio", 45096)); + const checks = run(); + expect(checks).toHaveLength(1); + expect(checks[0].name).toBe("Fleet peer forward (jjs-mac-studio)"); + expect(checks[0].ok).toBe(true); + expect(checks[0].detail).toMatch(/ServerAlive 15\/2 \+ TCPKeepAlive/); + expect(checks[0].detail).toContain("127.0.0.1:45096:127.0.0.1:4096"); + }); + + it("an attached peer with NO forward agent fails, naming the agent, with the install.sh fix line", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const [c] = run(); + expect(c.ok).toBe(false); + expect(c.detail).toMatch(/missing/); + expect(c.detail).toContain(managedFile("jjs-mac-studio")); + expect(c.fix).toMatch(/bash tools\/fleet\/install\.sh/); + }); + + it("several agents, stale values: each is reported; each stale/missing keepalive value is named; the fix line is the tunnel's", () => { + attach({ + good: "http://127.0.0.1:45001", + stale: "http://127.0.0.1:45002", + "no-tcp": "http://127.0.0.1:45003", + "wrong-port": "http://127.0.0.1:45004", + }); + placeAgent(managedFile("good"), rendered("good", 45001)); + placeAgent(managedFile("stale"), rendered("stale", 45002).replace("ServerAliveInterval=15", "ServerAliveInterval=30").replace("ServerAliveCountMax=2", "ServerAliveCountMax=3")); + placeAgent(managedFile("no-tcp"), rendered("no-tcp", 45003).replace("-o\n\t\tTCPKeepAlive=yes\n", "").replace("-o\n\t\tExitOnForwardFailure=yes\n", "")); + placeAgent(managedFile("wrong-port"), rendered("wrong-port", 45999)); + const byName = Object.fromEntries(run().map((c) => [c.name, c])); + expect(Object.keys(byName).sort()).toEqual([ + "Fleet peer forward (good)", "Fleet peer forward (no-tcp)", "Fleet peer forward (stale)", "Fleet peer forward (wrong-port)", + ]); + expect(byName["Fleet peer forward (good)"].ok).toBe(true); + + const stale = byName["Fleet peer forward (stale)"]; + expect(stale.ok).toBe(false); + expect(stale.detail).toMatch(/ServerAliveInterval is 30 \(stale, should be 15\)/); + expect(stale.detail).toMatch(/ServerAliveCountMax is 3 \(stale, should be 2\)/); + expect(stale.fix).toBe("bash tools/fleet/install.sh (Fleet — Repair regenerates it: 15/2 + TCPKeepAlive)"); + + const noTcp = byName["Fleet peer forward (no-tcp)"]; + expect(noTcp.ok).toBe(false); + expect(noTcp.detail).toMatch(/TCPKeepAlive yes missing/); + expect(noTcp.detail).toMatch(/ExitOnForwardFailure yes missing/); + + const wrongPort = byName["Fleet peer forward (wrong-port)"]; + expect(wrongPort.ok).toBe(false); + expect(wrongPort.detail).toMatch(/LocalForward 127\.0\.0\.1:45004:127\.0\.0\.1:4096 missing/); + }); + + it("a hand-installed legacy forward is enumerated by label convention and its stale 30/3 flagged — the fix names the backup-keeping replace", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + // The real hand-installed shape: 30/3, no TCPKeepAlive, unbound -L. + placeAgent("co.harmoniqs.amico-studio-forward.plist", `Labelco.harmoniqs.amico-studio-forward +ProgramArguments/usr/bin/ssh-N-L45096:127.0.0.1:4096 +-oExitOnForwardFailure=yes-oServerAliveInterval=30 +-oServerAliveCountMax=3jjs-mac-studio`); + placeAgent("co.harmoniqs.amico-tunnel.plist", "not a forward agent — the hub tunnel has its own check"); + const byName = Object.fromEntries(run().map((c) => [c.name, c])); + expect(Object.keys(byName).sort()).toEqual([ + "Fleet peer forward (hand-installed co.harmoniqs.amico-studio-forward)", + "Fleet peer forward (jjs-mac-studio)", + ]); + const legacy = byName["Fleet peer forward (hand-installed co.harmoniqs.amico-studio-forward)"]; + expect(legacy.ok).toBe(false); + expect(legacy.detail).toMatch(/ServerAliveInterval is 30 \(stale, should be 15\)/); + expect(legacy.detail).toMatch(/ServerAliveCountMax is 3 \(stale, should be 2\)/); + expect(legacy.detail).toMatch(/TCPKeepAlive yes missing/); + expect(legacy.fix).toMatch(/bash tools\/fleet\/install\.sh .*replaces it .*backup/); + expect(byName["Fleet peer forward (jjs-mac-studio)"].ok).toBe(false); // the managed agent is not there yet + }); + + it("a peer reached over https (tailscale/direct) needs no forward — no check; no attached peers and no agents → no checks", () => { + attach({ far: "https://far.tailnet-1234.ts.net" }); + expect(run()).toEqual([]); + }); + + it("skips off darwin — launchd agents only (the supervised linux/WSL link is #1260)", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + expect(checkPeerForwards({ launchAgentsDir, peerStoreFile, platform: "linux", topology: okServer })).toEqual([]); + }); + + it("the aggregate report carries the peer forward checks on EVERY role (a peer may be server, standalone, or client); a broken projection still surfaces only the role fail", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const base = { + repoGuardPath: REPO, + configuredBinary: INSTALLED, + configuredPort: 4096, + plistContent: `ServerAliveInterval=15 ServerAliveCountMax=2 TCPKeepAlive=yes 127.0.0.1:4096:127.0.0.1:4096`, + read: () => guardContent, + isExecutable: () => true, + platform: "darwin", + launchAgentsDir, + peerStoreFile, + }; + for (const [topology, n] of [[okServer, 2], [okStandalone, 2], [absent, 2], [okClient, 5]] as const) { + const r = fleetHealthReport({ ...base, topology }); + expect(r, `role ${topology.kind}`).toHaveLength(n); + const peer = r.find((c) => c.name === "Fleet peer forward (jjs-mac-studio)"); + expect(peer?.ok).toBe(false); + expect(peer?.fix).toMatch(/install\.sh/); + } + expect(fleetHealthReport({ ...base, topology: broken })).toHaveLength(1); + }); +}); diff --git a/packages/extension/test/fleet_scripts_projection.test.ts b/packages/extension/test/fleet_scripts_projection.test.ts index 3f98c5a4a..0c661daec 100644 --- a/packages/extension/test/fleet_scripts_projection.test.ts +++ b/packages/extension/test/fleet_scripts_projection.test.ts @@ -13,9 +13,11 @@ // failure, cache untouched. import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { spawnSync } from "node:child_process"; -import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync, readdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { checkPeerForwards } from "../src/fleet_health"; +import { FLEET_MULTIPLEX_FLAG, fleetMultiplexEnabled } from "../src/amicode_service/server"; const REPO = join(__dirname, "..", "..", ".."); const GUARD = join(REPO, "tools", "fleet", "amico-opencode-fleet-guard"); @@ -148,10 +150,54 @@ function fakeFrozenBinary(): void { chmodSync(join(dir, "opencode"), 0o755); } +/** #1775: NO run in this file may reach the real OS service manager — the + * installer writes launch agents and (re)loads them, and a real `launchctl + * load` of a fabricated plist would start a real ssh agent on the test host. + * Every run gets TWO recording stubs: + * - the SEAM: `$AMICO_FLEET_LAUNCHCTL` → `/stubs/launchctl` (OFF the + * PATH), appending one argv line per call to `/launchctl.argv`; + * - the BACKSTOP: a `launchctl` + `systemctl` FIRST on the child PATH that + * append to `/launchctl-path.argv` — reached only if the installer + * ever bypasses its seam (a test asserts it never does). + * `launchctl getenv ` prints `/stubs/getenv-` when present. */ +function writeLoaderStubs(): void { + const stubs = join(tmp, "stubs"); + const bin = join(tmp, "fakebin"); + mkdirSync(stubs, { recursive: true }); + mkdirSync(bin, { recursive: true }); + const recorder = (log: string): string => [ + "#!/usr/bin/env bash", + `printf '%s\\n' "$*" >> "${log}"`, + `if [[ "\${1:-}" == "getenv" && -f "${stubs}/getenv-\${2:-}" ]]; then cat "${stubs}/getenv-\${2:-}"; fi`, + "exit 0", + "", + ].join("\n"); + writeFileSync(join(stubs, "launchctl"), recorder(join(tmp, "launchctl.argv"))); + chmodSync(join(stubs, "launchctl"), 0o755); + for (const name of ["launchctl", "systemctl"]) { + writeFileSync(join(bin, name), recorder(join(tmp, "launchctl-path.argv"))); + chmodSync(join(bin, name), 0o755); + } +} + +/** The seam stub's recorded argv lines (empty when it was never called). */ +function launchctlCalls(): string[] { + const log = join(tmp, "launchctl.argv"); + return existsSync(log) ? readFileSync(log, "utf8").split("\n").filter((l) => l !== "") : []; +} + +/** Calls that bypassed the seam and hit the PATH backstop (must stay empty). */ +function backstopCalls(): string[] { + const log = join(tmp, "launchctl-path.argv"); + return existsSync(log) ? readFileSync(log, "utf8").split("\n").filter((l) => l !== "") : []; +} + function runScript(script: string, args: string[], opts: { path?: string } = {}): { code: number; out: string } { + writeLoaderStubs(); const env: Record = { HOME: tmp, PATH: opts.path ?? `${join(tmp, "fakebin")}:/usr/bin:/bin`, + AMICO_FLEET_LAUNCHCTL: join(tmp, "stubs", "launchctl"), }; // The guard/installer must never find the REAL machine's amico or node-installed fleet state. delete env.AMICISSIMO_ROOT; @@ -373,6 +419,11 @@ describe("the installer writes the platform-correct settings path (#1261 AC4)", const r = runScript(INSTALL, [], installEnv()); expect(r.code).toBe(0); expect(existsSync(join(tmp, "Library", "Application Support", "Code", "User", "settings.json"))).toBe(true); + // #1775: the tunnel (re)load went through the launchctl SEAM — never the + // PATH's launchctl (on a real host that would start a real ssh agent). + const tunnel = join(tmp, "Library", "LaunchAgents", "co.harmoniqs.amico-tunnel.plist"); + expect(launchctlCalls()).toContain(`load ${tunnel}`); + expect(backstopCalls()).toEqual([]); }); it("--check on LINUX catches settings drift (the check twin runs cross-platform, never 'skipped (not darwin)')", () => { @@ -481,11 +532,411 @@ describe.skipIf(!existsSync(HUB_RUNNER_BUNDLE))("#1258 the installer provisions // ── both copies ship byte-identical (the VSIX's packaged copy is the installer users run) ── describe("the packaged fleet scripts stay in sync with the repo copies", () => { - it("guard + installer: packages/extension/tools/fleet copies are byte-identical", () => { - for (const f of ["amico-opencode-fleet-guard", "install.sh"]) { + it("guard + installer + launchd templates: packages/extension/tools/fleet copies are byte-identical", () => { + for (const f of ["amico-opencode-fleet-guard", "install.sh", "co.harmoniqs.amico-tunnel.plist", "co.harmoniqs.amico-peer-forward.plist"]) { const pkg = join(REPO, "packages", "extension", "tools", "fleet", f); expect(existsSync(pkg)).toBe(true); expect(readFileSync(pkg, "utf8")).toBe(readFileSync(join(REPO, "tools", "fleet", f), "utf8")); } }); }); + +// ── #1775: installer-managed per-peer SSH forwards ───────────────────────────── +// The SSH forward a machine uses to reach a PEER's hub was a hand-installed +// launch agent with stale 30/3 keepalives. The installer now generates one +// agent per attached peer — label from the peer's machine id, local port from +// the peer token store's loopback base URL, remote port = the fleet's canonical +// hub port, ssh destination = the peer's roster sshAlias (else its machine id). +// Every run is hermetic: temp HOME, the launchctl seam stub, the PATH backstop. +const PEER_STORE_REL = [".amico", "fleet-peer-tokens-reader.json"]; +const peerForwardPlist = (id: string): string => join(tmp, "Library", "LaunchAgents", `co.harmoniqs.amico-peer-${id}-forward.plist`); + +/** The reader peer-token store (#1438) — { store_version, peers: { id: { base_url, token } } }. */ +function writePeerStore(peers: Record): void { + mkdirSync(join(tmp, ".amico"), { recursive: true }); + const doc = { store_version: 1, peers: Object.fromEntries(Object.entries(peers).map(([id, baseUrl]) => [id, { base_url: baseUrl, token: `tok-${id}` }])) }; + writeFileSync(join(tmp, ...PEER_STORE_REL), JSON.stringify(doc, null, 2)); +} + +/** The fleet roster cache — the peer's sshAlias lives on its row. */ +function writeRoster(rows: Array<{ machine_id: string; sshAlias: string }>): void { + const dir = join(tmp, ".amico", "ops", "fleet"); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "roster.json"), JSON.stringify({ schema_version: 1, rows: rows.map((r) => ({ ...r, transport: "ssh", server_mode: "server" })) }, null, 2)); +} + +/** A launchd plist's Label + ProgramArguments, read straight from the XML. */ +function readAgent(file: string): { label: string; args: string[] } { + const xml = readFileSync(file, "utf8"); + const label = /Label<\/key>\s*([^<]*)<\/string>/.exec(xml)?.[1] ?? ""; + const array = /ProgramArguments<\/key>\s*([\s\S]*?)<\/array>/.exec(xml)?.[1] ?? ""; + const args = [...array.matchAll(/([^<]*)<\/string>/g)].map((m) => m[1]); + return { label, args }; +} + +/** The standard forward argv (fleet_transport's options) for one peer. */ +const standardForward = (localPort: number, remotePort: number, alias: string): string[] => [ + "/usr/bin/ssh", "-N", + "-o", "ExitOnForwardFailure=yes", + "-o", "ServerAliveInterval=15", + "-o", "ServerAliveCountMax=2", + "-o", "TCPKeepAlive=yes", + "-o", "BatchMode=yes", + "-L", `127.0.0.1:${localPort}:127.0.0.1:${remotePort}`, + alias, +]; + +describe("#1775 the installer generates one SSH forward agent per attached peer", () => { + const installEnv = (): { path: string } => ({ + path: `${join(tmp, "fakebin")}:${process.env.PATH ?? ""}`, + }); + const verbFor = (role: string): string => + role === "standalone" ? verbJson() : verbJson(role, { host: "hq", port: 4096, sshAlias: "hq" }); + + it.each(["server", "client", "standalone"])( + "a %s attached to a peer gets that peer's forward agent — 15 s keepalive, 2 misses, TCPKeepAlive, ExitOnForwardFailure, the store's local port; a re-run is idempotent", + (role) => { + fakeAmico({ code: 0, stdout: verbFor(role) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + writeRoster([{ machine_id: "jjs-mac-studio", sshAlias: "studio-ssh" }]); + + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + const plist = peerForwardPlist("jjs-mac-studio"); + expect(existsSync(plist), r.out).toBe(true); + const agent = readAgent(plist); + expect(agent.label).toBe("co.harmoniqs.amico-peer-jjs-mac-studio-forward"); + expect(agent.args).toEqual(standardForward(45096, 4096, "studio-ssh")); + expect(readFileSync(plist, "utf8")).toMatch(/KeepAlive<\/key>\s*/); + expect(launchctlCalls()).toContain(`load ${plist}`); + + // Re-run: byte-identical agent, no reload, still exactly one forward agent. + const first = readFileSync(plist, "utf8"); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const again = runScript(INSTALL, [], installEnv()); + expect(again.code, again.out).toBe(0); + expect(readFileSync(plist, "utf8")).toBe(first); + expect(launchctlCalls().filter((l) => l.includes(plist))).toEqual([]); + expect(again.out).toMatch(/peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward unchanged/); + expect(readdirSync(join(tmp, "Library", "LaunchAgents")).filter((f) => f.endsWith("-forward.plist"))).toEqual([ + "co.harmoniqs.amico-peer-jjs-mac-studio-forward.plist", + ]); + expect(backstopCalls()).toEqual([]); + }, + ); + + it("the ssh destination falls back to the peer's machine id when the roster has no row for it", () => { + fakeAmico({ code: 0, stdout: verbFor("server") }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(readAgent(peerForwardPlist("jjs-mac-studio")).args).toEqual(standardForward(45096, 4096, "jjs-mac-studio")); + }); + + it("one agent per peer: each loopback peer gets its own label + port; a peer reached over https (tailscale/direct) gets no ssh forward", () => { + fakeAmico({ code: 0, stdout: verbFor("server") }); + fakeUname("Darwin"); + writePeerStore({ + "jjs-mac-studio": "http://127.0.0.1:45096", + "lab-box": "http://127.0.0.1:45097", + "far-node": "https://far-node.tailnet-1234.ts.net", + }); + writeRoster([{ machine_id: "lab-box", sshAlias: "lab" }]); + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(readAgent(peerForwardPlist("jjs-mac-studio")).args).toEqual(standardForward(45096, 4096, "jjs-mac-studio")); + expect(readAgent(peerForwardPlist("lab-box")).args).toEqual(standardForward(45097, 4096, "lab")); + expect(existsSync(peerForwardPlist("far-node"))).toBe(false); + expect(r.out).toMatch(/far-node.*no SSH forward/); + }); + + it("Fleet — Repair runs the PACKAGED installer copy — it generates the same agent from the packaged template", () => { + fakeAmico({ code: 0, stdout: verbFor("server") }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const packaged = join(REPO, "packages", "extension", "tools", "fleet", "install.sh"); + const r = runScript(packaged, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(readAgent(peerForwardPlist("jjs-mac-studio")).args).toEqual(standardForward(45096, 4096, "jjs-mac-studio")); + expect(backstopCalls()).toEqual([]); + }); +}); + +/** A hand-installed forward in the legacy `co.harmoniqs.amico--forward` + * shape — byte-for-byte the agent found on a real fleet laptop (stale 30/3, + * unbound `-L :127.0.0.1:`, BatchMode + accept-new). */ +function legacyForwardPlist(name: string, localPort: number, host: string): string { + return ` + + + + Label + co.harmoniqs.amico-${name}-forward + ProgramArguments + + /usr/bin/ssh + -N + -L + ${localPort}:127.0.0.1:4096 + -o + ExitOnForwardFailure=yes + -o + ServerAliveInterval=30 + -o + ServerAliveCountMax=3 + -o + BatchMode=yes + -o + StrictHostKeyChecking=accept-new + ${host} + + RunAtLoad + + KeepAlive + + StandardOutPath + /tmp/amico-${name}-forward.log + StandardErrorPath + /tmp/amico-${name}-forward.log + ThrottleInterval + 10 + + +`; +} + +describe("#1775 a hand-installed forward for the same peer is replaced — a backup is kept, never a silent delete", () => { + const installEnv = (): { path: string } => ({ + path: `${join(tmp, "fakebin")}:${process.env.PATH ?? ""}`, + }); + const launchAgents = (): string => join(tmp, "Library", "LaunchAgents"); + const backupDir = (): string => join(tmp, ".amico", "ops", "fleet", "launchagents-backup"); + const placeLegacy = (name: string, localPort: number, host: string): { file: string; content: string } => { + mkdirSync(launchAgents(), { recursive: true }); + const file = join(launchAgents(), `co.harmoniqs.amico-${name}-forward.plist`); + const content = legacyForwardPlist(name, localPort, host); + writeFileSync(file, content); + return { file, content }; + }; + + it.each([ + ["it binds the same local port", 45096, "jjs-mac-studio"], + ["it targets the same ssh destination", 47001, "jjs-mac-studio"], + ])("recognized because %s: unloaded, moved byte-for-byte to the backup dir, and the managed agent takes its place; a re-run makes no second backup", (_why, legacyPort, legacyHost) => { + fakeAmico({ code: 0, stdout: verbJson("server", { host: "hq", port: 4096 }) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const legacy = placeLegacy("studio", legacyPort, legacyHost); + + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(existsSync(legacy.file)).toBe(false); + const backups = readdirSync(backupDir()); + expect(backups).toHaveLength(1); + expect(backups[0].startsWith("co.harmoniqs.amico-studio-forward.plist.")).toBe(true); + expect(readFileSync(join(backupDir(), backups[0]), "utf8")).toBe(legacy.content); + expect(r.out).toMatch(/replaced hand-installed peer forward .*co\.harmoniqs\.amico-studio-forward\.plist.*backup/); + + // The legacy job is unloaded BEFORE the managed one loads (they bind the same port). + const managed = peerForwardPlist("jjs-mac-studio"); + const calls = launchctlCalls(); + expect(calls).toContain(`unload ${legacy.file}`); + expect(calls.indexOf(`unload ${legacy.file}`)).toBeLessThan(calls.indexOf(`load ${managed}`)); + expect(readAgent(managed).args).toEqual(standardForward(45096, 4096, "jjs-mac-studio")); + + const again = runScript(INSTALL, [], installEnv()); + expect(again.code, again.out).toBe(0); + expect(readdirSync(backupDir())).toEqual(backups); + expect(backstopCalls()).toEqual([]); + }); + + it("a forward for a DIFFERENT peer (another port, another host) is left exactly as it was — never unloaded, never moved", () => { + fakeAmico({ code: 0, stdout: verbJson("server", { host: "hq", port: 4096 }) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const other = placeLegacy("lab", 47000, "lab-box"); + + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(readFileSync(other.file, "utf8")).toBe(other.content); + expect(launchctlCalls().filter((l) => l.includes(other.file))).toEqual([]); + expect(existsSync(backupDir())).toBe(false); + }); +}); + +// ── #1775: `install.sh --check` is the CLI twin of the peer-forward healthcheck ── +describe("#1775 --check holds every peer forward agent to the standard (the healthcheck's CLI twin)", () => { + const installEnv = (): { path: string } => ({ + path: `${join(tmp, "fakebin")}:${process.env.PATH ?? ""}`, + }); + const launchAgents = (): string => join(tmp, "Library", "LaunchAgents"); + const attachStudio = (): void => { + fakeAmico({ code: 0, stdout: verbJson("client", { host: "hq", port: 4096, sshAlias: "hq" }) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + }; + const listing = (): string => readdirSync(launchAgents()).map((f) => `${f}:${readFileSync(join(launchAgents(), f), "utf8").length}`).sort().join("\n"); + + it("after an install, --check passes end to end and names the peer forward; --check itself never calls launchctl and writes nothing", () => { + attachStudio(); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const before = listing(); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code, r.out).toBe(0); + expect(r.out).toMatch(/ok peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward \(15\/2 \+ TCPKeepAlive, 127\.0\.0\.1:45096 → jjs-mac-studio:4096\)/); + expect(r.out).toMatch(/fleet check: all ok/); + expect(launchctlCalls()).toEqual([]); + expect(backstopCalls()).toEqual([]); + expect(listing()).toBe(before); + }); + + it("an attached peer with no forward agent fails --check with the fix line", () => { + attachStudio(); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + rmSync(join(launchAgents(), "co.harmoniqs.amico-peer-jjs-mac-studio-forward.plist")); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code).toBe(1); + expect(r.out).toMatch(/FAIL peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward missing at .*\(run: bash tools\/fleet\/install\.sh\)/); + }); + + it("stale keepalive values fail --check, each value named (the tunnel check's lines)", () => { + attachStudio(); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + const agent = join(launchAgents(), "co.harmoniqs.amico-peer-jjs-mac-studio-forward.plist"); + writeFileSync(agent, readFileSync(agent, "utf8").replace("ServerAliveInterval=15", "ServerAliveInterval=30").replace("ServerAliveCountMax=2", "ServerAliveCountMax=3")); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code).toBe(1); + expect(r.out).toMatch(/FAIL peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward ServerAliveInterval 15 missing/); + expect(r.out).toMatch(/FAIL peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward ServerAliveCountMax 2 missing/); + expect(r.out).not.toMatch(/fleet check: all ok/); + }); + + it("a hand-installed forward for an attached peer fails --check — Fleet — Repair replaces it, keeping a backup", () => { + attachStudio(); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + writeFileSync(join(launchAgents(), "co.harmoniqs.amico-studio-forward.plist"), legacyForwardPlist("studio", 45096, "jjs-mac-studio")); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code).toBe(1); + expect(r.out).toMatch(/FAIL hand-installed peer forward .*co\.harmoniqs\.amico-studio-forward\.plist for jjs-mac-studio .*replaces it.*backup/); + // a check only reports: the legacy agent is neither unloaded nor moved + expect(launchctlCalls()).toEqual([]); + expect(existsSync(join(launchAgents(), "co.harmoniqs.amico-studio-forward.plist"))).toBe(true); + }); + + it("the installer and the healthcheck agree: every agent the installer renders passes checkPeerForwards", () => { + fakeAmico({ code: 0, stdout: verbJson("server", { host: "hq", port: 4096 }) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096", "lab-box": "http://127.0.0.1:45097" }); + writeRoster([{ machine_id: "lab-box", sshAlias: "lab" }]); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + const checks = checkPeerForwards({ + launchAgentsDir: launchAgents(), + peerStoreFile: join(tmp, ...PEER_STORE_REL), + platform: "darwin", + topology: { + kind: "ok", role: "server", canonical: { host: "hq", port: 4096 }, mode: "fleet", posture: "ok", + freshness: {}, provenanceSource: "fleet.json", projection: { schema_version: 1, contract_version: 1, sections: {} }, + }, + }); + expect(checks.map((c) => [c.name, c.ok])).toEqual([ + ["Fleet peer forward (jjs-mac-studio)", true], + ["Fleet peer forward (lab-box)", true], + ]); + }); +}); + +// ── #1775: the stray session-wide multiplex-flag agent is retired ────────────── +// `co.harmoniqs.amico-fleet-flag` ran `launchctl setenv AMICO_FLEET_MULTIPLEX 1` +// at login, leaking the flag into every GUI process — though the flag already +// defaults ON (#1448). The installer unloads + removes it and unsets the +// variable; asserted on the STUB's argv, never the real launchctl. +const FLAG_AGENT = ` + + + + Label + co.harmoniqs.amico-fleet-flag + ProgramArguments + + /bin/launchctl + setenv + AMICO_FLEET_MULTIPLEX + 1 + + RunAtLoad + + + +`; + +describe("#1775 the installer retires the session-wide multiplex-flag agent", () => { + const installEnv = (): { path: string } => ({ + path: `${join(tmp, "fakebin")}:${process.env.PATH ?? ""}`, + }); + const flagPlist = (): string => join(tmp, "Library", "LaunchAgents", "co.harmoniqs.amico-fleet-flag.plist"); + const placeFlagAgent = (): void => { + mkdirSync(join(tmp, "Library", "LaunchAgents"), { recursive: true }); + writeFileSync(flagPlist(), FLAG_AGENT); + }; + /** What the stubbed `launchctl getenv AMICO_FLEET_MULTIPLEX` prints (the session value). */ + const sessionFlag = (value: string | null): void => { + mkdirSync(join(tmp, "stubs"), { recursive: true }); + const f = join(tmp, "stubs", "getenv-AMICO_FLEET_MULTIPLEX"); + if (value === null) rmSync(f, { force: true }); + else writeFileSync(f, `${value}\n`); + }; + const verbFor = (role: string): string => + role === "standalone" ? verbJson() : verbJson(role, { host: "hq", port: 4096, sshAlias: "hq" }); + + it.each(["server", "client", "standalone"])("on a %s: the flag agent is unloaded + removed and the variable unset; a re-run is a no-op; the flag still defaults ON", (role) => { + fakeAmico({ code: 0, stdout: verbFor(role) }); + fakeUname("Darwin"); + placeFlagAgent(); + sessionFlag("1"); + + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(existsSync(flagPlist())).toBe(false); + expect(launchctlCalls()).toContain(`unload ${flagPlist()}`); + expect(launchctlCalls()).toContain("unsetenv AMICO_FLEET_MULTIPLEX"); + expect(r.out).toMatch(/removed the session-wide multiplex-flag agent/); + expect(backstopCalls()).toEqual([]); + + // Fleet behavior is unchanged: with the variable gone the multiplexer is ON by default. + delete process.env[FLEET_MULTIPLEX_FLAG]; + expect(fleetMultiplexEnabled()).toBe(true); + + // Re-run after the unset: nothing left to retire — no unload, no unsetenv. + sessionFlag(null); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const again = runScript(INSTALL, [], installEnv()); + expect(again.code, again.out).toBe(0); + expect(launchctlCalls().filter((l) => l.includes("fleet-flag") || l.startsWith("unsetenv"))).toEqual([]); + }); + + it("a deliberate session opt-out (AMICO_FLEET_MULTIPLEX=0, no flag agent) is left alone — only the agent's leaked value is unset", () => { + fakeAmico({ code: 0, stdout: verbFor("server") }); + fakeUname("Darwin"); + sessionFlag("0"); + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(launchctlCalls().filter((l) => l.startsWith("unsetenv"))).toEqual([]); + }); + + it("--check fails on a present flag agent with the fix line — and never calls launchctl", () => { + fakeAmico({ code: 0, stdout: verbFor("client") }); + fakeUname("Darwin"); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + placeFlagAgent(); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code).toBe(1); + expect(r.out).toMatch(/FAIL session-wide multiplex-flag agent present at .*co\.harmoniqs\.amico-fleet-flag\.plist.*run: bash tools\/fleet\/install\.sh/); + expect(existsSync(flagPlist())).toBe(true); + expect(launchctlCalls()).toEqual([]); + }); +}); diff --git a/packages/extension/test/merged_projection_archived.test.ts b/packages/extension/test/merged_projection_archived.test.ts index 67d91b1a8..60e9d8f38 100644 --- a/packages/extension/test/merged_projection_archived.test.ts +++ b/packages/extension/test/merged_projection_archived.test.ts @@ -128,3 +128,86 @@ describe("#1647 S2 — buildOwnerRoutingProjection unions active + archived for expect(arch.amicode_control).toEqual({ controlState: "interactive", reason: null, eligibility: "none" }); }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1771 — the owner-routing projection reports EACH PEER's refresh outcome (the +// OwnerMapFeed logs it). Never throws: every failure is a classified record. +// The served projection shape is unchanged — this rides only the routing union. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1771 — buildOwnerRoutingProjection classifies each peer's refresh", () => { + const json = (v: unknown, status = 200) => new Response(JSON.stringify(v), { status }); + + it("ok / unauthorized / timeout / error — with each peer's entry count (active ∪ archived)", async () => { + const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/global/health")) return json({ version: "t" }); + if (url.startsWith("http://ok")) return json(/archived=true/.test(url) ? [{ id: "ses-ok-arch" }] : [{ id: "ses-ok-1" }]); + if (url.startsWith("http://denied")) return new Response("no", { status: 401 }); + if (url.startsWith("http://broken")) return new Response("bad gateway", { status: 502 }); + if (url.startsWith("http://slow")) { + // a peer that never answers: the projection's own AbortSignal.timeout fires + return new Promise((_, reject) => init?.signal?.addEventListener("abort", () => reject(init.signal!.reason))); + } + return json([]); // local + }) as unknown as typeof fetch; + const p = await buildOwnerRoutingProjection({ + localMachineId: "local-mac", + local: { getUrl: () => "http://local" }, + peers: [ + { machineId: "ok", getUrl: () => "http://ok", token: "t" }, + { machineId: "denied", getUrl: () => "http://denied", token: "t" }, + { machineId: "slow", getUrl: () => "http://slow", token: "t" }, + { machineId: "broken", getUrl: () => "http://broken", token: "t" }, + { machineId: "gone", getUrl: () => undefined, token: "t" }, + ], + rosterLookup: () => undefined, + fetchImpl, + timeoutMs: 1, + }); + const summary = p.peers.map(({ machineId, outcome, entries, reason, detail }) => ({ machineId, outcome, entries, reason, detail })); + expect(summary).toEqual([ + { machineId: "ok", outcome: "ok", entries: 2, reason: undefined, detail: undefined }, + { machineId: "denied", outcome: "unauthorized", entries: 0, reason: "unauthorized", detail: undefined }, + { machineId: "slow", outcome: "timeout", entries: 0, reason: "fetch-failed", detail: expect.stringContaining("timeout") }, + { machineId: "broken", outcome: "error", entries: 0, reason: "fetch-failed", detail: "HTTP 502" }, + { machineId: "gone", outcome: "error", entries: 0, reason: "no-upstream", detail: undefined }, + ]); + // the routing union itself is unchanged: only the healthy peer's sessions land + expect(p.sessions.map((s) => s.id).sort()).toEqual(["ses-ok-1", "ses-ok-arch"]); + }); + + it("each peer's duration is measured on the injected clock", async () => { + let t = 1_000; + const held: Array<() => void> = []; + // plain promise-backed responses: every read is a microtask, nothing waits + // on a stream or a timer + const fake = (v: unknown) => ({ ok: true, status: 200, json: async () => v }) as unknown as Response; + const fetchImpl = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/global/health")) return fake({ version: "t" }); + if (url.startsWith("http://slowish")) return new Promise((resolve) => held.push(() => resolve(fake([{ id: "ses-s" }])))); + return fake([]); + }) as unknown as typeof fetch; + const pending = buildOwnerRoutingProjection({ + localMachineId: "local-mac", + local: { getUrl: () => "http://local" }, + peers: [ + { machineId: "quick", getUrl: () => "http://quick", token: "t" }, + { machineId: "slowish", getUrl: () => "http://slowish", token: "t" }, + ], + rosterLookup: () => undefined, + fetchImpl, + now: () => t, + }); + // Every fetch but the held ones resolves at once, so one macrotask turn + // drains them all (microtasks only) — the quick peer is done, the slow + // peer's two reads (active + archived) are in flight. + await new Promise((r) => setImmediate(r)); + expect(held).toHaveLength(2); + t = 2_500; // 1.5 s pass while the slow peer answers + held.forEach((release) => release()); + const p = await pending; + expect(p.peers.find((r) => r.machineId === "quick")).toMatchObject({ outcome: "ok", durationMs: 0, entries: 0 }); + expect(p.peers.find((r) => r.machineId === "slowish")).toMatchObject({ outcome: "ok", durationMs: 1_500, entries: 1 }); + }); +}); diff --git a/packages/extension/test/slow/engine_global_stream_scratch.test.ts b/packages/extension/test/slow/engine_global_stream_scratch.test.ts new file mode 100644 index 000000000..5a31b337f --- /dev/null +++ b/packages/extension/test/slow/engine_global_stream_scratch.test.ts @@ -0,0 +1,173 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import http from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// ============================================================================ +// Scratch-engine check of the sequenced, replayable global stream (#1776), +// against the COMPILED engine binary — the runtime (Bun's node:http shim) +// whose client-abort behaviour the in-process suite cannot vouch for alone. +// +// Opt-in: SCRATCH_ENGINE_E2E=1, after `pnpm --filter amicode run build:binary` +// (not an AMICODE_* name: test/setup_hermetic_env.ts deletes those before this +// module's gate is read). It spawns the vendored binary of THIS checkout on a spare port +// (default 4195; never a hub port) with every state dir and the session DB in +// one temp dir, and kills it afterwards (SCRATCH_ENGINE_BIN points it at another +// binary, e.g. a baseline build). Live hubs are never touched. +// ============================================================================ + +const ENABLED = process.env.SCRATCH_ENGINE_E2E === "1"; +const BIN = + process.env.SCRATCH_ENGINE_BIN ?? join(__dirname, "..", "..", "vendor", "opencode", `${process.platform}-${process.arch}`, "opencode"); +const PORT = Number(process.env.SCRATCH_ENGINE_PORT ?? 4195); +const ID = /^([A-Za-z0-9_-]+):(\d+)$/; + +type Frame = { id: string; type: string; data: string }; + +function seqOf(frame: Frame) { + const match = ID.exec(frame.id); + if (!match) throw new Error(`frame id ${JSON.stringify(frame.id)} is not :`); + return { epoch: match[1], seq: Number(match[2]) }; +} + +function stream(headers: Record = {}) { + const frames: Frame[] = []; + let buffer = ""; + let req!: http.ClientRequest; + const opened = new Promise((resolve, reject) => { + req = http.get({ host: "127.0.0.1", port: PORT, path: "/global/event", headers: { accept: "text/event-stream", ...headers } }, (res) => { + res.setEncoding("utf8"); + res.on("data", (chunk: string) => { + buffer += chunk; + let end = buffer.indexOf("\n\n"); + while (end !== -1) { + const block = buffer.slice(0, end); + buffer = buffer.slice(end + 2); + const id = /^id: (.*)$/m.exec(block)?.[1] ?? ""; + const data = block.split("\n").filter((l) => l.startsWith("data: ")).map((l) => l.slice(6)).join("\n"); + if (data) frames.push({ id, data, type: JSON.parse(data).payload?.type }); + end = buffer.indexOf("\n\n"); + } + }); + res.on("error", () => {}); + resolve(); + }); + req.on("error", (e: NodeJS.ErrnoException) => (e.code === "ECONNRESET" ? undefined : reject(e))); + }); + return { frames, opened, abort: () => req.destroy() }; +} + +function post(path: string): Promise { + return new Promise((resolve, reject) => { + const req = http.request({ host: "127.0.0.1", port: PORT, path, method: "POST", headers: { "content-type": "application/json" } }, (res) => { + res.resume(); + resolve(res.statusCode ?? 0); + }); + req.on("error", reject); + req.end("{}"); + }); +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +async function until(probe: () => T | undefined, what: string, ms = 10_000): Promise { + const deadline = Date.now() + ms; + for (;;) { + const value = probe(); + if (value !== undefined && value !== false) return value; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await sleep(25); + } +} + +describe.skipIf(!ENABLED)("scratch engine: sequenced, replayable global stream (#1776)", () => { + let engine: ChildProcess; + let scratch: string; + let log = ""; + const lines = (message: string) => log.split("\n").filter((l) => l.includes(`message="${message}"`)); + + beforeAll(async () => { + if (!(PORT >= 4190 && PORT <= 4199)) throw new Error(`scratch port ${PORT} outside 4190-4199`); + if (!existsSync(BIN)) throw new Error(`no compiled engine at ${BIN} — run build:binary first`); + scratch = mkdtempSync(join(tmpdir(), "amicode-scratch-engine-")); + const proj = join(scratch, "proj"); + mkdirSync(proj); + mkdirSync(join(scratch, "home")); + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) if (v !== undefined && !/^(OPENCODE|AMICO)/.test(k)) env[k] = v; + Object.assign(env, { + HOME: join(scratch, "home"), + XDG_DATA_HOME: join(scratch, "data"), + XDG_CONFIG_HOME: join(scratch, "config"), + XDG_STATE_HOME: join(scratch, "state"), + XDG_CACHE_HOME: join(scratch, "cache"), + OPENCODE_DB: join(scratch, "db.sqlite"), + }); + engine = spawn(BIN, ["serve", "--port", String(PORT), "--hostname", "127.0.0.1", "--print-logs", "--log-level", "INFO", "--pure"], { + cwd: proj, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + engine.stdout?.on("data", (d: Buffer) => (log += d)); + engine.stderr?.on("data", (d: Buffer) => (log += d)); + await until(() => log.includes(`listening on http://127.0.0.1:${PORT}`) || undefined, "engine listen", 60_000); + }, 90_000); + + afterAll(async () => { + engine?.kill("SIGTERM"); + await sleep(1_000); + if (engine && engine.exitCode === null && engine.signalCode === null) engine.kill("SIGKILL"); + if (scratch) rmSync(scratch, { recursive: true, force: true }); + }); + + it("replays exactly the frames after a Last-Event-ID cursor, then continues live", async () => { + const a = stream(); + await a.opened; + const connected = await until(() => a.frames[0], "connected frame"); + expect(connected.type).toBe("server.connected"); + const { epoch } = seqOf(connected); + + expect(await post("/session")).toBe(200); + await until(() => a.frames.length >= 3 || undefined, "session events on the first connection"); + await sleep(500); + const [first, ...afterCursor] = a.frames.slice(1); + a.abort(); + + expect(await post("/session")).toBe(200); // published while no one is connected + const b = stream({ "last-event-id": first.id }); + await b.opened; + await until(() => b.frames.length >= afterCursor.length + 3 || undefined, "replay and gap frames"); + await sleep(500); + + expect(b.frames[0].type).toBe("server.connected"); + expect(b.frames[0].id).toBe(first.id); + const replayed = b.frames.slice(1); + // the frames the first connection saw after the cursor, byte for byte, then the gap + expect(replayed.slice(0, afterCursor.length)).toEqual(afterCursor); + const gap = replayed.slice(afterCursor.length).filter((f) => f.type !== "server.heartbeat"); + expect(gap.map((f) => f.type)).toContain("session.created"); + // one epoch; ids never decrease and bus events never skip a sequence + let previous = seqOf(first).seq; + for (const frame of replayed) { + const { epoch: e, seq } = seqOf(frame); + expect(e).toBe(epoch); + expect(seq - previous === 0 || seq - previous === 1).toBe(true); + previous = seq; + } + b.abort(); + }, 30_000); + + it("a client that aborts mid-stream releases its subscription, and the release is logged", async () => { + const clients = Array.from({ length: 5 }, () => stream()); + await Promise.all(clients.map((c) => c.opened)); + await until(() => clients.every((c) => c.frames.length > 0) || undefined, "connected frames"); + for (const c of clients) c.abort(); + await until( + () => lines("global event disconnected").length === lines("global event connected").length || undefined, + "one disconnect per connect", + ); + expect(lines("global event connected").length).toBeGreaterThanOrEqual(5); + expect(lines("global event disconnected").at(-1)).toContain("subscribers=0"); + }, 30_000); +}); diff --git a/packages/extension/test/slow/fleet_link_kill_resume_e2e.test.ts b/packages/extension/test/slow/fleet_link_kill_resume_e2e.test.ts new file mode 100644 index 000000000..3270947a3 --- /dev/null +++ b/packages/extension/test/slow/fleet_link_kill_resume_e2e.test.ts @@ -0,0 +1,344 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import http from "node:http"; +import net from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { SseFanInDriver } from "../../src/amicode_service/sse_fanin_driver"; +import { SessionOwnerMap } from "../../src/amicode_service/session_multiplexer"; +import { FanInCensus } from "../../src/amicode_service/fanin_census"; +import { parseCompositeCursor } from "../../src/amicode_service/sse_composite_cursor"; +// The CLIENT's own resume cursor + contiguity check (the app overlay's pure +// module the webview's stream loop runs) — the observer applies every frame +// through it, exactly as the app does. +import { createStreamCursor } from "../../../app-bundle/overlay/packages/app/src/context/stream-cursor"; + +// ============================================================================ +// #1777 AC6 — TWO-PEER FLEET E2E WITH A LINK KILL, against two REAL compiled +// engines (the #1776 sequenced, replaying global stream). +// +// engine A (:4191) ── the observer's local engine ──────────┐ +// engine B (:4192) ── the owner peer ── link (:4193, TCP) ──┤ fan-in (in-test, +// │ SseFanInDriver + +// observer (SSE client, the app's │ the real reader) +// stream-cursor) ◀── /global/event ──────┘ +// +// A turn runs on B (a shell tool streaming output: user message, assistant +// message, a running tool part updated per chunk, completion). MID-TURN the +// link is KILLED — every proxied socket destroyed and the port closed (a dead +// SSH forward: connection refused) — and later RESTORED. The fan-in's peer arm +// fails, its re-opens are refused while the link is down, then one resumes from +// the last delivered engine id and B replays what the observer missed. After +// the turn the observer's transcript must equal a fresh load from B, with no +// gap frame and no resync owed (no hole, no reset). +// +// Opt-in: SCRATCH_ENGINE_E2E=1, after `pnpm --filter amicode run build:binary` +// (not an AMICODE_* name: test/setup_hermetic_env.ts deletes those before this +// module's gate is read). Both engines run the vendored binary of THIS checkout +// on spare ports in 4190–4199 (never a hub port), each with every state dir and +// its session DB in its own temp dir, and are killed afterwards. Live hubs and +// the live peer link are never touched. +// ============================================================================ + +const ENABLED = process.env.SCRATCH_ENGINE_E2E === "1"; +const BIN = + process.env.SCRATCH_ENGINE_BIN ?? join(__dirname, "..", "..", "vendor", "opencode", `${process.platform}-${process.arch}`, "opencode"); +const PORT_A = Number(process.env.SCRATCH_ENGINE_PORT_A ?? 4191); +const PORT_B = Number(process.env.SCRATCH_ENGINE_PORT_B ?? 4192); +const PORT_LINK = Number(process.env.SCRATCH_LINK_PORT ?? 4193); + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +async function until(probe: () => T | undefined | false, what: string, ms = 15_000): Promise { + const deadline = Date.now() + ms; + for (;;) { + const value = probe(); + if (value !== undefined && value !== false) return value; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await sleep(20); + } +} + +interface Engine { + proc: ChildProcess; + dir: string; + url: string; + log(): string; +} + +async function startEngine(port: number, name: string): Promise { + if (!(port >= 4190 && port <= 4199)) throw new Error(`scratch port ${port} outside 4190-4199`); + const dir = mkdtempSync(join(tmpdir(), `amicode-scratch-${name}-`)); + const proj = join(dir, "proj"); + mkdirSync(proj); + mkdirSync(join(dir, "home")); + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) if (v !== undefined && !/^(OPENCODE|AMICO)/.test(k)) env[k] = v; + Object.assign(env, { + HOME: join(dir, "home"), + XDG_DATA_HOME: join(dir, "data"), + XDG_CONFIG_HOME: join(dir, "config"), + XDG_STATE_HOME: join(dir, "state"), + XDG_CACHE_HOME: join(dir, "cache"), + OPENCODE_DB: join(dir, "db.sqlite"), + }); + let log = ""; + const proc = spawn(BIN, ["serve", "--port", String(port), "--hostname", "127.0.0.1", "--print-logs", "--log-level", "INFO", "--pure"], { + cwd: proj, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + proc.stdout?.on("data", (d: Buffer) => (log += d)); + proc.stderr?.on("data", (d: Buffer) => (log += d)); + await until(() => log.includes(`listening on http://127.0.0.1:${port}`) || proc.exitCode !== null || undefined, `engine ${name} listen`, 60_000); + if (proc.exitCode !== null) throw new Error(`engine ${name} exited early:\n${log}`); + return { proc, dir, url: `http://127.0.0.1:${port}`, log: () => log }; +} + +async function stopEngine(e: Engine | undefined): Promise { + if (!e) return; + e.proc.kill("SIGTERM"); + for (let i = 0; i < 40 && e.proc.exitCode === null && e.proc.signalCode === null; i++) await sleep(50); + if (e.proc.exitCode === null && e.proc.signalCode === null) e.proc.kill("SIGKILL"); + rmSync(e.dir, { recursive: true, force: true }); +} + +/** The peer link: a TCP forward 127.0.0.1:PORT_LINK → engine B. `kill()` is a + * dead SSH forward — every proxied socket destroyed, the port closed + * (connection refused); `restore()` listens again. */ +function peerLink(target: number) { + const sockets = new Set(); + let server: net.Server | undefined; + let accepted = 0; + const listen = () => + new Promise((resolve, reject) => { + server = net.createServer((client) => { + accepted += 1; + const upstream = net.connect(target, "127.0.0.1"); + sockets.add(client); + sockets.add(upstream); + const drop = () => { + client.destroy(); + upstream.destroy(); + sockets.delete(client); + sockets.delete(upstream); + }; + client.on("error", drop).on("close", drop); + upstream.on("error", drop).on("close", drop); + client.pipe(upstream); + upstream.pipe(client); + }); + server.once("error", reject); + server.listen(PORT_LINK, "127.0.0.1", () => resolve()); + }); + return { + url: `http://127.0.0.1:${PORT_LINK}`, + restore: listen, + kill: () => + new Promise((resolve) => { + for (const s of sockets) s.destroy(); + sockets.clear(); + if (!server) return resolve(); + server.close(() => resolve()); + server = undefined; + }), + accepted: () => accepted, + }; +} + +async function post(url: string, body: unknown): Promise { + const res = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + if (!res.ok) throw new Error(`POST ${url} → ${res.status} ${await res.text()}`); + return res.json(); +} + +// ── the transcript a viewer holds: messages and parts of ONE session ──────── + +type Json = Record; +type Transcript = Map }>; + +function applyEvent(t: Transcript, sessionID: string, type: string, p: Json): void { + const msg = (id: string) => { + let m = t.get(id); + if (!m) t.set(id, (m = { parts: new Map() })); + return m; + }; + if (type === "message.updated" && p.info?.sessionID === sessionID) msg(p.info.id).info = p.info; + else if (type === "message.removed" && p.sessionID === sessionID) t.delete(p.messageID); + else if (type === "message.part.updated" && p.part?.sessionID === sessionID) msg(p.part.messageID).parts.set(p.part.id, p.part); + else if (type === "message.part.removed" && p.sessionID === sessionID) t.get(p.messageID)?.parts.delete(p.partID); + else if (type === "message.part.delta" && p.sessionID === sessionID) { + const part = t.get(p.messageID)?.parts.get(p.partID); + if (part) part[p.field] = String(part[p.field] ?? "") + p.delta; + } +} + +/** A comparable transcript. The wire drops summary patch bodies (#1778 — the + * stored record keeps them), so they are dropped from both sides. */ +function canonical(messages: Array<{ info?: Json; parts: Json[] }>): unknown { + const strip = (info: Json | undefined) => { + if (!info?.summary?.diffs) return info; + return { ...info, summary: { ...info.summary, diffs: info.summary.diffs.map(({ patch: _p, ...rest }: Json) => rest) } }; + }; + return [...messages] + .sort((a, b) => String(a.info?.id).localeCompare(String(b.info?.id))) + .map((m) => ({ info: strip(m.info), parts: [...m.parts].sort((a, b) => String(a.id).localeCompare(String(b.id))) })); +} + +describe.skipIf(!ENABLED)("#1777 AC6 — two-peer fleet E2E: a link kill mid-turn loses nothing (two scratch engines)", () => { + let a: Engine | undefined; + let b: Engine | undefined; + + beforeAll(async () => { + if (!existsSync(BIN)) throw new Error(`no compiled engine at ${BIN} — run build:binary first`); + if (!(PORT_LINK >= 4190 && PORT_LINK <= 4199)) throw new Error(`link port ${PORT_LINK} outside 4190-4199`); + [a, b] = await Promise.all([startEngine(PORT_A, "a"), startEngine(PORT_B, "b")]); + }, 120_000); + + afterAll(async () => { + await Promise.all([stopEngine(a), stopEngine(b)]); + }); + + it("killing the peer link mid-turn and restoring it leaves the observer's transcript identical to a fresh load", async () => { + const link = peerLink(PORT_B); + await link.restore(); + const ownerMap = new SessionOwnerMap(); + const census: string[] = []; + const driver = new SseFanInDriver({ + ownerMap, + localMachineId: "observer", + localEventUrl: () => a!.url, + peerBaseUrl: (id) => (id === "studio" ? link.url : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: link.url, token: "scratch-peer-token" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 200, // the production cadence drives every re-open + census: new FanInCensus({ plane: "observe", sink: (l) => census.push(l) }), + }); + const front = http.createServer((req, res) => void driver.handle(req, res)); + await new Promise((r) => front.listen(0, "127.0.0.1", r)); + const frontUrl = `http://127.0.0.1:${(front.address() as net.AddressInfo).port}`; + + // ── the observer: the app's cursor applied to every frame ──────────────── + const cursor = createStreamCursor(); + const transcript: Transcript = new Map(); + const seen: Array<{ id?: string; type?: string }> = []; + const resyncs: unknown[] = []; + const gaps: unknown[] = []; + let duplicates = 0; + let sessionID = ""; + const pendingBeforeSession: Array<{ type: string; p: Json }> = []; + let buffer = ""; + const observer = http.get(`${frontUrl}/global/event`, (res) => { + res.setEncoding("utf8"); + res.on("data", (chunk: string) => { + buffer += chunk; + let end: number; + while ((end = buffer.indexOf("\n\n")) >= 0) { + const block = buffer.slice(0, end); + buffer = buffer.slice(end + 2); + const id = /^id: (.*)$/m.exec(block)?.[1]; + const dataLine = block.split("\n").find((l) => l.startsWith("data:")); + if (!dataLine) continue; + const data = JSON.parse(dataLine.slice(5).trim()) as Json; + const type: string = data.payload?.type ?? data.type; + seen.push({ id, type }); + if (type === "amicode.sync.gap") { + gaps.push(data); + cursor.gap(data.namespaces ?? []); + continue; + } + const verdict = cursor.frame({ id, type }); + if (verdict.resync) resyncs.push(verdict.resync); + if (!verdict.apply) { + duplicates += 1; + continue; + } + const props = (data.payload?.properties ?? {}) as Json; + if (sessionID) applyEvent(transcript, sessionID, type, props); + else pendingBeforeSession.push({ type, p: props }); + } + }); + res.on("error", () => undefined); + }); + observer.on("error", () => undefined); + + try { + await until(() => seen.some((f) => f.type === "server.connected"), "the observer's stream"); + + // a session on the PEER, owned by it; the fan-in opens the peer arm + const session = (await post(`${b!.url}/session`, {})) as { id: string }; + sessionID = session.id; + for (const e of pendingBeforeSession) applyEvent(transcript, sessionID, e.type, e.p); + ownerMap.update([{ id: sessionID, amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }]); + await until(() => seen.some((f) => f.type === "server.connected" && parseCompositeCursor(f.id).has("studio")), "the peer arm"); + + // ── the turn: a shell tool streaming output for ~3 s ──────────────────── + const ticks = Array.from({ length: 30 }, (_, i) => `t${i}`).join(" "); + const turn = post(`${b!.url}/session/${sessionID}/shell`, { + agent: "build", + model: { providerID: "scratch", modelID: "scratch" }, + command: `for x in ${ticks}; do echo tick; sleep 0.1; done`, + }); + const runningPart = () => + [...transcript.values()].flatMap((m) => [...m.parts.values()]).find((p) => p.type === "tool" && p.state?.status === "running" && String(p.state?.metadata?.output ?? "").includes("tick")); + await until(() => runningPart(), "the running tool part with output"); + + // ── KILL the link mid-turn ────────────────────────────────────────────── + const killedAt = seen.length; + await link.kill(); + await until(() => census.some((l) => l.includes("arm-closed") && l.includes("arm=studio") && l.includes("reason=upstream-")), "the peer arm to fail"); + await sleep(800); // the turn goes on while the link is down; re-opens are refused + const framesWhileDown = seen.length - killedAt; + const reopensRefused = census.filter((l) => l.includes("arm=studio") && l.includes("arm-closed")).length; + expect(runningPart()).toBeDefined(); // still mid-turn: the completion has not reached the observer + + // ── RESTORE it ────────────────────────────────────────────────────────── + await link.restore(); + await turn; // the turn completes on the peer + // the observer converges on the fresh load (it keeps streaming; poll the owner) + const freshLoad = async () => canonical((await (await fetch(`${b!.url}/session/${sessionID}/message`)).json()) as Array<{ info: Json; parts: Json[] }>); + const observed = () => canonical([...transcript.values()].map((m) => ({ info: m.info, parts: [...m.parts.values()] }))); + let fresh: unknown = await freshLoad(); + const deadline = Date.now() + 15_000; + while (JSON.stringify(observed()) !== JSON.stringify(fresh) && Date.now() < deadline) { + await sleep(100); + fresh = await freshLoad(); + } + + // the replay that recovered it, as the peer engine logged it + const replayLines = b!.log().split("\n").filter((l) => l.includes("global event connected") && l.includes("resume=replay")); + const replayed = replayLines.map((l) => Number(/replayed=(\d+)/.exec(l)?.[1] ?? 0)); + console.log( + "[#1777 AC6]", + JSON.stringify({ + observerFrames: seen.length, + framesWhileDown, + studioArmClosesWhileDown: reopensRefused, + linkConnections: link.accepted(), + peerReplays: replayed, + gaps: gaps.length, + resyncs, + duplicatesDropped: duplicates, + transcriptMessages: transcript.size, + cursor: cursor.current(), + }), + ); + + expect(observed()).toEqual(fresh); // identical to a fresh load + const parts = [...transcript.values()].flatMap((m) => [...m.parts.values()]); + expect(parts.find((p) => p.type === "tool")?.state?.status).toBe("completed"); + expect(gaps).toEqual([]); // no gap signal … + expect(resyncs).toEqual([]); // … and no hole or reset: nothing needed a resync + expect(census.some((l) => l.includes("gap-emitted"))).toBe(false); + expect(replayed.some((n) => n > 0)).toBe(true); // the peer REPLAYED the missed stretch + expect(parseCompositeCursor(cursor.current()).get("studio")).toMatch(/^[A-Za-z0-9_-]+:\d+$/); // the observer's cursor holds the peer's engine id + } finally { + observer.destroy(); + front.closeAllConnections?.(); + await new Promise((r) => front.close(() => r())); + await link.kill(); + } + }, 90_000); +}); diff --git a/packages/extension/test/sse_fanin_aggregator.test.ts b/packages/extension/test/sse_fanin_aggregator.test.ts index bfa3ca665..2c29dd316 100644 --- a/packages/extension/test/sse_fanin_aggregator.test.ts +++ b/packages/extension/test/sse_fanin_aggregator.test.ts @@ -41,6 +41,7 @@ import { import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; import { peerAuthHeader } from "../src/amicode_service/merged_projection"; import type { PeerTokenRead } from "../src/amicode_service/fleet_peer_store"; +import { fakeEngine, sequencedEvent, legacyEvent, engineId, engineFrame, controlEvent, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; // ── test helpers ────────────────────────────────────────────────────────────── @@ -69,6 +70,23 @@ function frame(...lines: string[]): string { return lines.join("\n") + "\n\n"; } +/** #1771: ENGINE-SHAPED arm frames (test/fixtures/engine_stream) — what each + * arm's upstream `/global/event` actually sends: a sequenced (#1776) bus event + * `id: :` + one `data:` line (no `event:` line), one engine epoch + * per namespace. `frame()` above stays for the pure id-parsing/relay units. */ +const MINI_EPOCH = "bWluaTAx"; +const EPOCH_OF: Record = { [LOCAL_NAMESPACE]: LOCAL_EPOCH, studio: PEER_EPOCH, mini: MINI_EPOCH }; +function nsFrame(ns: string, n: number): string { + return sequencedEvent(EPOCH_OF[ns], n, { ns, n }); +} +/** The engine id `nsFrame(ns, n)` carries. */ +const idIn = (ns: string, n: number) => engineId(EPOCH_OF[ns], n); +/** The seq of an `:` id. */ +const seqOf = (id: string | undefined) => Number(id?.split(":")[1]); +/** #1777: an engine per-connection control frame (connected / heartbeat / reset) at `id`. */ +const controlFrame = (id: string, type: string, properties: Record = {}) => + engineFrame(controlEvent(`evt_ctl_${id}`, type, properties), { id }); + /** A frame source that yields a fixed list of raw frames, then ends. */ function fixedSource(frames: string[]): SseFrameSource { let i = 0; @@ -89,19 +107,25 @@ function fixedSource(frames: string[]): SseFrameSource { // ══════════════════════════════════════════════════════════════════════════════ describe("#1511 AC1 (D4) — fleet-of-one byte-identity is the #1264 regression guard", () => { it("with zero peers, the aggregate stream is frame-for-frame identical to the local source", () => { - const sink = collectingSink(); - const agg = new SseFanInAggregator({ sink }); - agg.connect(); // no cursor, no peers, no focus provider → pure passthrough - - const f1 = frame("event: message", "data: {\"a\":1}", "id: 1"); - const f2 = frame("event: message", "data: {\"b\":2}", "id: 2"); - const f3 = frame("data: {\"c\":3}", "id: 3"); // no event: line — still verbatim - agg.ingest(LOCAL_NAMESPACE, f1); - agg.ingest(LOCAL_NAMESPACE, f2); - agg.ingest(LOCAL_NAMESPACE, f3); - - // BYTE-IDENTICAL: no id rewrite, no namespacing, no reordering, no loss. - expect(sink.text()).toBe(f1 + f2 + f3); + // #1771: the local engine's REAL stream, from an old (id-less) engine and a + // #1776 (sequenced) one: connected, an event, the plain + sync twin of a + // stored event, a heartbeat. + for (const engine of [fakeEngine({ legacy: true }), fakeEngine({ epoch: LOCAL_EPOCH })]) { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); // no cursor, no peers, no focus provider → pure passthrough + + const frames = [ + engine.connected(), + engine.event("session.status", { a: 1 }), + ...engine.stored("message.part.updated", { b: 2 }, { aggregateID: "ses_1" }), + engine.heartbeat(), + ]; + for (const f of frames) agg.ingest(LOCAL_NAMESPACE, f); + + // BYTE-IDENTICAL: no id rewrite, no namespacing, no reordering, no loss. + expect(sink.text()).toBe(frames.join("")); + } }); it("`?lastEventID=local=` AND bare `` both resume the local arm at ", () => { @@ -151,6 +175,22 @@ describe("#1511 AC4 (D3) — composite cursor", () => { expect(parseCompositeCursor(undefined).size).toBe(0); }); + it("#1777: a bare ENGINE id `:` is the local position (the `:` is not a composite), and a composite of engine ids round-trips", () => { + // the cursor a fleet-of-one client resumes with: the engine's id, verbatim + expect([...parseCompositeCursor(idIn(LOCAL_NAMESPACE, 12))]).toEqual([[LOCAL_NAMESPACE, idIn(LOCAL_NAMESPACE, 12)]]); + // epochs are base64url (`-` and `_` included) — still one bare local id + expect([...parseCompositeCursor("a-b_c9Zx:7")]).toEqual([[LOCAL_NAMESPACE, "a-b_c9Zx:7"]]); + // the cursor a fanned-in client resumes with: every origin's engine id + const composite = `local=${idIn(LOCAL_NAMESPACE, 12)};studio=${idIn("studio", 40)}`; + expect(formatCompositeCursor(parseCompositeCursor(composite))).toBe(composite); + const agg = new SseFanInAggregator({ sink: collectingSink() }); + const resume = agg.connect(composite); + expect(resume.get(LOCAL_NAMESPACE)).toBe(idIn(LOCAL_NAMESPACE, 12)); + expect(resume.get("studio")).toBe(idIn("studio", 40)); + // and it is where each arm resumes from until the arm delivers + expect(agg.cursorFor("studio")).toBe(idIn("studio", 40)); + }); + it("connect() re-subscribes each upstream with ITS OWN resume id", () => { const sink = collectingSink(); const agg = new SseFanInAggregator({ sink }); @@ -166,14 +206,18 @@ describe("#1511 AC4 (D3) — composite cursor", () => { agg.connect(); agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); // studio advances to 9 - agg.ingest("studio", frame("event: message", "data: {}", "id: 9")); - expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe("9"); - // an id-less studio frame (heartbeat/comment) must NOT move the cursor - agg.ingest("studio", frame(": keep-alive comment")); - expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe("9"); + agg.ingest("studio", nsFrame("studio", 9)); + expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe(idIn("studio", 9)); + // an id-less studio frame must NOT move the cursor: an old engine's + // heartbeat and event (#1771 — the id-less frames production sees), or a comment + const old = fakeEngine({ legacy: true }); + for (const f of [old.heartbeat(), old.event("session.status", { s: 1 }), frame(": keep-alive comment")]) { + agg.ingest("studio", f); + expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe(idIn("studio", 9)); + } // and neither does a frame with an empty id value agg.ingest("studio", frame("data: {}", "id:")); - expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe("9"); + expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe(idIn("studio", 9)); }); }); @@ -204,14 +248,11 @@ describe("#1511 AC3 (D2) — namespaced relay + res adapter", () => { it("pipeToResponse writes well-formed namespaced SSE bytes and flushes per frame", async () => { const sink = collectingSink(); - const src = fixedSource([ - frame("event: message", "data: {\"a\":1}", "id: 1"), - frame("event: message", "data: {\"b\":2}", "id: 2"), - ]); + const src = fixedSource([nsFrame("studio", 1), nsFrame("studio", 2)]); await pipeToResponse(src, sink, { namespace: "studio" }); const text = sink.text(); - expect(text).toContain(`id: studio${NS_SEP}1`); - expect(text).toContain(`id: studio${NS_SEP}2`); + expect(text).toContain(`id: studio${NS_SEP}${idIn("studio", 1)}`); + expect(text).toContain(`id: studio${NS_SEP}${idIn("studio", 2)}`); // every frame ends with the SSE blank-line delimiter expect(text.endsWith("\n\n")).toBe(true); // one flush per frame (no head-of-line buffering) @@ -378,9 +419,9 @@ describe("#1511 AC6 (A) — focus-on-connect snapshot", () => { expect(sink.text()).toContain("\"focusedMachineId\":\"studio\""); // a following local event frame comes AFTER the focus seed - agg.ingest(LOCAL_NAMESPACE, frame("data: {}", "id: 1")); + agg.ingest(LOCAL_NAMESPACE, nsFrame(LOCAL_NAMESPACE, 1)); const idx = sink.text().indexOf("amicode.fleet.focus"); - const evtIdx = sink.text().indexOf("id: 1"); + const evtIdx = sink.text().indexOf(`id: ${idIn(LOCAL_NAMESPACE, 1)}`); expect(idx).toBeGreaterThanOrEqual(0); expect(evtIdx).toBeGreaterThan(idx); }); @@ -479,9 +520,6 @@ describe("#1511 AC7 (B) — per-namespace back-pressure", () => { }; } - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("one overflowing peer drops in ISOLATION; the other namespaces and local keep flowing", () => { const sink = controllableSink(); @@ -522,11 +560,11 @@ describe("#1511 AC7 (B) — per-namespace back-pressure", () => { // studio's cursor sits at the last DELIVERED id, never a dropped one — // so a reconnect with this composite replays the gap losslessly (D3 × B). - const studioCursor = Number(parseCompositeCursor(agg.cursor()).get("studio")); + const studioCursor = seqOf(parseCompositeCursor(agg.cursor()).get("studio")); expect(studioCursor).toBeLessThan(5); // mini + local cursors reflect full delivery - expect(parseCompositeCursor(agg.cursor()).get("mini")).toBe("2"); - expect(parseCompositeCursor(agg.cursor()).get("local")).toBe("1"); + expect(parseCompositeCursor(agg.cursor()).get("mini")).toBe(idIn("mini", 2)); + expect(parseCompositeCursor(agg.cursor()).get("local")).toBe(idIn(LOCAL_NAMESPACE, 1)); }); it("steady state (downstream flowing) never buffers — frames deliver directly", () => { @@ -572,9 +610,6 @@ describe("#1617 — backpressure re-arm + gap signal", () => { }, }; } - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("the WEDGE (no overflow): one trip-frame strands one frame behind it; resume() delivers BOTH in order", () => { const sink = controllableSink(); @@ -596,7 +631,7 @@ describe("#1617 — backpressure re-arm + gap signal", () => { // order preserved: frame 1 before frame 2 expect(sink.text().indexOf('"n":1')).toBeLessThan(sink.text().indexOf('"n":2')); // cursor advanced past both delivered frames - expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe("2"); + expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe(idIn("studio", 2)); }); it("onBackpressure fires exactly when a write() returns false (the driver's upstream-pause hook)", () => { @@ -689,8 +724,8 @@ describe("#1617 — backpressure re-arm + gap signal", () => { const agg = new SseFanInAggregator({ sink }); agg.connect(); // zero peers → the §D4 verbatim branch - const f1 = frame("event: message", 'data: {"big":"tool-output"}', "id: 7"); - const f2 = frame("event: session.status", 'data: {"status":"idle"}', "id: 8"); // the rail-clearing frame + const f1 = sequencedEvent(LOCAL_EPOCH, 7, { big: "tool-output" }); + const f2 = sequencedEvent(LOCAL_EPOCH, 8, { status: "idle" }, "session.status"); // the rail-clearing frame sink.block(); // the socket fills on the NEXT write agg.ingest(LOCAL_NAMESPACE, f1); // trip-frame: written, write()→false @@ -707,7 +742,7 @@ describe("#1617 — backpressure re-arm + gap signal", () => { // composite id, no namespacing, no gap frame (no overflow occurred). expect(sink.text()).toBe(f1 + f2); expect(sink.text()).not.toContain("local\u001f"); // no NS_SEP namespacing - expect(sink.text()).not.toContain("local=8"); // no composite-cursor id rewrite + expect(sink.text()).not.toContain(`local=${idIn(LOCAL_NAMESPACE, 8)}`); // no composite-cursor id rewrite expect(sink.text()).not.toContain("amicode.sync.gap"); }); @@ -715,8 +750,8 @@ describe("#1617 — backpressure re-arm + gap signal", () => { const sink = controllableSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); - const f1 = frame("event: message", 'data: {"a":1}', "id: 1"); - const f2 = frame("data: {\"c\":3}", "id: 3"); // no event: line + // an old engine's id-less stream: the plain + sync twin of a stored event + const [f1, f2] = fakeEngine({ legacy: true }).stored("message.updated", { a: 1 }, { aggregateID: "ses_1" }); agg.ingest(LOCAL_NAMESPACE, f1); agg.ingest(LOCAL_NAMESPACE, f2); expect(sink.text()).toBe(f1 + f2); // untouched fast path @@ -732,9 +767,6 @@ describe("#1617 — backpressure re-arm + gap signal", () => { // the verbatim stream would break #1264, so NO gap is emitted there. // ══════════════════════════════════════════════════════════════════════════════ describe("#1638 — per-namespace cursor accessor (cursorFor)", () => { - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("cursorFor(ns) returns the live last-DELIVERED id for that namespace, advancing per delivered frame", () => { const sink = collectingSink(); @@ -746,18 +778,18 @@ describe("#1638 — per-namespace cursor accessor (cursorFor)", () => { expect(agg.cursorFor("studio")).toBeUndefined(); agg.ingest("studio", nsFrame("studio", 1)); - expect(agg.cursorFor("studio")).toBe("1"); // advanced to the delivered id + expect(agg.cursorFor("studio")).toBe(idIn("studio", 1)); // advanced to the delivered id agg.ingest("studio", nsFrame("studio", 2)); - expect(agg.cursorFor("studio")).toBe("2"); // advanced again + expect(agg.cursorFor("studio")).toBe(idIn("studio", 2)); // advanced again }); it("cursorFor tracks the LOCAL namespace on the verbatim (fleet-of-one) path too", () => { const sink = collectingSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); // zero peers → verbatim - agg.ingest(LOCAL_NAMESPACE, nsFrame("local", 5)); - expect(agg.cursorFor(LOCAL_NAMESPACE)).toBe("5"); + agg.ingest(LOCAL_NAMESPACE, nsFrame(LOCAL_NAMESPACE, 5)); + expect(agg.cursorFor(LOCAL_NAMESPACE)).toBe(idIn(LOCAL_NAMESPACE, 5)); }); it("an id-less frame does NOT advance cursorFor (matches the composite-cursor invariant)", () => { @@ -766,22 +798,22 @@ describe("#1638 — per-namespace cursor accessor (cursorFor)", () => { agg.connect(); agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); agg.ingest("studio", nsFrame("studio", 1)); - agg.ingest("studio", frame(": heartbeat")); // id-less - expect(agg.cursorFor("studio")).toBe("1"); // unchanged by the id-less frame + agg.ingest("studio", fakeEngine({ legacy: true }).heartbeat()); // id-less (an old engine's heartbeat) + expect(agg.cursorFor("studio")).toBe(idIn("studio", 1)); // unchanged by the id-less frame }); }); describe("#1638 — composite-mode reopenArm emits exactly one id-less gap; fleet-of-one emits none", () => { - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } + // #1777: the gap is now the FALLBACK for an upstream with no ids to resume + // from (an engine older than #1776) — a sequenced arm resumes instead (see the + // #1777 suite below) — so these frames come from a legacy (id-less) engine. it("COMPOSITE MODE: a non-initial re-open emits EXACTLY ONE id-less amicode.sync.gap naming that namespace", () => { const sink = collectingSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); - agg.ingest("studio", nsFrame("studio", 1)); + agg.ingest("studio", legacyEvent({ ns: "studio", n: 1 })); agg.reopenArm("studio"); // the #1601 re-open path (arm dropped, reconcile re-opens) @@ -802,7 +834,7 @@ describe("#1638 — composite-mode reopenArm emits exactly one id-less gap; flee const sink = collectingSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); // zero peers → verbatim, no focus provider - const f1 = frame("event: message", 'data: {"a":1}', "id: 1"); + const f1 = nsFrame(LOCAL_NAMESPACE, 1); agg.ingest(LOCAL_NAMESPACE, f1); agg.reopenArm(LOCAL_NAMESPACE); // the common #1601 local re-open in solo mode @@ -824,6 +856,84 @@ describe("#1638 — composite-mode reopenArm emits exactly one id-less gap; flee }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1773 AC1 — "delivered" means ANY data frame, not only an id'd one. The re-open +// gap used to fire only for arms that had delivered a frame carrying an SSE id — +// which a peer running an older engine never sends — so a re-opened arm never +// announced its gap. Engine-shaped fixtures in BOTH modes: legacy (id-less) and +// sequenced (#1776, `id: :`). +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1773 AC1 — a re-open of an arm that delivered ANY data frame emits exactly one gap (composite mode)", () => { + const gapFrames = (text: string) => text.split("\n\n").filter((f) => f.includes("amicode.sync.gap")); + const gapOf = (f: string) => JSON.parse(f.split("\n").find((l) => l.startsWith("data:"))!.slice(5).trim()); + + // #1777: in BOTH modes "delivered" still means any data frame — but only the + // legacy (id-less) arm gaps: the sequenced arm has engine ids to resume from, + // so its re-open RESUMES (the upstream replays) and writes nothing at all. + for (const mode of ["legacy", "sequenced"] as const) { + it(`${mode} engine: a non-initial re-open ${mode === "legacy" ? "emits exactly one id-less gap naming that namespace" : "resumes — no gap"}, and never advances a cursor`, () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(`local=${LOCAL_EPOCH}:3`); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + const studio = mode === "legacy" ? fakeEngine({ legacy: true }) : fakeEngine({ epoch: PEER_EPOCH }); + + // the initial open: no gap + expect(agg.reopenArm("studio")).toBe("open"); + // the arm delivers the engine's real stream: connected, an event, a stored + // event's plain + sync twin, a heartbeat + for (const f of [ + studio.connected(), + studio.event("session.status", { sessionID: "ses_1", status: { type: "busy" } }), + ...studio.stored("message.part.updated", { part: { id: "prt_1" } }, { aggregateID: "ses_1" }), + studio.heartbeat(), + ]) + agg.ingest("studio", f); + const cursorBefore = agg.cursor(); + expect(sink.text()).not.toContain("amicode.sync.gap"); + + // the arm drops and the driver re-opens it + if (mode === "sequenced") { + const before = sink.text(); + expect(agg.reopenArm("studio")).toBe("resume"); + expect(agg.cursorFor("studio")).toBe(studio.idAt(studio.seq)); // … from the last delivered engine id + expect(sink.text()).toBe(before); // nothing written + expect(agg.cursor()).toBe(cursorBefore); + return; + } + expect(agg.reopenArm("studio")).toBe("gap"); + + const gaps = gapFrames(sink.text()); + expect(gaps).toHaveLength(1); + expect(gapOf(gaps[0])).toEqual({ type: "amicode.sync.gap", namespaces: ["studio"] }); + expect(gaps[0].split("\n").some((l) => l.startsWith("id:"))).toBe(false); // id-less + expect(agg.cursor()).toBe(cursorBefore); // a gap signal never advances a cursor + }); + } + + it("an arm that delivered only a comment frame (no data line) has not delivered — its re-open is still an initial open", () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + agg.ingest("studio", frame(": keep-alive")); + expect(agg.reopenArm("studio")).toBe("open"); + expect(sink.text()).not.toContain("amicode.sync.gap"); + }); + + it("the LOCAL arm (legacy engine) in composite mode gaps on re-open too, naming `local`", () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + agg.ingest(LOCAL_NAMESPACE, fakeEngine({ legacy: true }).connected()); + expect(agg.reopenArm(LOCAL_NAMESPACE)).toBe("gap"); + const gaps = gapFrames(sink.text()); + expect(gaps).toHaveLength(1); + expect(gapOf(gaps[0]).namespaces).toEqual([LOCAL_NAMESPACE]); + }); +}); + describe("#1638 — resume() no-op WHEN ALREADY FLOWING (not when the sink is full); #1617 drain-flush preserved", () => { function controllableSink() { const chunks: string[] = []; @@ -848,9 +958,6 @@ describe("#1638 — resume() no-op WHEN ALREADY FLOWING (not when the sink is fu }, }; } - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("resume() while ALREADY FLOWING is a no-op: it does not re-flush or duplicate already-delivered frames", () => { const sink = controllableSink(); @@ -871,8 +978,8 @@ describe("#1638 — resume() no-op WHEN ALREADY FLOWING (not when the sink is fu const sink = controllableSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); // fleet-of-one verbatim path - const f1 = frame("event: message", 'data: {"big":"tool-output"}', "id: 7"); - const f2 = frame("event: session.status", 'data: {"status":"idle"}', "id: 8"); // the rail-clearing idle frame + const f1 = sequencedEvent(LOCAL_EPOCH, 7, { big: "tool-output" }); + const f2 = sequencedEvent(LOCAL_EPOCH, 8, { status: "idle" }, "session.status"); // the rail-clearing idle frame sink.block(); agg.ingest(LOCAL_NAMESPACE, f1); // trip-frame, flowing→false agg.ingest(LOCAL_NAMESPACE, f2); // stranded in the verbatim buffer @@ -905,9 +1012,6 @@ describe("#1638 — control/gap/focus writes are return-checked and drain in tru }, }; } - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("a control frame (gap) written DURING backpressure emerges AFTER the data frames that preceded it", () => { const sink = controllableSink(); @@ -916,8 +1020,9 @@ describe("#1638 — control/gap/focus writes are return-checked and drain in tru agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); sink.block(); - agg.ingest("studio", nsFrame("studio", 1)); // trip-frame: written, flowing→false - agg.ingest("studio", nsFrame("studio", 2)); // buffered behind the trip-frame + // #1777: an id-less (legacy) peer, so the re-open below takes the gap fallback + agg.ingest("studio", legacyEvent({ ns: "studio", n: 1 })); // trip-frame: written, flowing→false + agg.ingest("studio", legacyEvent({ ns: "studio", n: 2 })); // buffered behind the trip-frame // a control frame arrives mid-backpressure — it MUST NOT jump ahead of the // buffered data frame, nor be swallowed into the full sink. agg.reopenArm("studio"); // emits a gap through the return-checked helper @@ -943,3 +1048,240 @@ describe("#1638 — control/gap/focus writes are return-checked and drain in tru expect(n2Idx).toBeLessThan(gapIdx); // control frame AFTER the preceding data frame }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1771 — the aggregator's census counters (per arm, per connection) and its +// transition observer. Observation only: the frames on the sink are unchanged. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1771 — per-arm traffic counters, pending depth, and the drop/gap observer", () => { + function blockableSink() { + const chunks: string[] = []; + let blocked = false; + return { + write(chunk: string): boolean { + chunks.push(chunk); + return !blocked; + }, + flush() {}, + block() { + blocked = true; + }, + unblock() { + blocked = false; + }, + text: () => chunks.join(""), + }; + } + + it("counts every ingested frame per arm — frames, UTF-8 bytes, largest, last-frame time — whether delivered or buffered", () => { + let t = 500; + const sink = blockableSink(); + const agg = new SseFanInAggregator({ sink, now: () => t }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + const local = fakeEngine({ epoch: LOCAL_EPOCH }); + const studio = fakeEngine({ legacy: true }); // an old peer engine: id-less frames + const localFrames = [local.connected(), ...local.stored("message.part.updated", { text: "résumé" }, { aggregateID: "ses_1" })]; + for (const f of localFrames) agg.ingest(LOCAL_NAMESPACE, f); + t = 900; + sink.block(); + const studioFrames = [studio.event("session.status", { status: "busy" }), studio.heartbeat()]; + for (const f of studioFrames) agg.ingest("studio", f); // the 1st trips backpressure, the 2nd buffers + + expect(agg.traffic(LOCAL_NAMESPACE)).toEqual({ + frames: 3, + bytes: localFrames.reduce((n, f) => n + Buffer.byteLength(f), 0), + largestFrame: Math.max(...localFrames.map((f) => Buffer.byteLength(f))), + lastFrameAt: 500, + drops: 0, + gaps: 0, + }); + expect(agg.traffic("studio")).toMatchObject({ frames: 2, lastFrameAt: 900, drops: 0 }); + expect(agg.pendingDepth("studio")).toBe(1); + expect(agg.pendingDepth(LOCAL_NAMESPACE)).toBe(0); + expect(agg.traffic("mini")).toEqual({ frames: 0, bytes: 0, largestFrame: 0, drops: 0, gaps: 0 }); // never ingested + }); + + it("the observer sees the FIRST drop of an overflow episode, then the gap that ends it; drops stay cumulative", () => { + const sink = blockableSink(); + const drops: Array<[string, number]> = []; + const gaps: Array<[string[], string]> = []; + const agg = new SseFanInAggregator({ + sink, + bufferBound: 2, + observer: { onOverflowDrop: (ns, pending) => drops.push([ns, pending]), onGap: (nss, cause) => gaps.push([nss, cause]) }, + }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + sink.block(); + for (let n = 1; n <= 6; n++) agg.ingest("studio", sequencedEvent(PEER_EPOCH, n, { n })); // 1 written, 2 buffered, 3 dropped + expect(drops).toEqual([["studio", 2]]); // one episode → one callback + expect(agg.traffic("studio").drops).toBe(3); + + sink.unblock(); + agg.resume(); + expect(gaps).toEqual([[["studio"], "overflow"]]); + expect(agg.traffic("studio")).toMatchObject({ drops: 3, gaps: 1 }); + expect(agg.pendingDepth("studio")).toBe(0); + + // a NEW episode after the gap is a new first drop + sink.block(); + for (let n = 7; n <= 10; n++) agg.ingest("studio", sequencedEvent(PEER_EPOCH, n, { n })); + expect(drops).toHaveLength(2); + + // #1777: a sequenced arm's re-open resumes (no gap); an upstream that then + // answers WITHOUT ids cannot have replayed — that is the reopen gap + expect(agg.reopenArm("studio")).toBe("resume"); + sink.unblock(); + agg.resume(); + agg.ingest("studio", legacyEvent({ n: 11 })); + expect(gaps.at(-1)).toEqual([["studio"], "reopen"]); + }); + + it("a throwing observer never breaks delivery", () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ + sink, + observer: { + onGap: () => { + throw new Error("broken census"); + }, + }, + }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + const f = legacyEvent({ n: 1 }); // #1777: an id-less peer — its re-open gaps + agg.ingest("studio", f); + expect(() => agg.reopenArm("studio")).not.toThrow(); + expect(sink.text()).toContain("amicode.sync.gap"); + }); +}); + +describe("#1771 AC5 — the census observer changes no delivered byte", () => { + it("an overflow + gap + reopen sequence writes the identical bytes with and without an observer", () => { + function run(observed: boolean): string { + const chunks: string[] = []; + let blocked = false; + const sink = { write: (c: string) => (chunks.push(c), !blocked), flush() {} }; + const agg = new SseFanInAggregator({ + sink, + bufferBound: 3, + ...(observed ? { now: () => 7, observer: { onOverflowDrop() {}, onGap() {} } } : {}), + }); + agg.connect(`local=${LOCAL_EPOCH}:0;studio=${PEER_EPOCH}:0`); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + const local = fakeEngine({ legacy: true }); + const studio = fakeEngine({ epoch: PEER_EPOCH }); + for (const f of [local.connected(), ...local.stored("message.updated", { n: 1 }, { aggregateID: "ses_1" })]) agg.ingest(LOCAL_NAMESPACE, f); + blocked = true; + for (let i = 0; i < 9; i++) agg.ingest("studio", studio.event("message.part.delta", { i })); + agg.ingest(LOCAL_NAMESPACE, local.heartbeat()); + blocked = false; + agg.resume(); + agg.reopenArm("studio"); + agg.ingest("studio", studio.heartbeat()); + return chunks.join(""); + } + const plain = run(false); + expect(run(true)).toBe(plain); + expect(plain).toContain("amicode.sync.gap"); // the sequence did exercise drop + gap + }); +}); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1777 — LOSSLESS RESUME in the aggregator core. An arm that delivered engine +// ids (#1776 `:`) RESUMES on re-open: `cursorFor` is where its +// replay starts, nothing is written, and the upstream's first answer decides — +// its `server.connected` confirms the replay; an id-less answer means it could +// not resume (the #1773 fallback announces the loss). +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1777 AC1 — the aggregator resumes a sequenced arm from its last id", () => { + function blockableSink() { + const chunks: string[] = []; + let blocked = false; + return { + write(chunk: string): boolean { + chunks.push(chunk); + return !blocked; + }, + flush() {}, + block() { + blocked = true; + }, + unblock() { + blocked = false; + }, + text: () => chunks.join(""), + }; + } + + it("a re-open while frames still queue behind backpressure resumes AFTER them — the replay never re-sends one", () => { + const sink = blockableSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + sink.block(); + for (const n of [1, 2, 3]) agg.ingest("studio", nsFrame("studio", n)); // 1 written (trips), 2 + 3 queued + expect(agg.pendingDepth("studio")).toBe(2); + expect(agg.cursorFor("studio")).toBe(idIn("studio", 3)); // queued = on its way to the listener + expect(agg.reopenArm("studio")).toBe("resume"); + sink.unblock(); + agg.resume(); + // the resumed upstream answers from 3: connected at 3, then 4 — each frame exactly once + agg.ingest("studio", controlFrame(idIn("studio", 3), "server.connected")); + agg.ingest("studio", nsFrame("studio", 4)); + const ns = [...sink.text().matchAll(/"n":(\d+)/g)].map((m) => Number(m[1])); + expect(ns).toEqual([1, 2, 3, 4]); + expect(sink.text()).not.toContain("amicode.sync.gap"); + expect(sink.text()).not.toContain("server.connected"); // the upstream's handshake is not forwarded + }); + + it("an overflow-dropped frame never advances the resume point (that stretch is the overflow gap's)", () => { + const sink = blockableSink(); + const agg = new SseFanInAggregator({ sink, bufferBound: 1 }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + sink.block(); + for (const n of [1, 2, 3, 4]) agg.ingest("studio", nsFrame("studio", n)); // 1 written, 2 queued, 3 + 4 dropped + expect(agg.dropped("studio")).toBe(2); + expect(agg.cursorFor("studio")).toBe(idIn("studio", 2)); + }); + + it("composite: a resumed arm whose upstream answers WITHOUT ids (it cannot replay) gets one reopen gap, then its frames", () => { + const gaps: Array<[string[], string]> = []; + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink, observer: { onGap: (nss, cause) => gaps.push([nss, cause]) } }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + agg.ingest("studio", nsFrame("studio", 1)); + expect(agg.reopenArm("studio")).toBe("resume"); + expect(sink.text()).not.toContain("amicode.sync.gap"); + expect(agg.ingest("studio", fakeEngine({ legacy: true }).connected())).toBe("ok"); + expect(gaps).toEqual([[["studio"], "reopen"]]); + const text = sink.text(); + expect(text.indexOf("amicode.sync.gap")).toBeLessThan(text.indexOf("server.connected")); // gap first, then the frame + }); + + it("fleet-of-one: the same id-less answer severs (ingest says so, nothing is written — #1264)", () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); + const f1 = nsFrame(LOCAL_NAMESPACE, 1); + agg.ingest(LOCAL_NAMESPACE, f1); + expect(agg.reopenArm(LOCAL_NAMESPACE)).toBe("resume"); + expect(agg.ingest(LOCAL_NAMESPACE, fakeEngine({ legacy: true }).connected())).toBe("sever"); + expect(sink.text()).toBe(f1); + }); + + it("an arm with only id-less frames keeps the #1773 fallback (gap in composite, sever in fleet-of-one)", () => { + const composite = new SseFanInAggregator({ sink: collectingSink() }); + composite.connect(); + composite.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + composite.ingest("studio", legacyEvent({ n: 1 })); + expect(composite.reopenArm("studio")).toBe("gap"); + const solo = new SseFanInAggregator({ sink: collectingSink() }); + solo.connect(); + solo.ingest(LOCAL_NAMESPACE, legacyEvent({ n: 1 })); + expect(solo.reopenArm(LOCAL_NAMESPACE)).toBe("sever"); + }); +}); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 4d56ed51a..0eb0d9b3e 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -3,37 +3,67 @@ // pump() method removes it from the internal sources Map so the next // reconcile() tick can re-open the arm — not leave a zombie that blocks // reconnection indefinitely. -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; import * as http from "node:http"; -import { SseFanInDriver } from "../src/amicode_service/sse_fanin_driver"; +import { appendFileSync } from "node:fs"; +import { StringDecoder } from "node:string_decoder"; +import { SseFanInDriver, SseFrameSplitter, openHttpSseSource } from "../src/amicode_service/sse_fanin_driver"; import type { SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; -import { LOCAL_NAMESPACE } from "../src/amicode_service/sse_fanin_aggregator"; +import { LOCAL_NAMESPACE, frameRawId, isDataFrame } from "../src/amicode_service/sse_fanin_aggregator"; import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; +import { FanInCensus, FANIN_CENSUS_INTERVAL_MS, FANIN_CLOSE_REASONS } from "../src/amicode_service/fanin_census"; +import { + fakeEngine, + startEngineSseServer, + writeChunks, + largeSummaryFrame, + sequencedEvent, + legacyEvent, + engineId, + chunkBytes, + CHUNK_64K, + TEN_MB, + LOCAL_EPOCH, + PEER_EPOCH, + replayingEngine, + type ReplayingEngine, +} from "./fixtures/engine_stream/engine_frames"; +import { parseCompositeCursor } from "../src/amicode_service/sse_composite_cursor"; // ── helpers ────────────────────────────────────────────────────────────────── -function sseFrame(...lines: string[]): string { - return lines.join("\n") + "\n\n"; -} - -/** A controllable frame source: push frames into it, end it on demand. */ +/** #1771: ENGINE-SHAPED upstream frames (test/fixtures/engine_stream) — the + * wire the engine's `/global/event` actually writes: a sequenced (#1776) bus + * event `id: :` + one `data:` line of the serialized global event, + * no `event:` line. These replace the synthetic `event: message` / `id: l1` + * frames the suite used to feed (a wire production never sees). */ +const localEvt = (seq: number, properties: Record, type?: string) => sequencedEvent(LOCAL_EPOCH, seq, properties, type); +const peerEvt = (seq: number, properties: Record, type?: string) => sequencedEvent(PEER_EPOCH, seq, properties, type); +const localId = (seq: number) => engineId(LOCAL_EPOCH, seq); +const peerId = (seq: number) => engineId(PEER_EPOCH, seq); + +/** A controllable frame source: push frames into it, end it on demand. #1773: + * `end("error")` models an upstream that FAILED (the real reader's + * `endReason()`); a bare `end()` is a clean upstream end. */ interface Ctl { push(frame: string): void; - end(): void; + end(why?: "end" | "error"): void; source: SseFrameSource; } function controllableSource(): Ctl { const queue: string[] = []; const waiters: Array<(v: string | null) => void> = []; let ended = false; + let reason: "end" | "error" | undefined; const push = (f: string): void => { const w = waiters.shift(); if (w) w(f); else queue.push(f); }; - const end = (): void => { + const end = (why: "end" | "error" = "end"): void => { if (ended) return; ended = true; + reason = why; let w: ((v: string | null) => void) | undefined; while ((w = waiters.shift())) w(null); }; @@ -50,36 +80,82 @@ function controllableSource(): Ctl { close(): void { end(); }, + endReason: () => reason, }, }; } -/** A mock ServerResponse that collects writes. */ -function mockRes(): http.ServerResponse & { text(): string; fireClose(): void } { +/** A mock ServerResponse that collects writes. #1772 — also records whether the + * response was ENDED (an evicted listener is ended by the driver) and the + * status/headers/body of a non-stream answer (the beyond-cap refusal). #1773 — + * and whether it was DESTROYED (an error close: the listener sees a transport + * error, not a clean end), with the error; like a real socket teardown, a + * destroy fires the response's `close` listeners. */ +type MockRes = http.ServerResponse & { + text(): string; + fireClose(): void; + ended: boolean; + destroyed: boolean; + destroyError?: Error; + status?: number; + headerMap: Record; + body: string; +}; +function mockRes(): MockRes { const chunks: string[] = []; const closeListeners: Array<() => void> = []; - return { + const res = { headersSent: false, + ended: false, + destroyed: false, + destroyError: undefined as Error | undefined, + status: undefined as number | undefined, + headerMap: {} as Record, + body: "", write(chunk: string) { chunks.push(chunk); return true; }, - writeHead() { - return this; + writeHead(status: number, headers?: Record) { + res.status = status; + for (const [k, v] of Object.entries(headers ?? {})) res.headerMap[k.toLowerCase()] = String(v); + res.headersSent = true; + return res; + }, + setHeader(k: string, v: string | number) { + res.headerMap[k.toLowerCase()] = String(v); + return res; + }, + set statusCode(s: number) { + res.status = s; + }, + get statusCode(): number { + return res.status ?? 200; }, flushHeaders() {}, on(event: string, cb: (...args: unknown[]) => void) { if (event === "close") closeListeners.push(cb as () => void); - return this; + return res; + }, + end(chunk?: string) { + if (typeof chunk === "string") res.body += chunk; + res.ended = true; + }, + destroy(err?: Error) { + if (res.destroyed) return res; + res.destroyed = true; + res.destroyError = err; + for (const cb of closeListeners) cb(); + return res; }, - end() {}, text() { return chunks.join(""); }, fireClose() { for (const cb of closeListeners) cb(); }, - } as unknown as http.ServerResponse & { text(): string; fireClose(): void }; + }; + return res as unknown as MockRes; } /** #1617 — a mock ServerResponse that models REAL Node backpressure: `write()` @@ -184,8 +260,8 @@ describe("#1566 — dead arm zombie: pump exit cleans up source so reconcile re- { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]); - const F1 = sseFrame("event: message", 'data: {"from":"studio","n":1}', "id: p1"); - const F2 = sseFrame("event: message", 'data: {"from":"studio","n":2}', "id: p2"); + const F1 = peerEvt(1, { from: "studio", n: 1 }); + const F2 = peerEvt(2, { from: "studio", n: 2 }); // Track every openUpstream call and the controllable sources they return. const openCalls: string[] = []; @@ -327,8 +403,8 @@ describe("#1601 — dead LOCAL arm: reconcile re-opens the local arm", () => { { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]); - const L1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1"); - const L2 = sseFrame("event: message", 'data: {"from":"macbook","n":2}', "id: l2"); + const L1 = localEvt(1, { from: "macbook", n: 1 }); + const L2 = localEvt(2, { from: "macbook", n: 2 }); // Track every openUpstream call; hand out a FRESH local source per open so we // can distinguish the re-opened arm from the original. @@ -467,8 +543,8 @@ describe("#1617 — driver wires drain→resume + upstream pause/resume", () => { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]) - const F1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1") - const F2 = sseFrame("event: message", 'data: {"from":"macbook","n":2}', "id: l2") // the stranded idle-equivalent + const F1 = localEvt(1, { from: "macbook", n: 1 }) + const F2 = localEvt(2, { from: "macbook", n: 2 }) // the stranded idle-equivalent const localCtl = pausableSource() const peerCtl = pausableSource() @@ -525,8 +601,8 @@ describe("#1617 — driver wires drain→resume + upstream pause/resume", () => // the solo-machine wedge the composite test above does NOT exercise. const ownerMap = new SessionOwnerMap() - const F1 = sseFrame("event: message", 'data: {"big":"tool-output"}', "id: 7") - const F2 = sseFrame("event: session.status", 'data: {"status":"idle"}', "id: 8") // clears the rail + const F1 = localEvt(7, { big: "tool-output" }) + const F2 = localEvt(8, { status: "idle" }, "session.status") // clears the rail const localCtl = pausableSource() @@ -563,7 +639,7 @@ describe("#1617 — driver wires drain→resume + upstream pause/resume", () => expect(res.text()).toContain('"status":"idle"') expect(res.text()).toBe(F1 + F2) expect(res.text()).not.toContain("local\u001f") - expect(res.text()).not.toContain("local=8") + expect(res.text()).not.toContain(`local=${localId(8)}`) expect(localCtl.resumeCalls).toBeGreaterThan(0) localCtl.end() @@ -587,8 +663,8 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]) - const L1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1") - const L2 = sseFrame("event: message", 'data: {"from":"macbook","n":2}', "id: l2") + const L1 = localEvt(1, { from: "macbook", n: 1 }) + const L2 = localEvt(2, { from: "macbook", n: 2 }) const opened: Array<{ namespace: string; lastEventId?: string }> = [] const localCtls: Ctl[] = [] @@ -616,12 +692,12 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco }) const res = mockRes() - driver.handle(mockReq("/event?lastEventID=local=l0;studio=p0"), res) + driver.handle(mockReq(`/event?lastEventID=local=${localId(0)};studio=${peerId(0)}`), res) await tick() // initial local open used the connect cursor l0 const initialLocalOpen = opened.find((o) => o.namespace === LOCAL_NAMESPACE) - expect(initialLocalOpen?.lastEventId).toBe("l0") + expect(initialLocalOpen?.lastEventId).toBe(localId(0)) // deliver two local frames — the aggregator's live local cursor advances to l2 localCtls[0].push(L1) @@ -640,23 +716,30 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco const reopenLocal = opened.filter((o) => o.namespace === LOCAL_NAMESPACE) expect(reopenLocal.length).toBe(2) - expect(reopenLocal[1].lastEventId).toBe("l2") // <── last-delivered, not l0 + expect(reopenLocal[1].lastEventId).toBe(localId(2)) // <── last-delivered, not the connect-time id localCtls.forEach((c) => c.end()) peerCtl.end() res.fireClose() }) - it("COMPOSITE MODE: re-opening a dropped peer arm emits exactly one amicode.sync.gap naming it", async () => { + // #1773 AC1: in BOTH engine modes — an older peer engine sends no `id:` lines, + // and its re-open must gap all the same ("delivered" = any data frame). + // #1777: the sequenced peer has engine ids to resume from, so its re-open + // RESUMES from the last delivered id instead — no gap (the replay covers it). + for (const mode of ["sequenced", "legacy"] as const) + it(`COMPOSITE MODE (${mode} peer engine): re-opening a dropped peer arm ${mode === "legacy" ? "emits exactly one amicode.sync.gap naming it" : "resumes from its last delivered id, with no gap"}`, async () => { const ownerMap = new SessionOwnerMap() ownerMap.update([ { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]) - const P1 = sseFrame("event: message", 'data: {"from":"studio","n":1}', "id: p1") + const studio = mode === "legacy" ? fakeEngine({ legacy: true }) : fakeEngine({ epoch: PEER_EPOCH }) + const P1 = studio.event("message.part.updated", { from: "studio", n: 1 }) const localCtl = controllableSource() const peerCtls: Ctl[] = [] + const peerOpens: Array = [] const driver = new SseFanInDriver({ ownerMap, @@ -670,6 +753,7 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco reconcileMs: 999_999, openUpstream: (r) => { if (r.namespace === LOCAL_NAMESPACE) return localCtl.source + peerOpens.push(r.lastEventId) const ctl = controllableSource() peerCtls.push(ctl) return ctl.source @@ -687,29 +771,40 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco const beforeGap = (res.text().match(/amicode\.sync\.gap/g) ?? []).length expect(beforeGap).toBe(0) // initial open emitted no gap - // drop the peer arm, then reconcile re-opens it → exactly one gap + // drop the peer arm, then reconcile re-opens it peerCtls[0].end() await tick() driver.reconcile() await tick() const gapMatches = res.text().match(/event: amicode\.sync\.gap/g) ?? [] - expect(gapMatches.length).toBe(1) - const gapFrame = res.text().split("\n\n").find((f) => f.includes("amicode.sync.gap"))! - const dataLine = gapFrame.split("\n").find((l) => l.startsWith("data:"))! - const parsed = JSON.parse(dataLine.slice("data:".length).trim()) - expect(parsed.namespaces).toContain("studio") + if (mode === "sequenced") { + expect(peerOpens).toEqual([undefined, studio.idAt(1)]) // resumed from the last delivered id + expect(gapMatches.length).toBe(0) + } else { + expect(peerOpens).toEqual([undefined, undefined]) // nothing to resume from + expect(gapMatches.length).toBe(1) + const gapFrame = res.text().split("\n\n").find((f) => f.includes("amicode.sync.gap"))! + const dataLine = gapFrame.split("\n").find((l) => l.startsWith("data:"))! + const parsed = JSON.parse(dataLine.slice("data:".length).trim()) + expect(parsed.namespaces).toContain("studio") + } localCtl.end() peerCtls.forEach((c) => c.end()) res.fireClose() }) - it("FLEET-OF-ONE (zero peers): a re-opened LOCAL arm emits NO gap and stays byte-identical (#1264)", async () => { + // #1773 AC2 supersedes #1638's "the solo-mode arm re-opens silently": a silent + // re-open loses the stretch between the arm's end and its re-subscription with + // nothing to announce it. The #1264 guards stay: no synthetic frame, and the + // stream is byte-identical to what the local source delivered while connected. + // #1777: the sever is the fallback for a local engine WITHOUT ids (older than + // #1776) — a sequenced one resumes instead (the #1777 suite) — so L1 is legacy. + it("FLEET-OF-ONE (zero peers): a re-opened LOCAL arm emits NO gap, stays byte-identical (#1264), and severs the listener (#1773)", async () => { const ownerMap = new SessionOwnerMap() // no owners → fleet-of-one - const L1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1") - const L2 = sseFrame("event: message", 'data: {"from":"macbook","n":2}', "id: l2") + const L1 = legacyEvent({ from: "macbook", n: 1 }) const localCtls: Ctl[] = [] @@ -738,23 +833,25 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco await tick() driver.reconcile() await tick() - // the re-opened arm delivers another frame - localCtls[localCtls.length - 1].push(L2) - await tick() - // NO synthetic frame anywhere; the verbatim stream is exactly the two frames. + // NO synthetic frame anywhere; the verbatim stream is exactly what was delivered. expect(res.text()).not.toContain("amicode.sync.gap") expect(res.text()).not.toContain("local\u001f") - expect(res.text()).toBe(L1 + L2) // byte-identical, in order + expect(res.text()).toBe(L1) // byte-identical + // the loss is announced by severing: a transport error, not a clean end + expect(res.destroyed).toBe(true) + expect(res.destroyError).toBeInstanceOf(Error) + expect(res.ended).toBe(false) + expect(localCtls).toHaveLength(1) // and the severed listener opens nothing more localCtls.forEach((c) => c.end()) res.fireClose() }) - it("a re-opened LOCAL arm in fleet-of-one resumes from the last-delivered id too", async () => { + it("fleet-of-one cursor bookkeeping: a never-delivered arm re-opens from the connect cursor; a delivered one resumes from its last-delivered id, and so does a reconnecting listener", async () => { const ownerMap = new SessionOwnerMap() - const L1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1") + const L1 = localEvt(1, { from: "macbook", n: 1 }) const opened: Array<{ namespace: string; lastEventId?: string }> = [] const localCtls: Ctl[] = [] @@ -775,21 +872,1962 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco }) const res = mockRes() - driver.handle(mockReq("/event?lastEventID=l0"), res) + driver.handle(mockReq(`/event?lastEventID=${localId(0)}`), res) await tick() - expect(opened[0].lastEventId).toBe("l0") // initial: connect cursor + expect(opened[0].lastEventId).toBe(localId(0)) // initial: connect cursor - localCtls[0].push(L1) + // the arm fails before delivering anything: nothing was lost → a plain re-open, + // still resumed from the connect-time cursor (the #1638 fallback) + localCtls[0].end("error") await tick() - localCtls[0].end() + driver.reconcile() + await tick() + expect(opened).toHaveLength(2) + expect(opened[1].lastEventId).toBe(localId(0)) + expect(res.destroyed).toBe(false) + + // #1777: the re-opened arm delivers, then drops → it RESUMES from the last + // delivered id (the engine replays the stretch) — the listener is not severed + localCtls[1].push(L1) + await tick() + localCtls[1].end() await tick() driver.reconcile() await tick() + expect(res.destroyed).toBe(false) + expect(opened).toHaveLength(3) + expect(opened[2].lastEventId).toBe(localId(1)) // last-delivered, not the stale connect id + + // the client's cursor is the last SSE id it saw on the verbatim stream … + const lastSeen = [...res.text().matchAll(/^id: (.*)$/gm)].at(-1)?.[1] + expect(lastSeen).toBe(localId(1)) + // … and its reconnect resumes the local arm from exactly that id + res.fireClose() + const again = mockRes() + driver.handle(mockReq(`/event?lastEventID=${lastSeen}`), again) + await tick() + expect(opened[3].lastEventId).toBe(localId(1)) - const localOpens = opened.filter((o) => o.namespace === LOCAL_NAMESPACE) - expect(localOpens[1].lastEventId).toBe("l1") // re-open resumes from last-delivered + localCtls.forEach((c) => c.end()) + again.fireClose() + }) +}) + +// ══════════════════════════════════════════════════════════════════════════════ +// #1773 AC2 — FLEET-OF-ONE loss is signalled by SEVERING, never by a frame. A +// synthetic gap frame may not enter the solo pass-through stream (#1264), so a +// re-opened local arm (one that delivered data) ends the listener's stream with a +// transport error: the client registers the disconnect and resyncs. While +// connected, the stream stays byte-identical to the local source. Engine-shaped +// fixtures in both modes (legacy id-less, sequenced). +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1773 AC2 — fleet-of-one: a re-opened local arm severs the listener with a transport error", () => { + // #1777: the sever is the fallback for a local engine without ids (legacy); a + // sequenced engine's arm RESUMES from its last delivered id instead — the + // replay announces nothing because nothing was lost, and the listener lives. + for (const mode of ["legacy", "sequenced"] as const) { + for (const why of ["end", "error"] as const) { + const title = + mode === "legacy" + ? `${mode} engine, upstream ${why}: no synthetic frame, byte-identical while connected, then an error close named ${why === "end" ? "upstream-end" : "upstream-error"}` + : `${mode} engine, upstream ${why}: no synthetic frame, and the arm resumes from its last id — no sever` + it(title, async () => { + const { census, lines } = testCensus() + const localCtls: Ctl[] = [] + const opened: Array = [] + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + census, + openUpstream: (r) => { + opened.push(r.lastEventId) + const ctl = controllableSource() + localCtls.push(ctl) + return ctl.source + }, + }) + const engine = mode === "legacy" ? fakeEngine({ legacy: true }) : fakeEngine({ epoch: LOCAL_EPOCH }) + const res = mockRes() + driver.handle(mockReq("/global/event"), res) + const frames = [ + engine.connected(), + engine.event("session.status", { sessionID: "ses_1", status: { type: "busy" } }), + ...engine.stored("message.part.updated", { part: { id: "prt_1", text: "ciao" } }, { aggregateID: "ses_1" }), + engine.heartbeat(), + ] + for (const f of frames) localCtls[0].push(f) + await flush() + expect(res.text()).toBe(frames.join("")) // connected: byte-identical (#1264) + + localCtls[0].end(why) // the local upstream is lost … + await flush() + expect(res.destroyed).toBe(false) // … nothing is decided until the re-open + driver.reconcile() // the re-open: resume, or sever — never re-open silently + await flush() + + expect(res.text()).toBe(frames.join("")) // no synthetic frame was injected + const reason = why === "end" ? "upstream-end" : "upstream-error" + expect(lines).toContain(`[fanin] arm-closed plane=observe conn=c1 arm=local reason=${reason}`) + if (mode === "sequenced") { + expect(res.destroyed).toBe(false) + expect(opened).toEqual([undefined, engine.idAt(engine.seq)]) + expect(lines.at(-1)).toBe("[fanin] arm-opened plane=observe conn=c1 arm=local opens=2") + expect(census.snapshot()).toMatchObject({ listeners: 1, accepted: 1, closed: 0 }) + res.fireClose() + return + } + expect(res.destroyed).toBe(true) + expect(res.destroyError).toBeInstanceOf(Error) + expect(res.ended).toBe(false) // an error close, never a clean end + expect(localCtls).toHaveLength(1) // the severed listener re-opened nothing + + // the census names the listener's close by its cause (#1771's set) + expect(lines).toContain(`[fanin] listener-closed plane=observe conn=c1 reason=${reason} listeners=0`) + expect(census.snapshot()).toMatchObject({ listeners: 0, accepted: 1, closed: 1, evictions: 0 }) + }) + } + } + it("a local arm that never delivered a data frame re-opens normally — no sever (nothing was lost)", async () => { + const localCtls: Ctl[] = [] + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + openUpstream: () => { + const ctl = controllableSource() + localCtls.push(ctl) + return ctl.source + }, + }) + const res = mockRes() + driver.handle(mockReq("/global/event"), res) + localCtls[0].push(": keep-alive\n\n") // a comment is not data + await flush() + localCtls[0].end("error") + await flush() + driver.reconcile() + await flush() + expect(res.destroyed).toBe(false) + expect(localCtls).toHaveLength(2) + const f = fakeEngine({ legacy: true }).connected() + localCtls[1].push(f) + await flush() + expect(res.text()).toBe(": keep-alive\n\n" + f) localCtls.forEach((c) => c.end()) res.fireClose() }) + + it("severing one listener leaves another listener's stream untouched", async () => { + const localCtls: Ctl[] = [] + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + openUpstream: () => { + const ctl = controllableSource() + localCtls.push(ctl) + return ctl.source + }, + }) + const engine = fakeEngine({ legacy: true }) + const resA = mockRes() + const resB = mockRes() + driver.handle(mockReq("/global/event"), resA) // localCtls[0] + driver.handle(mockReq("/global/event"), resB) // localCtls[1] + const c = engine.connected() + localCtls[0].push(c) + localCtls[1].push(c) + await flush() + localCtls[0].end() // only A's upstream is lost + await flush() + driver.reconcile() + await flush() + expect(resA.destroyed).toBe(true) + expect(resB.destroyed).toBe(false) + const h = engine.heartbeat() + localCtls[1].push(h) + await flush() + expect(resB.text()).toBe(c + h) + localCtls.forEach((x) => x.end()) + resB.fireClose() + }) + + it("over REAL sockets: the listener's response errors (no clean end) after exactly the frames the local engine sent", async () => { + const engine = fakeEngine({ legacy: true }) // an old engine: id-less frames + const sent = [engine.connected(), engine.event("session.status", { sessionID: "ses_1", status: { type: "idle" } })] + const upstream = await startEngineSseServer((res) => { + for (const f of sent) res.write(f) + setImmediate(() => res.end()) // the local engine ends its stream (a restart) + }) + const lines: string[] = [] + const w = lineWaiter(lines) + const driver = new SseFanInDriver({ + ownerMap: new SessionOwnerMap(), + localMachineId: "macbook", + localEventUrl: () => upstream.url, + peerBaseUrl: () => undefined, + peerToken: () => ({ ok: false as const, reason: "absent" as const }), + reconcileMs: 999_999, + census: new FanInCensus({ plane: "observe", sink: w.sink }), + }) + const front = http.createServer((req, res) => void driver.handle(req, res)) + await new Promise((r) => front.listen(0, "127.0.0.1", r)) + const { port } = front.address() as import("node:net").AddressInfo + try { + let body = "" + let haveAll: () => void = () => undefined + const received = new Promise((r) => (haveAll = r)) + const outcome = new Promise<"end" | "error">((resolve) => { + http.get(`http://127.0.0.1:${port}/global/event`, (res) => { + res.setEncoding("utf8") + res.on("data", (c: string) => { + body += c + if (body === sent.join("")) haveAll() + }) + res.on("end", () => resolve("end")) + res.on("error", () => resolve("error")) + }) + }) + await w.waitFor((l) => l.includes("arm=local reason=upstream-end")) + await received // the listener holds every frame (production's reconcile is a later timer tick) + driver.reconcile() // the re-open → sever + expect(await outcome).toBe("error") // a transport error, never a clean end of stream + expect(body).toBe(sent.join("")) // byte-identical while connected, nothing injected + expect(await w.waitFor((l) => l.startsWith("[fanin] listener-closed"))).toContain("reason=upstream-end") + } finally { + front.closeAllConnections?.() + await new Promise((r) => front.close(() => r())) + await upstream.stop() + } + }) }) + +// ══════════════════════════════════════════════════════════════════════════════ +// #1772 — the fan-in serves EVERY listener. Each global-stream listener gets its +// own FanInConnection (aggregator + upstream arms + reconcile); a second view +// (another window, a Chat Deck pane, a split pane) no longer evicts the first. +// The harness models the upstream origins as BROADCAST buses (the engine's +// global bus delivers every event to every open subscription), so a listener +// whose arms were torn down by an eviction visibly stops receiving frames. +// ══════════════════════════════════════════════════════════════════════════════ + +/** A fake set of upstream origins: every `openUpstream` call is a NEW + * subscription on its namespace's bus; `broadcast` delivers a frame to every + * OPEN subscription of that namespace. Records every open and every close. */ +interface FakeOrigins { + openUpstream: (r: { namespace: string; lastEventId?: string }) => SseFrameSource; + broadcast(namespace: string, frame: string): void; + opens(namespace: string): number; + closes(namespace: string): number; + live(namespace: string): number; +} +function fakeOrigins(): FakeOrigins { + const subs = new Map>(); + const openCount = new Map(); + const closeCount = new Map(); + const bump = (m: Map, ns: string) => m.set(ns, (m.get(ns) ?? 0) + 1); + return { + openUpstream(r) { + const ctl = controllableSource(); + const set = subs.get(r.namespace) ?? new Set(); + subs.set(r.namespace, set); + set.add(ctl); + bump(openCount, r.namespace); + return { + next: () => ctl.source.next(), + close: () => { + if (set.delete(ctl)) bump(closeCount, r.namespace); + ctl.end(); + }, + }; + }, + broadcast(namespace, frame) { + for (const ctl of subs.get(namespace) ?? []) ctl.push(frame); + }, + opens: (ns) => openCount.get(ns) ?? 0, + closes: (ns) => closeCount.get(ns) ?? 0, + live: (ns) => subs.get(ns)?.size ?? 0, + }; +} + +const STUDIO_OWNED = [ + { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, +]; + +function twoListenerDriver(origins: FakeOrigins, ownerMap: SessionOwnerMap, extra: Partial[0]> = {}): SseFanInDriver { + return new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => "http://local.invalid", + peerBaseUrl: (id) => (id === "studio" ? "http://studio.invalid" : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: "http://studio.invalid", token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 999_999, + openUpstream: origins.openUpstream, + ...extra, + }); +} + +describe("#1772 AC1 — two concurrent listeners both stay connected and both receive every frame", () => { + it("listener A and listener B each receive the local arm's AND the peer arm's frames; neither is ended", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap); + + const resA = mockRes(); + const resB = mockRes(); + expect(driver.handle(mockReq("/global/event"), resA)).toBe(true); + await tick(); + expect(driver.handle(mockReq("/global/event"), resB)).toBe(true); + await tick(); + + origins.broadcast(LOCAL_NAMESPACE, localEvt(1, { from: "macbook", n: 1 })); + origins.broadcast("studio", peerEvt(1, { from: "studio", n: 2 })); + await tick(); + + for (const res of [resA, resB]) { + expect(res.ended).toBe(false); // still connected — no eviction + expect(res.text()).toContain('"from":"macbook"'); + expect(res.text()).toContain('"from":"studio"'); + } + + resA.fireClose(); + resB.fireClose(); + }); +}); + +describe("#1772 AC2 — closing one listener closes ONLY that listener's upstream arms", () => { + it("A departs → exactly A's local + peer arms close; B keeps its arms and keeps streaming, uninterrupted", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap); + + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + await tick(); + driver.handle(mockReq("/global/event"), resB); + await tick(); + // Both listeners hold their own local + peer arm. + expect(origins.live(LOCAL_NAMESPACE)).toBe(2); + expect(origins.live("studio")).toBe(2); + + origins.broadcast(LOCAL_NAMESPACE, localEvt(1, { n: "before" })); + await tick(); + + // Listener A departs (its socket closes). + resA.fireClose(); + await tick(); + + // Exactly A's two arms closed; B's two arms untouched. + expect(origins.closes(LOCAL_NAMESPACE)).toBe(1); + expect(origins.closes("studio")).toBe(1); + expect(origins.live(LOCAL_NAMESPACE)).toBe(1); + expect(origins.live("studio")).toBe(1); + + // B keeps streaming on the SAME arms (no re-open), from both origins. + driver.reconcile(); + await tick(); + origins.broadcast(LOCAL_NAMESPACE, localEvt(2, { n: "after-local" })); + origins.broadcast("studio", peerEvt(2, { n: "after-peer" })); + await tick(); + expect(resB.ended).toBe(false); + expect(resB.text()).toContain('"n":"before"'); + expect(resB.text()).toContain('"n":"after-local"'); + expect(resB.text()).toContain('"n":"after-peer"'); + expect(origins.opens(LOCAL_NAMESPACE)).toBe(2); // one per listener, never re-opened + expect(origins.opens("studio")).toBe(2); + // The departed listener receives nothing further. + expect(resA.text()).not.toContain('"n":"after-local"'); + + resB.fireClose(); + await tick(); + expect(origins.live(LOCAL_NAMESPACE)).toBe(0); + expect(origins.live("studio")).toBe(0); + }); +}); + +/** Poll until `cond` holds (or time out) — for the cadence-driven tests, where + * each connection's own reconcile timer (not a manual call) does the work. */ +async function until(cond: () => boolean, timeoutMs = 2000): Promise { + const start = Date.now(); + while (!cond() && Date.now() - start < timeoutMs) await tick(5); + return cond(); +} + +describe("#1772 AC3 — reconcile updates EVERY live connection's membership", () => { + it("driver.reconcile() opens a newly-owned peer's arm on BOTH listeners, and closes it on both when it goes dark", async () => { + const ownerMap = new SessionOwnerMap(); // fleet-of-one at connect + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap); + + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await tick(); + expect(origins.opens("studio")).toBe(0); + + ownerMap.update(STUDIO_OWNED); + driver.reconcile(); + await tick(); + expect(origins.live("studio")).toBe(2); // one peer arm per listener + origins.broadcast("studio", peerEvt(1, { from: "studio" })); + await tick(); + expect(resA.text()).toContain('"from":"studio"'); + expect(resB.text()).toContain('"from":"studio"'); + + ownerMap.update([]); + driver.reconcile(); + await tick(); + expect(origins.live("studio")).toBe(0); // both listeners dropped the dark peer + expect(origins.live(LOCAL_NAMESPACE)).toBe(2); // local arms untouched (§D4) + + resA.fireClose(); + resB.fireClose(); + }); + + it("on its CADENCE (no manual reconcile): both listeners pick up a new owner, then drop it", async () => { + const ownerMap = new SessionOwnerMap(); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap, { reconcileMs: 20 }); + + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await tick(); + + ownerMap.update(STUDIO_OWNED); + expect(await until(() => origins.live("studio") === 2)).toBe(true); + origins.broadcast("studio", peerEvt(1, { from: "studio" })); + await tick(); + expect(resA.text()).toContain('"from":"studio"'); + expect(resB.text()).toContain('"from":"studio"'); + + ownerMap.update([]); + expect(await until(() => origins.live("studio") === 0)).toBe(true); + + resA.fireClose(); + resB.fireClose(); + // A closed listener's timer is stopped: a later owner change opens nothing. + const opensAfterClose = origins.opens("studio"); + ownerMap.update(STUDIO_OWNED); + await tick(80); + expect(origins.opens("studio")).toBe(opensAfterClose); + }); +}); + +describe("#1772 AC4 — a listener beyond the cap is refused (retryable), never by evicting one", () => { + it("default cap 16: the 17th listener gets 503 + Retry-After, opens no arm, and all 16 keep streaming; a freed slot is reusable", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap); + + const accepted: MockRes[] = []; + for (let i = 0; i < 16; i++) { + const res = mockRes(); + expect(driver.handle(mockReq("/global/event"), res)).toBe(true); + accepted.push(res); + } + await tick(); + expect(origins.live(LOCAL_NAMESPACE)).toBe(16); + + // The 17th is ANSWERED (handle owns the response) with a retryable refusal. + const refused = mockRes(); + expect(driver.handle(mockReq("/global/event"), refused)).toBe(true); + await tick(); + expect(refused.status).toBe(503); + expect(Number(refused.headerMap["retry-after"])).toBeGreaterThan(0); + expect(refused.ended).toBe(true); + const body = JSON.parse(refused.body) as { ok: boolean; reason: string; max_listeners: number }; + expect(body.ok).toBe(false); + expect(body.reason).toBe("listener-cap"); + expect(body.max_listeners).toBe(16); + + // No existing listener was closed to make room, and the refused one opened nothing. + expect(origins.opens(LOCAL_NAMESPACE)).toBe(16); + expect(origins.closes(LOCAL_NAMESPACE)).toBe(0); + expect(origins.live("studio")).toBe(16); + origins.broadcast(LOCAL_NAMESPACE, localEvt(1, { n: "after-refusal" })); + await tick(); + for (const res of accepted) { + expect(res.ended).toBe(false); + expect(res.text()).toContain('"n":"after-refusal"'); + } + expect(refused.text()).toBe(""); + + // The cap bounds CONCURRENT listeners: once one departs, a retry is accepted. + accepted[0].fireClose(); + await tick(); + const retry = mockRes(); + driver.handle(mockReq("/global/event"), retry); + await tick(); + expect(retry.status).toBe(200); + expect(retry.ended).toBe(false); + expect(origins.live(LOCAL_NAMESPACE)).toBe(16); + + for (const res of [...accepted, retry]) res.fireClose(); + }); +}); + +/** #1772 — a VIEW that behaves like the app's stream loop: when the driver ENDS + * its stream (an eviction is a clean end), the socket closes and the view + * reconnects after `delayMs` (the client's reconnect delay, shortened). Two + * such views against an evicting driver re-open against each other forever — + * the Studio's ~340-reconnects-in-15-minutes storm. */ +function reconnectingView(driver: SseFanInDriver, delayMs = 5): { connects(): number; current(): MockRes; stop(): void } { + let stopped = false; + let connects = 0; + let current!: MockRes; + const connect = (): void => { + if (stopped) return; + connects += 1; + const res = mockRes(); + const end = res.end.bind(res); + res.end = ((chunk?: string) => { + end(chunk); + setTimeout(() => { + res.fireClose(); + connect(); + }, delayMs); + }) as typeof res.end; + current = res; + driver.handle(mockReq("/global/event"), res); + }; + connect(); + return { + connects: () => connects, + current: () => current, + stop() { + stopped = true; + current.fireClose(); + }, + }; +} + +describe("#1772 AC5 — two listeners: one local arm each, kept open — no repeating re-open pattern", () => { + it("two reconnecting views over many live reconcile ticks: 2 local arms opened, 0 closed, 0 view reconnects", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap, { reconcileMs: 10 }); + + const viewA = reconnectingView(driver); + await tick(); + const viewB = reconnectingView(driver); + await tick(250); // ~25 reconcile ticks per connection + + try { + expect(viewA.connects()).toBe(1); + expect(viewB.connects()).toBe(1); + expect(origins.opens(LOCAL_NAMESPACE)).toBe(2); // one local arm per listener + expect(origins.closes(LOCAL_NAMESPACE)).toBe(0); // …and each is kept open + expect(origins.live(LOCAL_NAMESPACE)).toBe(2); + expect(origins.opens("studio")).toBe(2); + expect(origins.closes("studio")).toBe(0); + + // Both views are live on those same arms. + origins.broadcast(LOCAL_NAMESPACE, localEvt(1, { n: "steady" })); + await tick(); + expect(viewA.current().text()).toContain('"n":"steady"'); + expect(viewB.current().text()).toContain('"n":"steady"'); + } finally { + viewA.stop(); + viewB.stop(); + } + }); +}); + +describe("#1772 invariants — #1264 / #1601 / #1617 hold PER LISTENER", () => { + it("#1264 fleet-of-one: each of two listeners receives the local stream byte-identical", async () => { + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, new SessionOwnerMap()); + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await tick(); + + // #1771: the engine's real stream — an old (id-less) engine and a #1776 + // (sequenced) one — connected, an event, the plain + sync twin of a stored + // event, a heartbeat. Every byte relays verbatim to every listener. + for (const engine of [fakeEngine({ legacy: true }), fakeEngine({ epoch: LOCAL_EPOCH })]) { + const frames = [ + engine.connected(), + engine.event("session.status", { a: 1 }), + ...engine.stored("message.updated", { b: 2 }, { aggregateID: "ses_1" }), + engine.heartbeat(), + ]; + const before = [resA.text(), resB.text()]; + for (const f of frames) origins.broadcast(LOCAL_NAMESPACE, f); + await tick(); + expect(resA.text()).toBe(before[0] + frames.join("")); + expect(resB.text()).toBe(before[1] + frames.join("")); + } + + resA.fireClose(); + resB.fireClose(); + }); + + it("#1601 per connection: one listener's dead local arm is re-opened for THAT listener only", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const localCtls: Ctl[] = []; + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { + openUpstream: (r) => { + const ctl = controllableSource(); + if (r.namespace === LOCAL_NAMESPACE) localCtls.push(ctl); + return ctl.source; + }, + }); + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); // localCtls[0] + driver.handle(mockReq("/global/event"), resB); // localCtls[1] + await tick(); + expect(localCtls.length).toBe(2); + + localCtls[0].end(); // A's local upstream drops + await tick(); + driver.reconcile(); + await tick(); + expect(localCtls.length).toBe(3); // exactly one re-open — A's + + localCtls[2].push(localEvt(2, { to: "A" })); + localCtls[1].push(localEvt(1, { to: "B" })); + await tick(); + expect(resA.text()).toContain('"to":"A"'); + expect(resA.text()).not.toContain('"to":"B"'); + expect(resB.text()).toContain('"to":"B"'); + expect(resB.text()).not.toContain('"to":"A"'); + + localCtls.forEach((c) => c.end()); + resA.fireClose(); + resB.fireClose(); + }); + + it("#1617 per connection: a backpressured listener pauses only ITS upstreams and heals on ITS drain", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const ctls: PausableCtl[] = []; + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { + openUpstream: () => { + const ctl = pausableSource(); + ctls.push(ctl); + return ctl.source; + }, + }); + const resA = backpressureRes(); + driver.handle(mockReq("/global/event"), resA); // ctls[0] local, ctls[1] studio + await tick(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resB); // ctls[2] local, ctls[3] studio + await tick(); + const [aLocal, aPeer, bLocal, bPeer] = ctls; + + resA.block(); + aLocal.push(localEvt(1, { n: 1 })); // trips A's backpressure + await tick(); + aLocal.push(localEvt(2, { n: 2 })); // stranded behind it + await tick(); + expect(aLocal.pauseCalls).toBeGreaterThan(0); + expect(aPeer.pauseCalls).toBeGreaterThan(0); + expect(bLocal.pauseCalls).toBe(0); // B is not throttled by A's slow socket + expect(bPeer.pauseCalls).toBe(0); + + bLocal.push(localEvt(1, { to: "B" })); + await tick(); + expect(resB.text()).toContain('"to":"B"'); // B streams on while A is backpressured + expect(resA.text()).not.toContain('"n":2'); + + resA.fireDrain(); + await tick(); + expect(resA.text()).toContain('"n":2'); + expect(aLocal.resumeCalls).toBeGreaterThan(0); + + ctls.forEach((c) => c.end()); + resA.fireClose(); + resB.fireClose(); + }); +}); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1771 — the fan-in CENSUS. A wedged, evicted, or flapping arm used to look +// identical to a quiet one in every log: nothing recorded who subscribed, why an +// arm re-opened, or what was dropped. Every fan-in service now keeps counters per +// listener connection and per arm, logs one census line every 30 s, and one line +// immediately on each transition. The upstream frames below are ENGINE-SHAPED +// (test/fixtures/engine_stream): id-less or `:` data frames, the +// plain + sync twin of a stored event, heartbeats. +// ══════════════════════════════════════════════════════════════════════════════ + +/** A census wired to a collecting sink and a hand-driven clock. */ +function testCensus(plane = "observe") { + const lines: string[] = []; + let t = 1_000_000; + const census = new FanInCensus({ plane, sink: (l) => lines.push(l), now: () => t }); + return { + census, + lines, + advance(ms: number) { + t += ms; + }, + censusLines: () => lines.filter((l) => l.startsWith("[fanin] census ")), + }; +} + +/** One `key=value key=value` segment → a record. */ +function kv(segment: string): Record { + const out: Record = {}; + for (const tok of segment.trim().split(" ")) { + const i = tok.indexOf("="); + if (i > 0) out[tok.slice(0, i)] = tok.slice(i + 1); + } + return out; +} + +/** A census line: its service-level head, then one segment per (connection, arm). */ +function parseCensus(line: string): { head: Record; arms: Array> } { + const [head, ...arms] = line.split(" | "); + return { head: kv(head), arms: arms.map(kv) }; +} + +/** Drain the pumps: every pending microtask runs before the immediate fires — + * no wall-clock sleep. */ +const flush = () => new Promise((r) => setImmediate(r)); + +const bytesOf = (frames: string[]) => frames.reduce((n, f) => n + Buffer.byteLength(f), 0); + +describe("#1771 AC1 — a fan-in service logs a census line every 30 s", () => { + afterEach(() => vi.useRealTimers()); + + it("listeners, accepted/closed since boot, evictions, and per arm: frames, bytes, largest, time since last frame, opens, closes", async () => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const { census, advance, censusLines } = testCensus(); + census.start(); + const driver = twoListenerDriver(origins, ownerMap, { census }); + const local = fakeEngine({ epoch: LOCAL_EPOCH }); + const studio = fakeEngine({ epoch: PEER_EPOCH }); + + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await flush(); + + const localFrames = [ + local.connected(), + local.event("session.status", { sessionID: "ses_1", status: { type: "busy" } }), + ...local.stored("message.part.updated", { part: { id: "prt_1", text: "bravo" } }, { aggregateID: "ses_1" }), + local.heartbeat(), + ]; + for (const f of localFrames) origins.broadcast(LOCAL_NAMESPACE, f); + const studioFrame = studio.event("message.part.updated", { from: "studio" }); + origins.broadcast("studio", studioFrame); + await flush(); + resA.fireClose(); // listener A departs; B stays + await flush(); + advance(1500); + + expect(censusLines()).toHaveLength(0); // nothing before the first period + vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS); + expect(censusLines()).toHaveLength(1); + + const { head, arms } = parseCensus(censusLines()[0]); + expect(head).toMatchObject({ plane: "observe", listeners: "1", accepted: "2", closed: "1", evictions: "0" }); + // Only the LIVE listener's arms are listed: B's local arm and B's peer arm. + expect(arms.map((a) => a.arm).sort()).toEqual(["local", "studio"]); + expect(new Set(arms.map((a) => a.conn)).size).toBe(1); + const bLocal = arms.find((a) => a.arm === "local")!; + expect(bLocal).toMatchObject({ + frames: "5", + bytes: String(bytesOf(localFrames)), + largest: String(Math.max(...localFrames.map((f) => Buffer.byteLength(f)))), + since_last_ms: "1500", + opens: "1", + closes: "-", + pending: "0", + drops: "0", + gaps: "0", + }); + expect(arms.find((a) => a.arm === "studio")).toMatchObject({ + frames: "1", + bytes: String(Buffer.byteLength(studioFrame)), + largest: String(Buffer.byteLength(studioFrame)), + since_last_ms: "1500", + opens: "1", + closes: "-", + }); + + // The cadence repeats every 30 s, and stops with the census. + vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS); + expect(censusLines()).toHaveLength(2); + census.stop(); + vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS * 3); + expect(censusLines()).toHaveLength(2); + + resB.fireClose(); + }); + + it("per arm: pending depth while backpressured, overflow drops, and gaps emitted", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); // its sources cannot pause → frames keep arriving + const { census, censusLines } = testCensus(); + const driver = twoListenerDriver(origins, ownerMap, { census }); + const studio = fakeEngine({ epoch: PEER_EPOCH }); + + const res = backpressureRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + const studioArm = () => parseCensus(censusLines().at(-1)!).arms.find((a) => a.arm === "studio")!; + + res.block(); + // 1 trip-frame written, 256 buffered (the per-namespace bound), 43 dropped. + for (let i = 0; i < 300; i++) origins.broadcast("studio", studio.event("message.part.delta", { i })); + await flush(); + census.emit(); + expect(studioArm()).toMatchObject({ frames: "300", pending: "256", drops: "43", gaps: "0" }); + + res.fireDrain(); // the buffer flushes in order, then ONE overflow gap names the arm + await flush(); + census.emit(); + expect(studioArm()).toMatchObject({ frames: "300", pending: "0", drops: "43", gaps: "1" }); + + res.fireClose(); + }); +}); + +/** Resolve once the census sink has received a line matching `pred` — + * event-driven (the sink notifies), never a polling sleep. */ +function lineWaiter(lines: string[]): { sink: (l: string) => void; waitFor(pred: (l: string) => boolean): Promise } { + const waiters: Array<{ pred: (l: string) => boolean; resolve: (l: string) => void }> = []; + return { + sink(l: string) { + lines.push(l); + for (const w of [...waiters]) { + if (w.pred(l)) { + waiters.splice(waiters.indexOf(w), 1); + w.resolve(l); + } + } + }, + waitFor(pred) { + const hit = lines.find(pred); + if (hit !== undefined) return Promise.resolve(hit); + return new Promise((resolve) => waiters.push({ pred, resolve })); + }, + }; +} + +describe("#1771 AC2 — close reasons are an enumerated set", () => { + it("the set is exactly: upstream end, upstream error, membership change, listener closed, evicted, shutdown", () => { + expect([...FANIN_CLOSE_REASONS]).toEqual([ + "upstream-end", + "upstream-error", + "membership-change", + "listener-closed", + "evicted", + "shutdown", + ]); + }); + + it("an arm's closes are counted under their reason: upstream end, upstream error, membership change", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const { census, lines, censusLines } = testCensus(); + const localCtls: Ctl[] = []; + const failing: Array<(err: Error) => void> = []; + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { + census, + openUpstream: (r) => { + if (r.namespace === LOCAL_NAMESPACE) { + const ctl = controllableSource(); + localCtls.push(ctl); + return ctl.source; + } + // the peer's upstream FAILS: next() rejects + return { + next: () => new Promise((_, reject) => failing.push(reject)), + close() {}, + }; + }, + }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + + localCtls[0].end(); // the local engine ends the stream cleanly + failing.shift()!(new Error("ECONNRESET")); // the peer's stream errors + await flush(); + expect(lines).toContainEqual(expect.stringMatching(/^\[fanin\] arm-closed plane=observe conn=c1 arm=local reason=upstream-end$/)); + expect(lines).toContainEqual(expect.stringMatching(/^\[fanin\] arm-closed plane=observe conn=c1 arm=studio reason=upstream-error$/)); + + driver.reconcile(); // both re-open + await flush(); + ownerMap.update([]); // the peer's sessions end → its arm leaves the membership + driver.reconcile(); + await flush(); + expect(lines).toContainEqual(expect.stringMatching(/^\[fanin\] arm-closed plane=observe conn=c1 arm=studio reason=membership-change$/)); + + census.emit(); + const arms = parseCensus(censusLines().at(-1)!).arms; + expect(arms.find((a) => a.arm === "local")).toMatchObject({ opens: "2", closes: "upstream-end:1" }); + expect(arms.find((a) => a.arm === "studio")).toMatchObject({ opens: "2", closes: "upstream-error:1,membership-change:1" }); + expect(parseCensus(censusLines().at(-1)!).head.evictions).toBe("0"); // no eviction path exists (#1772) + + localCtls.forEach((c) => c.end()); + res.fireClose(); + }); + + it("the REAL upstream reader tells a clean end (upstream-end) from a mid-stream failure (upstream-error)", async () => { + const local = fakeEngine({ legacy: true }); // an old engine: id-less frames + const peer = fakeEngine({ legacy: true }); + const ends = await startEngineSseServer((res) => { + res.write(local.connected()); + res.end(); // clean end of stream + }); + const dies = await startEngineSseServer((res) => { + res.write(peer.connected()); + setImmediate(() => res.destroy()); // the link dies mid-stream + }); + const lines: string[] = []; + const w = lineWaiter(lines); + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const driver = new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => ends.url, + peerBaseUrl: (id) => (id === "studio" ? dies.url : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: dies.url, token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 999_999, + census: new FanInCensus({ plane: "observe", sink: w.sink }), + // no openUpstream → the production http reader + }); + const res = mockRes(); + try { + driver.handle(mockReq("/global/event"), res); + expect(await w.waitFor((l) => l.includes("arm=local reason="))).toMatch(/reason=upstream-end$/); + expect(await w.waitFor((l) => l.includes("arm=studio reason="))).toMatch(/reason=upstream-error$/); + } finally { + res.fireClose(); + await ends.stop(); + await dies.stop(); + } + }); + + // ──────────────────────────────────────────────────────────────────────────── + // #1773 AC4 — the arm reader (openHttpSseSource) reports upstream END and + // upstream ERROR as distinct reasons. A non-200 upstream (an auth failure, an + // overloaded peer, the beyond-cap 503) is an ERROR, not a clean end — and its + // body is never parsed as SSE frames. + // ──────────────────────────────────────────────────────────────────────────── + describe("#1773 AC4 — the arm reader's end-vs-error", () => { + /** Drain a source to its end: the frames it yielded, and its endReason. */ + async function drain(source: SseFrameSource): Promise<{ frames: string[]; reason: string | undefined }> { + const frames: string[] = []; + for (;;) { + const f = await source.next(); + if (f === null) break; + frames.push(f); + } + return { frames, reason: source.endReason?.() }; + } + const arm = (url: string) => openHttpSseSource({ namespace: "studio", url, path: "/global/event" }); + + it("a clean upstream end reads as `end`, after every frame", async () => { + const engine = fakeEngine({ epoch: PEER_EPOCH }); + const sent = [engine.connected(), engine.event("session.status", { s: 1 })]; + const server = await startEngineSseServer((res) => { + for (const f of sent) res.write(f); + res.end(); + }); + try { + expect(await drain(arm(server.url))).toEqual({ frames: sent, reason: "end" }); + } finally { + await server.stop(); + } + }); + + it("an upstream that dies mid-stream reads as `error`, after the frames it sent", async () => { + const engine = fakeEngine({ legacy: true }); + const sent = [engine.connected()]; + const server = await startEngineSseServer((res) => { + for (const f of sent) res.write(f); + setTimeout(() => res.destroy(), 20); + }); + try { + expect(await drain(arm(server.url))).toEqual({ frames: sent, reason: "error" }); + } finally { + await server.stop(); + } + }); + + for (const status of [401, 500, 503]) { + it(`a ${status} upstream is an \`error\` — never a clean end, and its body is not read as frames`, async () => { + const server = await startEngineSseServer( + (res) => res.end(`{"ok":false,"error":"refused"}\n\ndata: {"not":"a frame"}\n\n`), + { status }, + ); + try { + expect(await drain(arm(server.url))).toEqual({ frames: [], reason: "error" }); + } finally { + await server.stop(); + } + }); + } + + it("a refused connection is an `error`", async () => { + // port 1 (tcpmux) on loopback: nothing listens there, the dial is refused + expect(await drain(arm("http://127.0.0.1:1"))).toEqual({ frames: [], reason: "error" }); + }); + + it("through the driver: a peer answering 401 closes its arm as upstream-error in the census", async () => { + const local = await startEngineSseServer((res) => res.write(fakeEngine({ legacy: true }).connected())); + const peer = await startEngineSseServer((res) => res.end(`{"ok":false}`), { status: 401 }); + const lines: string[] = []; + const w = lineWaiter(lines); + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const driver = new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => local.url, + peerBaseUrl: (id) => (id === "studio" ? peer.url : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: peer.url, token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 999_999, + census: new FanInCensus({ plane: "observe", sink: w.sink }), + }); + const res = mockRes(); + try { + driver.handle(mockReq("/global/event"), res); + expect(await w.waitFor((l) => l.includes("arm=studio reason="))).toMatch(/reason=upstream-error$/); + } finally { + res.fireClose(); + await local.stop(); + await peer.stop(); + } + }); + }); + + it("a departing listener's arms close as listener-closed; after shutdown() the teardown's closes are shutdown", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const { census, lines } = testCensus(); + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { census }); + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await flush(); + + resA.fireClose(); // a view goes away + expect(lines.filter((l) => l.includes("conn=c1") && l.includes("reason="))).toEqual([ + "[fanin] arm-closed plane=observe conn=c1 arm=local reason=listener-closed", + "[fanin] arm-closed plane=observe conn=c1 arm=studio reason=listener-closed", + "[fanin] listener-closed plane=observe conn=c1 reason=listener-closed listeners=1", + ]); + + driver.shutdown(); // the service stops: its socket teardown closes the rest + resB.fireClose(); + expect(lines.filter((l) => l.includes("conn=c2") && l.includes("reason="))).toEqual([ + "[fanin] arm-closed plane=observe conn=c2 arm=local reason=shutdown", + "[fanin] arm-closed plane=observe conn=c2 arm=studio reason=shutdown", + "[fanin] listener-closed plane=observe conn=c2 reason=shutdown listeners=0", + ]); + // shutdown() closed nothing itself — B's stream ended only with its socket. + expect(lines.some((l) => l.includes("reason=evicted"))).toBe(false); + }); + + // #1773 AC3 — a fan-in connection never ends its listener's stream cleanly + // except on a graceful service shutdown: every other close reason is an ERROR + // close (the listener, or the relay in front of it, sees a transport error). + it("#1773 AC3: a listener-closed connection is an ERROR close; a shutdown is the only clean end", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const driver = twoListenerDriver(fakeOrigins(), ownerMap); + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await flush(); + + resA.fireClose(); // listener-closed + expect(resA.destroyed).toBe(true); + expect(resA.destroyError).toBeInstanceOf(Error); + expect(resA.destroyError?.message).toContain("listener-closed"); + expect(resA.ended).toBe(false); + + driver.shutdown(); + resB.fireClose(); // shutdown + expect(resB.ended).toBe(true); + expect(resB.destroyed).toBe(false); + }); + + it("#1773 AC3: the severed listener's error close names its cause (upstream-error) — census and error agree", async () => { + const { census, lines } = testCensus(); + const localCtls: Ctl[] = []; + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + census, + openUpstream: () => { + const ctl = controllableSource(); + localCtls.push(ctl); + return ctl.source; + }, + }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + localCtls[0].push(fakeEngine({ legacy: true }).connected()); // #1777: an id-less engine — the sever fallback + await flush(); + localCtls[0].end("error"); + await flush(); + driver.reconcile(); + expect(res.destroyError?.message).toContain("upstream-error"); + expect(lines.at(-1)).toBe("[fanin] listener-closed plane=observe conn=c1 reason=upstream-error listeners=0"); + // the socket's own close event afterwards changes nothing (closed once, first reason wins) + res.fireClose(); + expect(lines.filter((l) => l.includes("listener-closed")).length).toBe(1); + expect(census.snapshot()).toMatchObject({ listeners: 0, closed: 1 }); + }); +}); + +describe("#1771 AC3 — each transition logs one line immediately, with its connection and arm", () => { + it("listener accepted → arm opened (local, peer) — synchronously, before any frame or timer", () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const { census, lines } = testCensus(); + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { census }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + // no flush, no timer: the lines are already there + expect(lines).toEqual([ + "[fanin] listener-accepted plane=observe conn=c1 listeners=1", + "[fanin] arm-opened plane=observe conn=c1 arm=local opens=1", + "[fanin] arm-opened plane=observe conn=c1 arm=studio opens=1", + ]); + res.fireClose(); + }); + + // #1777: an id-less (legacy) arm's re-open logs its gap; a sequenced arm's + // re-open RESUMES — closed → opened, and no gap line at all. + for (const mode of ["legacy", "sequenced"] as const) + it(`frames are never logged; ${mode === "legacy" ? "an id-less arm that drops and re-opens logs closed → gap emitted → opened" : "a sequenced arm that drops and re-opens logs closed → opened (resumed, no gap)"}`, async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const { census, lines } = testCensus(); + const peerCtls: Ctl[] = []; + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { + census, + openUpstream: (r) => { + const ctl = controllableSource(); + if (r.namespace === "studio") peerCtls.push(ctl); + return ctl.source; + }, + }); + const studio = mode === "legacy" ? fakeEngine({ legacy: true }) : fakeEngine({ epoch: PEER_EPOCH }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + const before = lines.length; + for (let i = 0; i < 25; i++) peerCtls[0].push(studio.event("message.part.delta", { i })); + peerCtls[0].push(studio.heartbeat()); + await flush(); + expect(lines.length).toBe(before); // 26 frames delivered, zero lines + + peerCtls[0].end(); + await flush(); + driver.reconcile(); + expect(lines.slice(before)).toEqual([ + "[fanin] arm-closed plane=observe conn=c1 arm=studio reason=upstream-end", + ...(mode === "legacy" ? ["[fanin] gap-emitted plane=observe conn=c1 arm=studio cause=reopen"] : []), + "[fanin] arm-opened plane=observe conn=c1 arm=studio opens=2", + ]); + peerCtls.forEach((c) => c.end()); + res.fireClose(); + }); + + it("an overflow episode logs ONE overflow-drop line (not one per dropped frame), then its gap on drain", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const { census, lines } = testCensus(); + const driver = twoListenerDriver(origins, ownerMap, { census }); + const studio = fakeEngine({ epoch: PEER_EPOCH }); + const res = backpressureRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + const before = lines.length; + + res.block(); + for (let i = 0; i < 300; i++) origins.broadcast("studio", studio.event("message.part.delta", { i })); + await flush(); + expect(lines.slice(before)).toEqual(["[fanin] overflow-drop plane=observe conn=c1 arm=studio pending=256"]); + + res.fireDrain(); + await flush(); + expect(lines.slice(before + 1)).toEqual(["[fanin] gap-emitted plane=observe conn=c1 arm=studio cause=overflow"]); + res.fireClose(); + }); + + it("a beyond-cap refusal logs one line naming the cap (and counts in the census)", () => { + const { census, lines } = testCensus(); + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { census, maxListeners: 1 }); + const ok = mockRes(); + driver.handle(mockReq("/global/event"), ok); + driver.handle(mockReq("/global/event"), mockRes()); + expect(lines.at(-1)).toBe("[fanin] listener-refused plane=observe listeners=1 cap=1"); + expect(census.snapshot()).toMatchObject({ listeners: 1, accepted: 1, refusals: 1, evictions: 0 }); + ok.fireClose(); + }); +}); + +describe("#1771 AC5 — the census adds no frames to any stream and changes no delivery", () => { + afterEach(() => vi.useRealTimers()); + + it("FLEET-OF-ONE under a live census (cadence ticking, every transition logging): the stream is the engine's frames, byte-identical (#1264)", async () => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + const { census, lines } = testCensus(); + census.start(); + const localCtls: Ctl[] = []; + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + census, + openUpstream: () => { + const ctl = controllableSource(); + localCtls.push(ctl); + return ctl.source; + }, + }); + for (const engine of [fakeEngine({ legacy: true }), fakeEngine({ epoch: LOCAL_EPOCH })]) { + localCtls.length = 0; + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + const first = [engine.connected(), engine.event("session.status", { status: "busy" }), ...engine.stored("message.part.updated", { text: "ciao" }, { aggregateID: "ses_1" })]; + for (const f of first) localCtls[0].push(f); + await flush(); + vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS); // a census line mid-stream + localCtls[0].end(); // the arm drops … + await flush(); + driver.reconcile(); // … and the re-open severs the listener (#1773 — no gap frame in fleet-of-one) or, #1777, resumes it + const second = [engine.heartbeat(), engine.event("session.status", { status: "idle" })]; + if (engine.epoch !== undefined) { + // a sequenced engine: the arm resumed beneath the SAME listener — still + // the engine's frames, byte for byte + expect(res.destroyed).toBe(false); + for (const f of second) localCtls[1].push(f); + await flush(); + expect(res.text()).toBe(first.join("") + second.join("")); + res.fireClose(); + continue; + } + expect(res.text()).toBe(first.join("")); + expect(res.destroyed).toBe(true); + // the client reconnects; the stream resumes on the new listener + const again = mockRes(); + driver.handle(mockReq("/global/event"), again); + for (const f of second) localCtls[1].push(f); + await flush(); + expect(again.text()).toBe(second.join("")); + again.fireClose(); + } + expect(lines.some((l) => l.startsWith("[fanin] census "))).toBe(true); + expect(lines.some((l) => l.includes("arm-closed"))).toBe(true); + }); + + it("COMPOSITE: the same scripted sequence (overflow, gap, membership change, re-open) is byte-identical with and without a census sink", async () => { + async function run(sink: ((l: string) => void) | undefined): Promise { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap, sink ? { census: new FanInCensus({ sink, now: () => 0 }) } : {}); + const local = fakeEngine({ legacy: true }); // an old local engine: id-less + const studio = fakeEngine({ epoch: PEER_EPOCH }); + const res = backpressureRes(); + driver.handle(mockReq(`/global/event?lastEventID=studio=${studio.idAt(0)}`), res); + await flush(); + for (const f of [local.connected(), ...local.stored("message.updated", { n: 1 }, { aggregateID: "ses_1" }), local.heartbeat()]) { + origins.broadcast(LOCAL_NAMESPACE, f); + } + origins.broadcast("studio", studio.connected()); + await flush(); + res.block(); + for (let i = 0; i < 300; i++) origins.broadcast("studio", studio.event("message.part.delta", { i })); + await flush(); + res.fireDrain(); + await flush(); + ownerMap.update([]); + driver.reconcile(); // membership change closes the peer arm + ownerMap.update(STUDIO_OWNED); + driver.reconcile(); // … and it re-opens (a reopen gap) + await flush(); + origins.broadcast("studio", studio.heartbeat()); + await flush(); + res.fireClose(); + return res.text(); + } + const lines: string[] = []; + const silent = await run(undefined); + const logged = await run((l) => lines.push(l)); + expect(logged).toBe(silent); + expect(lines.length).toBeGreaterThan(5); // the census did see it all … + expect(logged).not.toContain("[fanin]"); // … and none of it reached the stream + expect(logged).not.toContain("[owner-map]"); + }); + + it("a census sink that throws never breaks delivery", async () => { + const census = new FanInCensus({ + sink: () => { + throw new Error("log pipe closed"); + }, + }); + census.start(); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, new SessionOwnerMap(), { census }); + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const res = mockRes(); + expect(() => driver.handle(mockReq("/global/event"), res)).not.toThrow(); + const f = engine.event("session.status", { status: "busy" }); + origins.broadcast(LOCAL_NAMESPACE, f); + await flush(); + expect(() => census.emit()).not.toThrow(); + expect(res.text()).toBe(f); + res.fireClose(); + census.stop(); + }); +}); + +describe("#1771 AC6 — a 10 MB engine frame delivered in 64 KB chunks (the #1779 surface)", () => { + it("the real reader reassembles it whole; the stream carries it byte-identical; the census counts its bytes and largest frame", async () => { + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const connected = engine.connected(); + const big = largeSummaryFrame({ epoch: LOCAL_EPOCH, seq: 1 }); + const after = engine.heartbeat(); + const chunks = chunkBytes(connected + big + after, CHUNK_64K); + expect(Buffer.byteLength(big)).toBeGreaterThanOrEqual(TEN_MB); + expect(chunks.length).toBeGreaterThan(160); + const server = await startEngineSseServer(async (res) => { + await writeChunks(res, chunks); + res.end(); + }); + const lines: string[] = []; + const w = lineWaiter(lines); + const census = new FanInCensus({ plane: "observe", sink: w.sink, now: () => 0 }); + const driver = new SseFanInDriver({ + ownerMap: new SessionOwnerMap(), // fleet-of-one: the verbatim relay + localMachineId: "macbook", + localEventUrl: () => server.url, + peerBaseUrl: () => undefined, + peerToken: () => ({ ok: false as const, reason: "absent" as const }), + reconcileMs: 999_999, + census, + }); + const res = mockRes(); + try { + driver.handle(mockReq("/global/event"), res); + await w.waitFor((l) => l.includes("arm=local reason=")); + expect(res.text()).toBe(connected + big + after); // whole, in order, byte-identical + census.emit(); + const local = parseCensus(lines.at(-1)!).arms.find((a) => a.arm === "local")!; + expect(local).toMatchObject({ + frames: "3", + bytes: String(Buffer.byteLength(connected + big + after)), + largest: String(Buffer.byteLength(big)), + }); + } finally { + res.fireClose(); + await server.stop(); + } + }); +}); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1779 — LINEAR-TIME FRAME SPLITTING in the relay's upstream reader. The reader +// (`openHttpSseSource`) decodes each network chunk (`res.setEncoding("utf8")`, a +// StringDecoder: a code point split across chunks is held back, never mangled) +// and hands the text to its frame splitter. `readFrames` is that data path minus +// the socket, so a test can choose the chunk boundaries exactly — the network +// cannot deliver a 10 MB frame in ONE chunk, which is the baseline AC1 compares to. +// ══════════════════════════════════════════════════════════════════════════════ + +/** The relay reader's data path: decode each byte chunk as `setEncoding("utf8")` + * does, split the text into whole frames. */ +function readFrames(chunks: Buffer[]): string[] { + const decoder = new StringDecoder("utf8"); + const splitter = new SseFrameSplitter(); + const frames: string[] = []; + for (const chunk of chunks) { + const text = decoder.write(chunk); + if (text !== "") frames.push(...splitter.push(text)); + } + return frames; +} + +const median = (xs: number[]): number => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)]; +const elapsedMs = (fn: () => unknown): number => { + const t0 = performance.now(); + fn(); + return performance.now() - t0; +}; + +/** #1779 AC1's assertion, made robust: a warm-up, then INTERLEAVED trials (both + * deliveries see the same machine noise), the median of each, and a generous + * factor. A quadratic splitter misses it by an order of magnitude. `SPLIT_RATIO_LOG` + * (a file path) records each run's ratio for a flake report. */ +function splitTimeRatio(read: (chunks: Buffer[]) => unknown, whole: Buffer[], chunked: Buffer[]): { ratio: number; wholeMs: number; chunkedMs: number } { + for (let i = 0; i < 2; i++) { + read(whole); + read(chunked); + } + const one: number[] = []; + const many: number[] = []; + for (let i = 0; i < 7; i++) { + one.push(elapsedMs(() => read(whole))); + many.push(elapsedMs(() => read(chunked))); + } + const wholeMs = median(one); + const chunkedMs = median(many); + return { ratio: chunkedMs / wholeMs, wholeMs, chunkedMs }; +} +const SPLIT_RATIO_FACTOR = 3; + +describe("#1779 AC1 — the relay reader splits a 10 MB frame in 64 KB chunks within a small constant factor of one chunk", () => { + it("chunked ÷ single-chunk split time (medians) stays ≤ 3×, and both yield the one frame byte-identical", () => { + const big = largeSummaryFrame({ epoch: LOCAL_EPOCH, seq: 1 }); + const whole = [Buffer.from(big, "utf8")]; + const chunked = chunkBytes(big, CHUNK_64K); + expect(Buffer.byteLength(big)).toBeGreaterThanOrEqual(TEN_MB); + expect(chunked.length).toBeGreaterThan(160); + for (const delivery of [whole, chunked]) { + const frames = readFrames(delivery); + expect(frames).toHaveLength(1); + expect(frames[0] === big).toBe(true); // byte-identical (#1264); no 10 MB diff on failure + } + const { ratio, wholeMs, chunkedMs } = splitTimeRatio(readFrames, whole, chunked); + if (process.env.SPLIT_RATIO_LOG) appendFileSync(process.env.SPLIT_RATIO_LOG, `relay ${ratio.toFixed(3)} ${wholeMs.toFixed(2)} ${chunkedMs.toFixed(2)}\n`); + expect(ratio, `chunked ${chunkedMs.toFixed(1)} ms vs whole ${wholeMs.toFixed(1)} ms`).toBeLessThanOrEqual(SPLIT_RATIO_FACTOR); + }, 60_000); +}); + +/** Every way to cut `text`'s bytes into two chunks, and one byte at a time. */ +function everySplit(text: string): Buffer[][] { + const all = Buffer.from(text, "utf8"); + const deliveries: Buffer[][] = []; + for (let k = 1; k < all.length; k++) deliveries.push([all.subarray(0, k), all.subarray(k)]); + deliveries.push(Array.from({ length: all.length }, (_, i) => all.subarray(i, i + 1))); + return deliveries; +} + +/** CRLF framing (SSE allows it; a non-engine upstream or a proxy may use it). */ +const crlf = (frame: string): string => frame.replace(/\n/g, "\r\n"); + +describe("#1779 AC2 — the relay reader finds frame boundaries split across chunks, LF and CRLF alike, and relays each frame verbatim", () => { + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const frames = [ + engine.connected(), // LF, the engine's own framing + crlf(engine.event("message.part.updated", { part: { id: "prt_1", text: "résumé → done" } })), // CRLF, non-ASCII + `event: message\nid: ${localId(2)}\r\ndata: {"n":2}\n\r\n`, // CRLF blank line after LF lines + `: keep-alive\r\n\n`, // LF blank line after a CRLF line + crlf(engine.heartbeat()), + engine.event("session.status", { status: "idle" }), + ]; + const stream = frames.join(""); + + it("cut at EVERY byte (the delimiter, a CRLF, a code point split between two chunks), and byte by byte: the same frames, verbatim", () => { + expect(readFrames([Buffer.from(stream, "utf8")])).toEqual(frames); + for (const delivery of everySplit(stream)) { + const got = readFrames(delivery); + if (got.length !== frames.length || got.some((f, i) => f !== frames[i])) { + expect({ cuts: delivery.map((c) => c.length), got }).toEqual({ cuts: delivery.map((c) => c.length), got: frames }); + } + } + }); + + it("through the real reader and a fleet-of-one driver: a CRLF stream whose delimiters straddle network chunks reaches the listener byte-identical (#1264)", async () => { + const all = Buffer.from(stream, "utf8"); + const startOf = (i: number): number => frames.slice(0, i).join("").length; + const byteAt = (charIndex: number): number => Buffer.byteLength(stream.slice(0, charIndex)); + const cuts = [ + byteAt(startOf(1) + frames[1].indexOf("\r\n") + 1), // a CRLF line ending: "\r" | "\n" + byteAt(startOf(1) + frames[1].indexOf("é")) + 1, // inside a code point + byteAt(startOf(2) - 1), // the CRLF delimiter: "\r\n\r" | "\n" + byteAt(startOf(3) - 2), // "\n" | "\r\n" (LF line, CRLF blank line) + byteAt(startOf(5) - 2), // the CRLF delimiter: "\r\n" | "\r\n" + ].sort((a, b) => a - b); + const chunks: Buffer[] = []; + let at = 0; + for (const end of cuts) { + chunks.push(all.subarray(at, end)); + at = end; + } + chunks.push(all.subarray(at)); + expect(Buffer.concat(chunks).equals(all)).toBe(true); + const server = await startEngineSseServer(async (res) => { + for (const c of chunks) { + res.write(c); + await new Promise((r) => setImmediate(r)); // separate network reads (best effort; the cut-everywhere test above is exact) + } + res.end(); + }); + const lines: string[] = []; + const w = lineWaiter(lines); + const census = new FanInCensus({ plane: "observe", sink: w.sink, now: () => 0 }); + const driver = new SseFanInDriver({ + ownerMap: new SessionOwnerMap(), // fleet-of-one: the verbatim relay + localMachineId: "macbook", + localEventUrl: () => server.url, + peerBaseUrl: () => undefined, + peerToken: () => ({ ok: false as const, reason: "absent" as const }), + reconcileMs: 999_999, + census, + }); + const res = mockRes(); + try { + driver.handle(mockReq("/global/event"), res); + await w.waitFor((l) => l.includes("arm=local reason=")); + expect(res.text()).toBe(stream); + // … and frame by frame: the census saw exactly these frames (a reader + // blind to CRLF would have merged them — the bytes alone cannot tell) + census.emit(); + const local = parseCensus(lines.at(-1)!).arms.find((a) => a.arm === "local")!; + expect(local).toMatchObject({ + frames: String(frames.length), + largest: String(Math.max(...frames.map((f) => Buffer.byteLength(f)))), + }); + } finally { + res.fireClose(); + await server.stop(); + } + }); +}); + +describe("#1779 AC3 — the relay reader's framing is unchanged: multi-line data, event names, ids, retry fields — each frame verbatim", () => { + // Pinned against the splitter as it was before #1779, whole and cut at every + // byte: where a frame was cut never changes which frames come out. + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const connected = engine.connected(); + const [plain, twin] = engine.stored("message.updated", { info: { id: "msg_1" } }, { aggregateID: "ses_1" }); + const frames = [ + connected, + plain, // a durable event … + twin, // … and its sync twin: the same id + `: a comment\nevent: message.part.updated\nid: ${localId(2)}\nretry: 5000\ndata: {"a":\ndata: 1}\n\n`, // every field, multi-line data + `data:no-space\n\n`, + `id:\ndata: {"b":2}\n\n`, // an empty id (it advances nothing) + `: keep-alive\n\n`, // comment-only (not a data frame) + `\ndata: {"c":3}\n\n`, // a stray blank line joins the frame after it + `\n\n`, // … and alone it is an empty frame + engine.heartbeat(), + ]; + const stream = frames.join(""); + + it("whole and cut at every byte: the same frames, verbatim, with the ids and data the aggregator reads", () => { + for (const delivery of [[Buffer.from(stream, "utf8")], ...everySplit(stream)]) { + const got = readFrames(delivery); + if (got.length !== frames.length || got.some((f, i) => f !== frames[i])) { + expect({ cuts: delivery.map((c) => c.length), got }).toEqual({ cuts: delivery.map((c) => c.length), got: frames }); + } + } + expect(frames.map(frameRawId)).toEqual([localId(0), localId(1), localId(1), localId(2), undefined, undefined, undefined, undefined, undefined, localId(1)]); + expect(frames.map(isDataFrame)).toEqual([true, true, true, true, true, true, false, true, false, true]); + }); +}); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1777 — LOSSLESS RESUME. The engine (#1776) sequences and replays its global +// stream; the fan-in now uses it. A re-opened arm resumes from its last delivered +// engine id, so the upstream REPLAYS what the listener missed — no gap, no resync. +// Only what the replay cannot serve is announced (an upstream reset → one gap; +// an old id-less engine → the #1773 gap / sever fallback). The upstream here is +// a REPLAYING engine (test/fixtures/engine_stream): it answers a cursor the way +// the #1776 route does — `server.connected` at the cursor, then the replay — or +// with the `server.cursor.unservable` reset first. +// ══════════════════════════════════════════════════════════════════════════════ + +/** The downstream stream as a client parses it: per frame, its `id:` and JSON data. */ +function sseFrames(text: string): Array<{ id?: string; data?: Record; type?: string }> { + const out: Array<{ id?: string; data?: Record; type?: string }> = []; + for (const block of text.split("\n\n")) { + if (block.trim() === "") continue; + const id = /^id: (.*)$/m.exec(block)?.[1]; + const dataLine = block.split("\n").find((l) => l.startsWith("data:")); + const data = dataLine ? (JSON.parse(dataLine.slice(5).trim()) as Record) : undefined; + const type: string | undefined = data ? (data.payload?.type ?? data.type) : undefined; + out.push({ ...(id !== undefined ? { id } : {}), ...(data ? { data } : {}), ...(type ? { type } : {}) }); + } + return out; +} + +/** A driver whose arms subscribe to in-process replaying engines. */ +function replayDriver( + engines: { local: ReplayingEngine; studio?: ReplayingEngine }, + ownerMap: SessionOwnerMap, + extra: Partial[0]> = {}, +): SseFanInDriver { + return new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => "http://local.invalid", + peerBaseUrl: (id) => (id === "studio" ? "http://studio.invalid" : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: "http://studio.invalid", token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 999_999, + openUpstream: (r) => (r.namespace === LOCAL_NAMESPACE ? engines.local : engines.studio!).subscribe(r.lastEventId).source, + ...extra, + }); +} + +const partEvent = (n: number) => ["message.part.updated", { part: { id: `prt_${n}`, text: `line ${n}` }, n }] as const; + +describe("#1777 AC1 — a peer arm that drops and re-opens within the replay window resumes: every missed frame, no gap, no resync", () => { + for (const why of ["end", "error"] as const) { + it(`upstream ${why}: the re-open resumes from the last delivered id and the viewer gets every frame exactly once, in order`, async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + const { census, lines } = testCensus(); + const driver = replayDriver({ local, studio }, ownerMap, { census }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + + for (const n of [1, 2, 3]) studio.publish(...partEvent(n)); // mid-turn + await flush(); + studio.subscriptions[0].end(why); // the peer link drops … + await flush(); + for (const n of [4, 5, 6]) studio.publish(...partEvent(n)); // … while the turn goes on + driver.reconcile(); // the re-open + await flush(); + for (const n of [7, 8]) studio.publish(...partEvent(n)); + await flush(); + + // the re-open asked for the last delivered id, and the engine could serve it + expect(studio.subscriptions).toHaveLength(2); + expect(studio.subscriptions[1].cursor).toBe(studio.idAt(3)); + expect(studio.subscriptions[1].reset).toBe(false); + + const frames = sseFrames(res.text()); + // every frame the peer published, exactly once, in order + expect(frames.filter((f) => f.type === "message.part.updated").map((f) => f.data!.payload.properties.n)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); + // no gap signal … + expect(frames.some((f) => f.type === "amicode.sync.gap")).toBe(false); + expect(lines.some((l) => l.includes("gap-emitted"))).toBe(false); + // … and no resync cue: the resumed arm's own handshake is not an event of + // the listener's stream (only the two initial arms' connected frames show) + expect(frames.filter((f) => f.type === "server.connected")).toHaveLength(2); + // the listener stays connected, and its cursor carries the peer's position + expect(res.destroyed).toBe(false); + expect(parseCompositeCursor(frames.at(-1)!.id).get("studio")).toBe(studio.idAt(8)); + res.fireClose(); + }); + } +}); + +const RESTARTED_EPOCH = "c3R1ZGlvMg"; // the peer's engine after a restart: a new process epoch + +describe("#1777 AC2 — an upstream reset on any arm is exactly one gap for that namespace downstream", () => { + const gapsOf = (text: string) => sseFrames(text).filter((f) => f.type === "amicode.sync.gap").map((f) => f.data!.namespaces); + + it("composite: a peer engine that restarted answers the resume with a reset → one gap naming it; the reset frame itself is not forwarded; the arm re-bases in the new epoch", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + const engines: { local: ReplayingEngine; studio: ReplayingEngine } = { local, studio }; + const { census, lines } = testCensus(); + const driver = replayDriver(engines, ownerMap, { census }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + for (const n of [1, 2]) studio.publish(...partEvent(n)); + await flush(); + studio.subscriptions[0].end("error"); // the link drops — and the peer's engine restarts + await flush(); + const restarted = replayingEngine({ epoch: RESTARTED_EPOCH }); + engines.studio = restarted; + restarted.publish(...partEvent(3)); // lost with the old process's buffer: unrecoverable + driver.reconcile(); + await flush(); + + expect(restarted.subscriptions[0]).toMatchObject({ cursor: studio.idAt(2), reset: true }); + expect(gapsOf(res.text())).toEqual([["studio"]]); // exactly one gap, naming the peer + expect(sseFrames(res.text()).some((f) => f.type === "server.cursor.unservable")).toBe(false); + expect(lines).toContain("[fanin] gap-emitted plane=observe conn=c1 arm=studio cause=reset"); + expect(res.destroyed).toBe(false); + + // re-based: the next frames carry the new epoch, and a later drop resumes in it + restarted.publish(...partEvent(4)); + await flush(); + expect(parseCompositeCursor(sseFrames(res.text()).at(-1)!.id).get("studio")).toBe(restarted.idAt(2)); + restarted.subscriptions[0].end(); + await flush(); + driver.reconcile(); + await flush(); + expect(restarted.subscriptions[1]).toMatchObject({ cursor: restarted.idAt(2), reset: false }); + expect(gapsOf(res.text())).toHaveLength(1); // still one + res.fireClose(); + }); + + it("composite: a drop longer than the replay window (the cursor was evicted) → one gap naming the peer", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + const driver = replayDriver({ local, studio }, ownerMap); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + studio.publish(...partEvent(1)); + await flush(); + studio.subscriptions[0].end("error"); + await flush(); + for (const n of [2, 3, 4]) studio.publish(...partEvent(n)); + studio.evictThrough(3); // the window moved past the cursor (seq 1) + driver.reconcile(); + await flush(); + expect(studio.subscriptions[1].reset).toBe(true); + expect(gapsOf(res.text())).toEqual([["studio"]]); + res.fireClose(); + }); + + it("composite: the LOCAL arm's reset (the local engine restarted) is one gap naming `local`", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const engines: { local: ReplayingEngine; studio: ReplayingEngine } = { + local: replayingEngine({ epoch: LOCAL_EPOCH }), + studio: replayingEngine({ epoch: PEER_EPOCH }), + }; + const driver = replayDriver(engines, ownerMap); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + engines.local.publish(...partEvent(1)); + await flush(); + engines.local.subscriptions[0].end(); + await flush(); + engines.local = replayingEngine({ epoch: "bG9jYWwy" }); + driver.reconcile(); + await flush(); + expect(gapsOf(res.text())).toEqual([[LOCAL_NAMESPACE]]); + res.fireClose(); + }); + + it("composite: a listener reconnecting with a cursor its peer can no longer serve gets one gap for that peer — and a replay for the servable one", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + for (const n of [1, 2]) local.publish(...partEvent(n)); + const driver = replayDriver({ local, studio }, ownerMap); + const res = mockRes(); + // a cursor from before the peer's restart: `studio` names a dead epoch + driver.handle(mockReq(`/global/event?lastEventID=local=${local.idAt(1)};studio=${RESTARTED_EPOCH}:9`), res); + await flush(); + expect(local.subscriptions[0]).toMatchObject({ cursor: local.idAt(1), reset: false }); + expect(studio.subscriptions[0]).toMatchObject({ cursor: `${RESTARTED_EPOCH}:9`, reset: true }); + expect(gapsOf(res.text())).toEqual([["studio"]]); + // the local replay arrived: frame 2 (the one after the cursor) + expect(sseFrames(res.text()).filter((f) => f.type === "message.part.updated").map((f) => f.data!.payload.properties.n)).toEqual([2]); + res.fireClose(); + }); + + it("fleet-of-one: the engine's reset frame passes VERBATIM (it is the engine's own signal — #1264: nothing synthetic), and the listener lives", async () => { + const engines: { local: ReplayingEngine } = { local: replayingEngine({ epoch: LOCAL_EPOCH }) }; + const sent: string[] = []; + const driver = replayDriver(engines as { local: ReplayingEngine }, new SessionOwnerMap(), { + openUpstream: (r) => { + const sub = engines.local.subscribe(r.lastEventId); + const next = sub.source.next; + sub.source.next = async () => { + const f = await next(); + if (f !== null) sent.push(f); + return f; + }; + return sub.source; + }, + }); + const res = mockRes(); + driver.handle(mockReq(`/global/event?lastEventID=${LOCAL_EPOCH}:4`), res); // stale: the engine is at 0 + await flush(); + engines.local.publish(...partEvent(1)); + await flush(); + engines.local.subscriptions[0].end(); // mid-stream: the local engine restarts + await flush(); + engines.local = replayingEngine({ epoch: "bG9jYWwy" }); + driver.reconcile(); // resume → the new process resets + await flush(); + engines.local.publish(...partEvent(2)); + await flush(); + + expect(res.text()).toBe(sent.join("")); // byte-identical to what the engine sent, ids included + const types = sseFrames(res.text()).map((f) => f.type); + expect(types.filter((t) => t === "server.cursor.unservable")).toHaveLength(2); // the stale connect cursor, then the restart + expect(types).not.toContain("amicode.sync.gap"); + expect(res.destroyed).toBe(false); + res.fireClose(); + }); +}); + +/** Serve a replaying engine's `/global/event` over a REAL socket (127.0.0.1, + * port 0, dialed at its own address): each request subscribes with its + * `?lastEventID=` and writes the subscription's frames as they come. Records + * every byte written per connection; `drop()` destroys the open connections + * (a link loss — the reader sees an error, not a clean end). */ +async function serveReplayingEngine(engine: ReplayingEngine) { + const written: string[][] = []; + const open = new Set(); + const server = http.createServer((req, res) => { + const cursor = new URL(req.url ?? "/", "http://engine").searchParams.get("lastEventID") ?? undefined; + const sub = engine.subscribe(cursor); + const mine: string[] = []; + written.push(mine); + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache, no-transform" }); + res.flushHeaders(); + open.add(res); + res.on("close", () => { + open.delete(res); + sub.end("error"); + }); + void (async () => { + for (;;) { + const f = await sub.source.next(); + if (f === null) return; + mine.push(f); + res.write(f); + } + })(); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const { port } = server.address() as import("node:net").AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + written, + drop: () => { + for (const res of open) res.destroy(); + }, + stop: () => + new Promise((r) => { + server.close(() => r()); + server.closeAllConnections?.(); + }), + }; +} + +describe("#1777 AC3 — fleet-of-one: the listener's stream is byte-identical to the engine's, ids included", () => { + it("over REAL sockets, across a link loss and the replayed re-open: every byte the engine wrote, and nothing else", async () => { + const engine = replayingEngine({ epoch: LOCAL_EPOCH, now: () => 1_700_000_000_000 }); + const upstream = await serveReplayingEngine(engine); + const lines: string[] = []; + const w = lineWaiter(lines); + const driver = new SseFanInDriver({ + ownerMap: new SessionOwnerMap(), // fleet-of-one: the verbatim relay + localMachineId: "macbook", + localEventUrl: () => upstream.url, + peerBaseUrl: () => undefined, + peerToken: () => ({ ok: false as const, reason: "absent" as const }), + reconcileMs: 999_999, + census: new FanInCensus({ plane: "observe", sink: w.sink }), + }); + const front = http.createServer((req, res) => void driver.handle(req, res)); + await new Promise((r) => front.listen(0, "127.0.0.1", r)); + const { port } = front.address() as import("node:net").AddressInfo; + let body = ""; + let wake: () => void = () => undefined; + // event-driven (each listener chunk wakes it); the deadline only names a regression + const until = async (pred: () => boolean, what: string) => { + const deadline = Date.now() + 3_000; + while (!pred()) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}; the listener got:\n${body}`); + await new Promise((r) => { + wake = r; + setTimeout(r, 100); + }); + } + }; + const listener = http.get(`http://127.0.0.1:${port}/global/event`, (res) => { + res.setEncoding("utf8"); + res.on("data", (c: string) => { + body += c; + wake(); + }); + res.on("error", () => undefined); + }); + listener.on("error", () => undefined); + const engineBytes = () => upstream.written.flat().join(""); + try { + await until(() => body.includes("server.connected"), "the connected frame"); + for (const n of [1, 2]) engine.publish(...partEvent(n)); + engine.heartbeat(); + await until(() => body === engineBytes() && body.includes("server.heartbeat"), "the first connection's frames"); + + upstream.drop(); // the link dies mid-stream + await w.waitFor((l) => l.includes("arm=local reason=upstream-error")); + for (const n of [3, 4]) engine.publish(...partEvent(n)); // missed while down + driver.reconcile(); // resume from the last id the listener has + await until(() => upstream.written.length === 2 && body.includes('"n":4'), "the replay"); + engine.publish(...partEvent(5)); + await until(() => body.includes('"n":5'), "the live frame after the replay"); + + expect(upstream.written).toHaveLength(2); + expect(body).toBe(engineBytes()); // byte-identical, ids included, across both connections + const ids = [...body.matchAll(/^id: (.*)$/gm)].map((m) => m[1]); + expect(ids.every((i) => /^bG9jYWwx:\d+$/.test(i))).toBe(true); // the engine's own bare ids — never a composite + expect(body).not.toContain("amicode.sync.gap"); + expect(body).not.toContain("local="); + } finally { + listener.destroy(); + front.closeAllConnections?.(); + await new Promise((r) => front.close(() => r())); + await upstream.stop(); + } + }); +}); + +describe("#1777 AC4 — a reconnecting listener resumes with the SSE id it last saw (composite when fanned in)", () => { + it("the last composite id a fanned-in listener saw resumes EVERY arm from its own engine id — each engine replays, no gap", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + const driver = replayDriver({ local, studio }, ownerMap); + const first = mockRes(); + driver.handle(mockReq("/global/event"), first); + await flush(); + local.publish(...partEvent(1)); + studio.publish(...partEvent(2)); + local.publish(...partEvent(3)); + await flush(); + const lastSeen = sseFrames(first.text()).at(-1)!.id!; // the client's cursor + expect(lastSeen).toBe(`local=${local.idAt(2)};studio=${studio.idAt(1)}`); + first.fireClose(); // the view reloads / its socket drops + + studio.publish(...partEvent(4)); // both origins move on while it is away + local.publish(...partEvent(5)); + const again = mockRes(); + driver.handle(mockReq(`/global/event?lastEventID=${encodeURIComponent(lastSeen)}`), again); + await flush(); + + expect(local.subscriptions.at(-1)).toMatchObject({ cursor: local.idAt(2), reset: false }); + expect(studio.subscriptions.at(-1)).toMatchObject({ cursor: studio.idAt(1), reset: false }); + const frames = sseFrames(again.text()); + expect(frames.filter((f) => f.type === "message.part.updated").map((f) => f.data!.payload.properties.n).sort()).toEqual([4, 5]); + expect(frames.some((f) => f.type === "amicode.sync.gap")).toBe(false); + // every frame's id carries both origins' positions, ending at the newest + expect(parseCompositeCursor(frames.at(-1)!.id)).toEqual( + new Map([ + [LOCAL_NAMESPACE, local.idAt(3)], + ["studio", studio.idAt(2)], + ]), + ); + again.fireClose(); + }); +}); diff --git a/packages/extension/tools/fleet/co.harmoniqs.amico-peer-forward.plist b/packages/extension/tools/fleet/co.harmoniqs.amico-peer-forward.plist new file mode 100644 index 000000000..6b07593c6 --- /dev/null +++ b/packages/extension/tools/fleet/co.harmoniqs.amico-peer-forward.plist @@ -0,0 +1,36 @@ + + + + + KeepAlive + + Label + co.harmoniqs.amico-peer-PEER_MACHINE_ID-forward + ProgramArguments + + /usr/bin/ssh + -N + -o + ExitOnForwardFailure=yes + -o + ServerAliveInterval=15 + -o + ServerAliveCountMax=2 + -o + TCPKeepAlive=yes + -o + BatchMode=yes + -L + 127.0.0.1:PEER_LOCAL_PORT:127.0.0.1:PEER_REMOTE_PORT + PEER_SSH_ALIAS + + RunAtLoad + + StandardErrorPath + /tmp/amico-peer-PEER_MACHINE_ID-forward.log + StandardOutPath + /tmp/amico-peer-PEER_MACHINE_ID-forward.log + ThrottleInterval + 10 + + diff --git a/packages/extension/tools/fleet/install.sh b/packages/extension/tools/fleet/install.sh index 171cd8449..9471381bc 100755 --- a/packages/extension/tools/fleet/install.sh +++ b/packages/extension/tools/fleet/install.sh @@ -10,6 +10,8 @@ # branch (stated + the grant pointer), IDENTICAL to a CLI-absent machine; # other non-zero exits die honestly — never a silent raw-file fallthrough. # Installs/updates the fleet guard + tunnel on this host and fixes machine-scoped settings. +# #1775: also generates one SSH forward agent per attached peer (replacing a +# hand-installed one with a backup) and retires the session-wide multiplex-flag agent. # Usage: # bash tools/fleet/install.sh # install/repair (writes files, reloads launchd) # bash tools/fleet/install.sh --check # check only (no writes), exit 1 on drift @@ -17,8 +19,13 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" GUARD_SRC="$REPO_ROOT/tools/fleet/amico-opencode-fleet-guard" GUARD_DST="$HOME/.local/bin/amico-opencode-fleet-guard" +# #1775: the machine-scoped launchd seams — the LaunchAgents dir and the +# launchctl binary. Both default to today's behavior; tests point them at a +# temp dir and a recording stub so no run ever touches the real service manager. +LAUNCH_AGENTS_DIR="${AMICO_FLEET_LAUNCH_AGENTS_DIR:-$HOME/Library/LaunchAgents}" +LAUNCHCTL="${AMICO_FLEET_LAUNCHCTL:-launchctl}" PLIST_SRC="$REPO_ROOT/tools/fleet/co.harmoniqs.amico-tunnel.plist" -PLIST_DST="$HOME/Library/LaunchAgents/co.harmoniqs.amico-tunnel.plist" +PLIST_DST="$LAUNCH_AGENTS_DIR/co.harmoniqs.amico-tunnel.plist" SETTINGS="$HOME/Library/Application Support/Code/User/settings.json" CHECK=0 if [[ "${1:-}" == "--check" ]]; then CHECK=1; fi @@ -89,13 +96,181 @@ PORT_PARSED="$(printf '%s\n' "$PARSED" | sed -n 2p)" SSH_ALIAS="$(printf '%s\n' "$PARSED" | sed -n 3p)" if [[ "$PORT_PARSED" =~ ^[0-9]+$ ]]; then FLEET_PORT="$PORT_PARSED"; fi -# Standalone: nothing to install — the guard/tunnel are irrelevant. +# --- per-peer SSH forwards (#1775) --- +# One launchd agent per attached PEER (peer studios, ADR 0027 / ADR 0032): the +# SSH `-L` forward this machine uses to reach that peer's hub. Generated from +# the repo template with the fleet transport's standard options (15 s +# keepalive, 2 misses, TCPKeepAlive, ExitOnForwardFailure — sshForwardArgs), +# never hand-installed. Role-agnostic: a peer may be standalone, server, or +# client — the signal is the reader peer-token store (fleet_peer_store.ts), +# whose loopback base_url IS the forward's local end. Darwin only (the +# supervised linux/WSL link is #1260). +# label co.harmoniqs.amico-peer--forward +# local port the store's base_url port (http://127.0.0.1:) +# remote port the fleet's canonical hub port ($FLEET_PORT) +# destination the peer's roster sshAlias, else its machine id +PEER_PLIST_SRC="$REPO_ROOT/tools/fleet/co.harmoniqs.amico-peer-forward.plist" +PEER_STORE="${AMICO_FLEET_PEER_TOKEN_FILE:-$HOME/.amico/fleet-peer-tokens-reader.json}" +ROSTER_FILE="${AMICO_FLEET_ROSTER_FILE:-$HOME/.amico/ops/fleet/roster.json}" + +# One line per peer: `forwardidlocal_portalias`, or +# `notemessage` for a peer that rides no ssh forward. Never prints a token. +peer_forward_targets() { + node -e ' + const fs = require("fs"); + const doc = (p) => { try { const j = JSON.parse(fs.readFileSync(p, "utf8")); return j && typeof j === "object" && !Array.isArray(j) ? j : {}; } catch (e) { return {}; } }; + const store = doc(process.argv[1]); + const peers = store.peers && typeof store.peers === "object" && !Array.isArray(store.peers) ? store.peers : {}; + const roster = doc(process.argv[2]); + const rows = Array.isArray(roster.rows) ? roster.rows : []; + const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + const SAFE_ALIAS = /^[A-Za-z0-9][A-Za-z0-9._@-]*$/; + const out = []; + for (const id of Object.keys(peers).sort()) { + const e = peers[id]; + const raw = e && typeof e === "object" && typeof e.base_url === "string" ? e.base_url.trim() : ""; + if (!SAFE_ID.test(id)) { out.push(["note", "peer id " + JSON.stringify(id) + " is not a safe launchd label fragment — no SSH forward generated"]); continue; } + let u = null; + try { u = new URL(raw); } catch (_) { u = null; } + const loopback = u !== null && u.protocol === "http:" && (u.hostname === "127.0.0.1" || u.hostname === "localhost"); + const port = u !== null ? Number(u.port) : NaN; + if (!loopback || !Number.isInteger(port) || port < 1 || port > 65535) { + out.push(["note", "peer " + id + " is reached at " + (raw || "(no base_url)") + " — no SSH forward (only a loopback http://127.0.0.1: base URL rides one)"]); + continue; + } + const row = rows.find((r) => r && r.machine_id === id); + const rowAlias = row && typeof row.sshAlias === "string" ? row.sshAlias.trim() : ""; + out.push(["forward", id, String(port), SAFE_ALIAS.test(rowAlias) ? rowAlias : id]); + } + process.stdout.write(out.map((f) => f.join("\t")).join("\n")); + ' "$PEER_STORE" "$ROSTER_FILE" +} + +# A hand-installed forward predates the managed convention: any +# co.harmoniqs.amico--forward.plist that is NOT a managed +# co.harmoniqs.amico-peer-*-forward agent. It is "for the same peer" when it +# binds the peer's local port (it would block the managed agent's -L under +# ExitOnForwardFailure) or targets the peer's ssh destination. +PEER_FORWARD_BACKUP_DIR="$HOME/.amico/ops/fleet/launchagents-backup" +is_legacy_forward_for() { # + local f="$1" lport="$2" alias="$3" peer_id="$4" base + base="$(basename "$f")" + if [[ "$base" == co.harmoniqs.amico-peer-*-forward.plist ]]; then return 1; fi + if grep -qE "((127\.0\.0\.1|localhost):)?${lport}:[^<]*" "$f"; then return 0; fi + if grep -qF "${alias}" "$f" || grep -qF "${peer_id}" "$f"; then return 0; fi + return 1 +} + +# Replace (never silently delete): unload the legacy job, then MOVE its plist +# to a timestamped backup OUTSIDE LaunchAgents (so launchd never reloads it). +replace_legacy_forwards() { # + local f bk + for f in "$LAUNCH_AGENTS_DIR"/co.harmoniqs.amico-*-forward.plist; do + if [[ ! -f "$f" ]]; then continue; fi + if ! is_legacy_forward_for "$f" "$2" "$3" "$1"; then continue; fi + mkdir -p "$PEER_FORWARD_BACKUP_DIR" + bk="$PEER_FORWARD_BACKUP_DIR/$(basename "$f").$(date -u +%Y%m%dT%H%M%SZ)" + if [[ -e "$bk" ]]; then bk="$bk.$$"; fi + "$LAUNCHCTL" unload "$f" 2>/dev/null || true + mv "$f" "$bk" + say "replaced hand-installed peer forward $f with the managed co.harmoniqs.amico-peer-$1-forward (backup: $bk)" + done +} + +peer_forwards() { + if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi + local targets kind peer_id lport alias label dst tmpf f peer_fail=0 + targets="$(peer_forward_targets)" || die "could not read the peer token store at $PEER_STORE (node failed)" + if [[ -z "$targets" ]]; then return 0; fi + if [[ $CHECK -eq 0 && ! -f "$PEER_PLIST_SRC" ]]; then + say "note: no peer-forward template at $PEER_PLIST_SRC — skipping peer forwards (git pull?)" + return 0 + fi + while IFS=$'\t' read -r kind peer_id lport alias; do + if [[ "$kind" == "note" ]]; then say "note: $peer_id"; continue; fi + label="co.harmoniqs.amico-peer-${peer_id}-forward" + dst="$LAUNCH_AGENTS_DIR/${label}.plist" + if [[ $CHECK -eq 1 ]]; then + # The healthcheck's CLI twin (fleet_health.ts checkPeerForwards) — report only. + for f in "$LAUNCH_AGENTS_DIR"/co.harmoniqs.amico-*-forward.plist; do + if [[ -f "$f" ]] && is_legacy_forward_for "$f" "$lport" "$alias" "$peer_id"; then + echo "[fleet] FAIL hand-installed peer forward $f for $peer_id — not managed by the installer (run: bash tools/fleet/install.sh — Fleet — Repair replaces it, keeping a backup)" + peer_fail=1 + fi + done + if [[ ! -f "$dst" ]]; then + echo "[fleet] FAIL peer forward $label missing at $dst (run: bash tools/fleet/install.sh)" + peer_fail=1 + continue + fi + local before=$peer_fail + grep -q "ServerAliveInterval=15" "$dst" || { echo "[fleet] FAIL peer forward $label ServerAliveInterval 15 missing"; peer_fail=1; } + grep -q "ServerAliveCountMax=2" "$dst" || { echo "[fleet] FAIL peer forward $label ServerAliveCountMax 2 missing"; peer_fail=1; } + grep -q "TCPKeepAlive=yes" "$dst" || { echo "[fleet] FAIL peer forward $label TCPKeepAlive yes missing"; peer_fail=1; } + grep -q "ExitOnForwardFailure=yes" "$dst" || { echo "[fleet] FAIL peer forward $label ExitOnForwardFailure yes missing"; peer_fail=1; } + grep -qF "127.0.0.1:${lport}:127.0.0.1:${FLEET_PORT}" "$dst" || { echo "[fleet] FAIL peer forward $label LocalForward 127.0.0.1:${lport}:127.0.0.1:${FLEET_PORT} missing"; peer_fail=1; } + grep -qF "${alias}" "$dst" || { echo "[fleet] FAIL peer forward $label destination is not ${alias}"; peer_fail=1; } + if [[ $peer_fail -eq $before ]]; then + say "ok peer forward $label (15/2 + TCPKeepAlive, 127.0.0.1:${lport} → ${alias}:${FLEET_PORT})" + fi + continue + fi + mkdir -p "$LAUNCH_AGENTS_DIR" + tmpf="$(mktemp)" + sed -e "s/PEER_MACHINE_ID/${peer_id}/g" -e "s/PEER_LOCAL_PORT/${lport}/g" -e "s/PEER_REMOTE_PORT/${FLEET_PORT}/g" -e "s/PEER_SSH_ALIAS/${alias}/g" "$PEER_PLIST_SRC" > "$tmpf" + replace_legacy_forwards "$peer_id" "$lport" "$alias" + if [[ -f "$dst" ]] && cmp -s "$tmpf" "$dst"; then + say "peer forward $label unchanged — skipping reload" + else + cp "$tmpf" "$dst" + "$LAUNCHCTL" unload "$dst" 2>/dev/null || true + "$LAUNCHCTL" load "$dst" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$dst" 2>/dev/null || true + say "installed peer forward $dst and (re)loaded (127.0.0.1:${lport} → ${alias}:${FLEET_PORT}, 15/2 + TCPKeepAlive)" + fi + rm -f "$tmpf" + done <<< "$targets" + if [[ $peer_fail -eq 1 ]]; then exit 1; fi +} + +# --- the session-wide multiplex-flag agent (#1775) --- +# co.harmoniqs.amico-fleet-flag ran `launchctl setenv AMICO_FLEET_MULTIPLEX 1` +# at login, leaking the flag into EVERY GUI process — and the flag already +# defaults ON (#1448, fleetMultiplexEnabled), so fleet behavior is unchanged +# without it. Retire it: unload + remove the agent, and unset the variable when +# it carries the agent's leaked value (a deliberate `0` opt-out is left alone). +FLAG_PLIST="$LAUNCH_AGENTS_DIR/co.harmoniqs.amico-fleet-flag.plist" +retire_flag_agent() { + if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi + if [[ $CHECK -eq 1 ]]; then + if [[ -f "$FLAG_PLIST" ]]; then + echo "[fleet] FAIL session-wide multiplex-flag agent present at $FLAG_PLIST (it leaks AMICO_FLEET_MULTIPLEX=1 into every GUI process; the flag already defaults on — run: bash tools/fleet/install.sh)" + exit 1 + fi + return 0 + fi + local session + if [[ -f "$FLAG_PLIST" ]]; then + "$LAUNCHCTL" unload "$FLAG_PLIST" 2>/dev/null || true + rm -f "$FLAG_PLIST" + say "removed the session-wide multiplex-flag agent $FLAG_PLIST (AMICO_FLEET_MULTIPLEX already defaults on — #1448)" + fi + session="$("$LAUNCHCTL" getenv AMICO_FLEET_MULTIPLEX 2>/dev/null || true)" + if [[ "$session" == "1" ]]; then + "$LAUNCHCTL" unsetenv AMICO_FLEET_MULTIPLEX 2>/dev/null || true + say "unset the session-wide AMICO_FLEET_MULTIPLEX (the multiplexer defaults on without it)" + fi +} + +# Standalone: no guard/tunnel/hub — but a standalone PEER still gets its +# per-peer forwards (#1775: peers stay standalone, ADR 0027). if [[ "$ROLE" == "standalone" ]]; then if [[ $CHECK -eq 1 ]]; then say "standalone (no fleet.json or role=standalone) — fleet checks skipped" else - say "standalone mode — nothing to install (use 'Amicode: Fleet — Enroll' to join a fleet)" + say "standalone mode — nothing to install for the fleet stance (use 'Amicode: Fleet — Enroll' to join a fleet)" fi + retire_flag_agent + peer_forwards exit 0 fi @@ -208,7 +383,7 @@ if [[ "$ROLE" != "client" ]]; then # A stale tunnel from a previous client enrollment must not linger on a # server: unload + remove it if present (idempotent, best-effort). if [[ "$(uname -s)" == "Darwin" && -f "$PLIST_DST" ]]; then - launchctl unload "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$PLIST_DST" 2>/dev/null || true rm -f "$PLIST_DST" say "removed stale tunnel plist $PLIST_DST (role is $ROLE, not client)" fi @@ -238,8 +413,8 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then else cp "$_plist_tmp" "$PLIST_DST" # reload - launchctl unload "$PLIST_DST" 2>/dev/null || true - launchctl load "$PLIST_DST" 2>/dev/null || launchctl bootstrap "gui/$(id -u)" "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" load "$PLIST_DST" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$PLIST_DST" 2>/dev/null || true say "installed tunnel $PLIST_DST and (re)loaded (alias ${SSH_ALIAS}, port ${FLEET_PORT})" fi rm -f "$_plist_tmp" @@ -247,6 +422,10 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then fi fi +# --- per-peer SSH forwards + the retired flag agent (every role — #1775) --- +retire_flag_agent +peer_forwards + # --- canonical hub service (SERVER role only — reboot-survival, #1258) --- # The canonical hub must survive a REBOOT with NO editor ever opened. Today the # hub is the extension-spawned detached server (ADR 0020): survives a window @@ -274,7 +453,7 @@ if [[ "$ROLE" == "server" ]]; then HUB_RUNNER="$HUB_EXT_ROOT/bin/dist/amicode-service-runner.mjs" HUB_APP_DIST="$HUB_EXT_ROOT/dist/app" # AMICODE_APP_DIST — the built app shelf the runner serves HUB_NODE="$(command -v node || true)" - HUB_PLIST_DST="$HOME/Library/LaunchAgents/${HUB_LABEL}.plist" + HUB_PLIST_DST="$LAUNCH_AGENTS_DIR/${HUB_LABEL}.plist" HUB_UNIT_DST="$HOME/.config/systemd/user/amico-hub.service" if [[ $CHECK -eq 1 ]]; then @@ -356,8 +535,8 @@ if [[ "$ROLE" == "server" ]]; then PLIST - launchctl unload "$HUB_PLIST_DST" 2>/dev/null || true - launchctl load "$HUB_PLIST_DST" 2>/dev/null || launchctl bootstrap "gui/$(id -u)" "$HUB_PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$HUB_PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" load "$HUB_PLIST_DST" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$HUB_PLIST_DST" 2>/dev/null || true say "installed hub service $HUB_PLIST_DST and (re)loaded (RunAtLoad+KeepAlive, port ${FLEET_PORT}, runs the #955 runner)" else mkdir -p "$(dirname "$HUB_UNIT_DST")" "$(dirname "$HUB_DB")" diff --git a/tools/fleet/README.md b/tools/fleet/README.md index 363e61533..edb57925e 100644 --- a/tools/fleet/README.md +++ b/tools/fleet/README.md @@ -131,6 +131,26 @@ between them in near real time. Users can work with fleet sessions from either m making one peer the sole host. The Attach control selects a peer; it does not merge the peers' local databases into one shared database. +### Per-peer SSH forwards (#1775) + +When a peer is reached over SSH, this machine holds a loopback forward to that peer's hub. The +installer (and `Fleet — Repair`) generates it — never hand-install one: + +| | | +|---|---| +| Template | `co.harmoniqs.amico-peer-forward.plist` — the tunnel's standard options: `ServerAliveInterval 15 / CountMax 2 / TCPKeepAlive yes / ExitOnForwardFailure yes` (+ `BatchMode yes`) | +| Label | `co.harmoniqs.amico-peer--forward` — one agent per peer | +| Local port | the port of the peer's `base_url` in the reader peer-token store (`~/.amico/fleet-peer-tokens-reader.json`, `http://127.0.0.1:`) | +| Remote port / host | the fleet's canonical hub port, on the peer's roster `sshAlias` (else its machine id) | + +A peer reached over https (Tailscale / direct) gets no SSH forward. A hand-installed +`co.harmoniqs.amico--forward` agent for the same peer (same local port or ssh host) is +unloaded and moved to `~/.amico/ops/fleet/launchagents-backup/` — replaced, never silently +deleted. The healthcheck and `install.sh --check` report every forward agent and flag +missing or stale keepalives. The installer also retires the old `co.harmoniqs.amico-fleet-flag` +agent (it set `AMICO_FLEET_MULTIPLEX=1` session-wide; the flag defaults on). Darwin only — +the supervised Linux/WSL link is #1260. + ### Making two hub machines into peers (no single point of failure) There is **no `amico fleet` peer subcommand and no Command Palette wizard** yet — peer entry diff --git a/tools/fleet/co.harmoniqs.amico-peer-forward.plist b/tools/fleet/co.harmoniqs.amico-peer-forward.plist new file mode 100644 index 000000000..6b07593c6 --- /dev/null +++ b/tools/fleet/co.harmoniqs.amico-peer-forward.plist @@ -0,0 +1,36 @@ + + + + + KeepAlive + + Label + co.harmoniqs.amico-peer-PEER_MACHINE_ID-forward + ProgramArguments + + /usr/bin/ssh + -N + -o + ExitOnForwardFailure=yes + -o + ServerAliveInterval=15 + -o + ServerAliveCountMax=2 + -o + TCPKeepAlive=yes + -o + BatchMode=yes + -L + 127.0.0.1:PEER_LOCAL_PORT:127.0.0.1:PEER_REMOTE_PORT + PEER_SSH_ALIAS + + RunAtLoad + + StandardErrorPath + /tmp/amico-peer-PEER_MACHINE_ID-forward.log + StandardOutPath + /tmp/amico-peer-PEER_MACHINE_ID-forward.log + ThrottleInterval + 10 + + diff --git a/tools/fleet/install.sh b/tools/fleet/install.sh index 171cd8449..9471381bc 100755 --- a/tools/fleet/install.sh +++ b/tools/fleet/install.sh @@ -10,6 +10,8 @@ # branch (stated + the grant pointer), IDENTICAL to a CLI-absent machine; # other non-zero exits die honestly — never a silent raw-file fallthrough. # Installs/updates the fleet guard + tunnel on this host and fixes machine-scoped settings. +# #1775: also generates one SSH forward agent per attached peer (replacing a +# hand-installed one with a backup) and retires the session-wide multiplex-flag agent. # Usage: # bash tools/fleet/install.sh # install/repair (writes files, reloads launchd) # bash tools/fleet/install.sh --check # check only (no writes), exit 1 on drift @@ -17,8 +19,13 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" GUARD_SRC="$REPO_ROOT/tools/fleet/amico-opencode-fleet-guard" GUARD_DST="$HOME/.local/bin/amico-opencode-fleet-guard" +# #1775: the machine-scoped launchd seams — the LaunchAgents dir and the +# launchctl binary. Both default to today's behavior; tests point them at a +# temp dir and a recording stub so no run ever touches the real service manager. +LAUNCH_AGENTS_DIR="${AMICO_FLEET_LAUNCH_AGENTS_DIR:-$HOME/Library/LaunchAgents}" +LAUNCHCTL="${AMICO_FLEET_LAUNCHCTL:-launchctl}" PLIST_SRC="$REPO_ROOT/tools/fleet/co.harmoniqs.amico-tunnel.plist" -PLIST_DST="$HOME/Library/LaunchAgents/co.harmoniqs.amico-tunnel.plist" +PLIST_DST="$LAUNCH_AGENTS_DIR/co.harmoniqs.amico-tunnel.plist" SETTINGS="$HOME/Library/Application Support/Code/User/settings.json" CHECK=0 if [[ "${1:-}" == "--check" ]]; then CHECK=1; fi @@ -89,13 +96,181 @@ PORT_PARSED="$(printf '%s\n' "$PARSED" | sed -n 2p)" SSH_ALIAS="$(printf '%s\n' "$PARSED" | sed -n 3p)" if [[ "$PORT_PARSED" =~ ^[0-9]+$ ]]; then FLEET_PORT="$PORT_PARSED"; fi -# Standalone: nothing to install — the guard/tunnel are irrelevant. +# --- per-peer SSH forwards (#1775) --- +# One launchd agent per attached PEER (peer studios, ADR 0027 / ADR 0032): the +# SSH `-L` forward this machine uses to reach that peer's hub. Generated from +# the repo template with the fleet transport's standard options (15 s +# keepalive, 2 misses, TCPKeepAlive, ExitOnForwardFailure — sshForwardArgs), +# never hand-installed. Role-agnostic: a peer may be standalone, server, or +# client — the signal is the reader peer-token store (fleet_peer_store.ts), +# whose loopback base_url IS the forward's local end. Darwin only (the +# supervised linux/WSL link is #1260). +# label co.harmoniqs.amico-peer--forward +# local port the store's base_url port (http://127.0.0.1:) +# remote port the fleet's canonical hub port ($FLEET_PORT) +# destination the peer's roster sshAlias, else its machine id +PEER_PLIST_SRC="$REPO_ROOT/tools/fleet/co.harmoniqs.amico-peer-forward.plist" +PEER_STORE="${AMICO_FLEET_PEER_TOKEN_FILE:-$HOME/.amico/fleet-peer-tokens-reader.json}" +ROSTER_FILE="${AMICO_FLEET_ROSTER_FILE:-$HOME/.amico/ops/fleet/roster.json}" + +# One line per peer: `forwardidlocal_portalias`, or +# `notemessage` for a peer that rides no ssh forward. Never prints a token. +peer_forward_targets() { + node -e ' + const fs = require("fs"); + const doc = (p) => { try { const j = JSON.parse(fs.readFileSync(p, "utf8")); return j && typeof j === "object" && !Array.isArray(j) ? j : {}; } catch (e) { return {}; } }; + const store = doc(process.argv[1]); + const peers = store.peers && typeof store.peers === "object" && !Array.isArray(store.peers) ? store.peers : {}; + const roster = doc(process.argv[2]); + const rows = Array.isArray(roster.rows) ? roster.rows : []; + const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + const SAFE_ALIAS = /^[A-Za-z0-9][A-Za-z0-9._@-]*$/; + const out = []; + for (const id of Object.keys(peers).sort()) { + const e = peers[id]; + const raw = e && typeof e === "object" && typeof e.base_url === "string" ? e.base_url.trim() : ""; + if (!SAFE_ID.test(id)) { out.push(["note", "peer id " + JSON.stringify(id) + " is not a safe launchd label fragment — no SSH forward generated"]); continue; } + let u = null; + try { u = new URL(raw); } catch (_) { u = null; } + const loopback = u !== null && u.protocol === "http:" && (u.hostname === "127.0.0.1" || u.hostname === "localhost"); + const port = u !== null ? Number(u.port) : NaN; + if (!loopback || !Number.isInteger(port) || port < 1 || port > 65535) { + out.push(["note", "peer " + id + " is reached at " + (raw || "(no base_url)") + " — no SSH forward (only a loopback http://127.0.0.1: base URL rides one)"]); + continue; + } + const row = rows.find((r) => r && r.machine_id === id); + const rowAlias = row && typeof row.sshAlias === "string" ? row.sshAlias.trim() : ""; + out.push(["forward", id, String(port), SAFE_ALIAS.test(rowAlias) ? rowAlias : id]); + } + process.stdout.write(out.map((f) => f.join("\t")).join("\n")); + ' "$PEER_STORE" "$ROSTER_FILE" +} + +# A hand-installed forward predates the managed convention: any +# co.harmoniqs.amico--forward.plist that is NOT a managed +# co.harmoniqs.amico-peer-*-forward agent. It is "for the same peer" when it +# binds the peer's local port (it would block the managed agent's -L under +# ExitOnForwardFailure) or targets the peer's ssh destination. +PEER_FORWARD_BACKUP_DIR="$HOME/.amico/ops/fleet/launchagents-backup" +is_legacy_forward_for() { # + local f="$1" lport="$2" alias="$3" peer_id="$4" base + base="$(basename "$f")" + if [[ "$base" == co.harmoniqs.amico-peer-*-forward.plist ]]; then return 1; fi + if grep -qE "((127\.0\.0\.1|localhost):)?${lport}:[^<]*" "$f"; then return 0; fi + if grep -qF "${alias}" "$f" || grep -qF "${peer_id}" "$f"; then return 0; fi + return 1 +} + +# Replace (never silently delete): unload the legacy job, then MOVE its plist +# to a timestamped backup OUTSIDE LaunchAgents (so launchd never reloads it). +replace_legacy_forwards() { # + local f bk + for f in "$LAUNCH_AGENTS_DIR"/co.harmoniqs.amico-*-forward.plist; do + if [[ ! -f "$f" ]]; then continue; fi + if ! is_legacy_forward_for "$f" "$2" "$3" "$1"; then continue; fi + mkdir -p "$PEER_FORWARD_BACKUP_DIR" + bk="$PEER_FORWARD_BACKUP_DIR/$(basename "$f").$(date -u +%Y%m%dT%H%M%SZ)" + if [[ -e "$bk" ]]; then bk="$bk.$$"; fi + "$LAUNCHCTL" unload "$f" 2>/dev/null || true + mv "$f" "$bk" + say "replaced hand-installed peer forward $f with the managed co.harmoniqs.amico-peer-$1-forward (backup: $bk)" + done +} + +peer_forwards() { + if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi + local targets kind peer_id lport alias label dst tmpf f peer_fail=0 + targets="$(peer_forward_targets)" || die "could not read the peer token store at $PEER_STORE (node failed)" + if [[ -z "$targets" ]]; then return 0; fi + if [[ $CHECK -eq 0 && ! -f "$PEER_PLIST_SRC" ]]; then + say "note: no peer-forward template at $PEER_PLIST_SRC — skipping peer forwards (git pull?)" + return 0 + fi + while IFS=$'\t' read -r kind peer_id lport alias; do + if [[ "$kind" == "note" ]]; then say "note: $peer_id"; continue; fi + label="co.harmoniqs.amico-peer-${peer_id}-forward" + dst="$LAUNCH_AGENTS_DIR/${label}.plist" + if [[ $CHECK -eq 1 ]]; then + # The healthcheck's CLI twin (fleet_health.ts checkPeerForwards) — report only. + for f in "$LAUNCH_AGENTS_DIR"/co.harmoniqs.amico-*-forward.plist; do + if [[ -f "$f" ]] && is_legacy_forward_for "$f" "$lport" "$alias" "$peer_id"; then + echo "[fleet] FAIL hand-installed peer forward $f for $peer_id — not managed by the installer (run: bash tools/fleet/install.sh — Fleet — Repair replaces it, keeping a backup)" + peer_fail=1 + fi + done + if [[ ! -f "$dst" ]]; then + echo "[fleet] FAIL peer forward $label missing at $dst (run: bash tools/fleet/install.sh)" + peer_fail=1 + continue + fi + local before=$peer_fail + grep -q "ServerAliveInterval=15" "$dst" || { echo "[fleet] FAIL peer forward $label ServerAliveInterval 15 missing"; peer_fail=1; } + grep -q "ServerAliveCountMax=2" "$dst" || { echo "[fleet] FAIL peer forward $label ServerAliveCountMax 2 missing"; peer_fail=1; } + grep -q "TCPKeepAlive=yes" "$dst" || { echo "[fleet] FAIL peer forward $label TCPKeepAlive yes missing"; peer_fail=1; } + grep -q "ExitOnForwardFailure=yes" "$dst" || { echo "[fleet] FAIL peer forward $label ExitOnForwardFailure yes missing"; peer_fail=1; } + grep -qF "127.0.0.1:${lport}:127.0.0.1:${FLEET_PORT}" "$dst" || { echo "[fleet] FAIL peer forward $label LocalForward 127.0.0.1:${lport}:127.0.0.1:${FLEET_PORT} missing"; peer_fail=1; } + grep -qF "${alias}" "$dst" || { echo "[fleet] FAIL peer forward $label destination is not ${alias}"; peer_fail=1; } + if [[ $peer_fail -eq $before ]]; then + say "ok peer forward $label (15/2 + TCPKeepAlive, 127.0.0.1:${lport} → ${alias}:${FLEET_PORT})" + fi + continue + fi + mkdir -p "$LAUNCH_AGENTS_DIR" + tmpf="$(mktemp)" + sed -e "s/PEER_MACHINE_ID/${peer_id}/g" -e "s/PEER_LOCAL_PORT/${lport}/g" -e "s/PEER_REMOTE_PORT/${FLEET_PORT}/g" -e "s/PEER_SSH_ALIAS/${alias}/g" "$PEER_PLIST_SRC" > "$tmpf" + replace_legacy_forwards "$peer_id" "$lport" "$alias" + if [[ -f "$dst" ]] && cmp -s "$tmpf" "$dst"; then + say "peer forward $label unchanged — skipping reload" + else + cp "$tmpf" "$dst" + "$LAUNCHCTL" unload "$dst" 2>/dev/null || true + "$LAUNCHCTL" load "$dst" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$dst" 2>/dev/null || true + say "installed peer forward $dst and (re)loaded (127.0.0.1:${lport} → ${alias}:${FLEET_PORT}, 15/2 + TCPKeepAlive)" + fi + rm -f "$tmpf" + done <<< "$targets" + if [[ $peer_fail -eq 1 ]]; then exit 1; fi +} + +# --- the session-wide multiplex-flag agent (#1775) --- +# co.harmoniqs.amico-fleet-flag ran `launchctl setenv AMICO_FLEET_MULTIPLEX 1` +# at login, leaking the flag into EVERY GUI process — and the flag already +# defaults ON (#1448, fleetMultiplexEnabled), so fleet behavior is unchanged +# without it. Retire it: unload + remove the agent, and unset the variable when +# it carries the agent's leaked value (a deliberate `0` opt-out is left alone). +FLAG_PLIST="$LAUNCH_AGENTS_DIR/co.harmoniqs.amico-fleet-flag.plist" +retire_flag_agent() { + if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi + if [[ $CHECK -eq 1 ]]; then + if [[ -f "$FLAG_PLIST" ]]; then + echo "[fleet] FAIL session-wide multiplex-flag agent present at $FLAG_PLIST (it leaks AMICO_FLEET_MULTIPLEX=1 into every GUI process; the flag already defaults on — run: bash tools/fleet/install.sh)" + exit 1 + fi + return 0 + fi + local session + if [[ -f "$FLAG_PLIST" ]]; then + "$LAUNCHCTL" unload "$FLAG_PLIST" 2>/dev/null || true + rm -f "$FLAG_PLIST" + say "removed the session-wide multiplex-flag agent $FLAG_PLIST (AMICO_FLEET_MULTIPLEX already defaults on — #1448)" + fi + session="$("$LAUNCHCTL" getenv AMICO_FLEET_MULTIPLEX 2>/dev/null || true)" + if [[ "$session" == "1" ]]; then + "$LAUNCHCTL" unsetenv AMICO_FLEET_MULTIPLEX 2>/dev/null || true + say "unset the session-wide AMICO_FLEET_MULTIPLEX (the multiplexer defaults on without it)" + fi +} + +# Standalone: no guard/tunnel/hub — but a standalone PEER still gets its +# per-peer forwards (#1775: peers stay standalone, ADR 0027). if [[ "$ROLE" == "standalone" ]]; then if [[ $CHECK -eq 1 ]]; then say "standalone (no fleet.json or role=standalone) — fleet checks skipped" else - say "standalone mode — nothing to install (use 'Amicode: Fleet — Enroll' to join a fleet)" + say "standalone mode — nothing to install for the fleet stance (use 'Amicode: Fleet — Enroll' to join a fleet)" fi + retire_flag_agent + peer_forwards exit 0 fi @@ -208,7 +383,7 @@ if [[ "$ROLE" != "client" ]]; then # A stale tunnel from a previous client enrollment must not linger on a # server: unload + remove it if present (idempotent, best-effort). if [[ "$(uname -s)" == "Darwin" && -f "$PLIST_DST" ]]; then - launchctl unload "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$PLIST_DST" 2>/dev/null || true rm -f "$PLIST_DST" say "removed stale tunnel plist $PLIST_DST (role is $ROLE, not client)" fi @@ -238,8 +413,8 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then else cp "$_plist_tmp" "$PLIST_DST" # reload - launchctl unload "$PLIST_DST" 2>/dev/null || true - launchctl load "$PLIST_DST" 2>/dev/null || launchctl bootstrap "gui/$(id -u)" "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" load "$PLIST_DST" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$PLIST_DST" 2>/dev/null || true say "installed tunnel $PLIST_DST and (re)loaded (alias ${SSH_ALIAS}, port ${FLEET_PORT})" fi rm -f "$_plist_tmp" @@ -247,6 +422,10 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then fi fi +# --- per-peer SSH forwards + the retired flag agent (every role — #1775) --- +retire_flag_agent +peer_forwards + # --- canonical hub service (SERVER role only — reboot-survival, #1258) --- # The canonical hub must survive a REBOOT with NO editor ever opened. Today the # hub is the extension-spawned detached server (ADR 0020): survives a window @@ -274,7 +453,7 @@ if [[ "$ROLE" == "server" ]]; then HUB_RUNNER="$HUB_EXT_ROOT/bin/dist/amicode-service-runner.mjs" HUB_APP_DIST="$HUB_EXT_ROOT/dist/app" # AMICODE_APP_DIST — the built app shelf the runner serves HUB_NODE="$(command -v node || true)" - HUB_PLIST_DST="$HOME/Library/LaunchAgents/${HUB_LABEL}.plist" + HUB_PLIST_DST="$LAUNCH_AGENTS_DIR/${HUB_LABEL}.plist" HUB_UNIT_DST="$HOME/.config/systemd/user/amico-hub.service" if [[ $CHECK -eq 1 ]]; then @@ -356,8 +535,8 @@ if [[ "$ROLE" == "server" ]]; then PLIST - launchctl unload "$HUB_PLIST_DST" 2>/dev/null || true - launchctl load "$HUB_PLIST_DST" 2>/dev/null || launchctl bootstrap "gui/$(id -u)" "$HUB_PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$HUB_PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" load "$HUB_PLIST_DST" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$HUB_PLIST_DST" 2>/dev/null || true say "installed hub service $HUB_PLIST_DST and (re)loaded (RunAtLoad+KeepAlive, port ${FLEET_PORT}, runs the #955 runner)" else mkdir -p "$(dirname "$HUB_UNIT_DST")" "$(dirname "$HUB_DB")"