From 721594a9831db17adbbe96d3681136ef0163299b Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:00:44 -0700 Subject: [PATCH 1/4] Run the per-scenario emulators locally instead of over the internet The hosted control plane put the public internet on the critical path of scenarios that are otherwise entirely local: connect ETIMEDOUT and 502s reaching the edge failed 17 shards in two weeks, and by Aug 20 graphql-introspection-health failed every run on a 403 rate limit the instance answered ahead of the fault it had armed - raising the budget from 10 to 100 changed nothing, so it was never ours to stay under. @executor-js/emulate is built for this ("local drop-in replacement services for CI and no-network sandboxes") and the suite already boots WorkOS and Autumn that way. Same package, same wire behaviour, same per-run isolation, minus the network. Scoped, so the emulator dies with the scenario. --- e2e/scenarios/connect-handoff.test.ts | 84 ++-- e2e/scenarios/google-health-checks.test.ts | 410 +++++++++--------- .../graphql-introspection-health.test.ts | 216 ++++----- e2e/scenarios/oauth-client-handoff.test.ts | 338 ++++++++------- e2e/src/emulator-instance.ts | 160 ++++--- 5 files changed, 635 insertions(+), 573 deletions(-) diff --git a/e2e/scenarios/connect-handoff.test.ts b/e2e/scenarios/connect-handoff.test.ts index e9f7696cd..2e1d96a4c 100644 --- a/e2e/scenarios/connect-handoff.test.ts +++ b/e2e/scenarios/connect-handoff.test.ts @@ -120,51 +120,53 @@ const mintEmulatorApiKey = (client: EmulatorClient) => scenario( "Connect · the agentic handoff URL opens this deployment's add-account flow and the pasted key works", { timeout: 240_000 }, - Effect.gen(function* () { - const target = yield* Target; - const mcp = yield* Mcp; - const browser = yield* Browser; - const { client: makeApiClient } = yield* Api; + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const mcp = yield* Mcp; + const browser = yield* Browser; + const { client: makeApiClient } = yield* Api; - const integration = unique("resendhf"); - const emailSubject = unique("connect-handoff"); - const emulatorClient = yield* emulator; - const apiKey = yield* mintEmulatorApiKey(emulatorClient); + const integration = unique("resendhf"); + const emailSubject = unique("connect-handoff"); + const emulatorClient = yield* emulator; + const apiKey = yield* mintEmulatorApiKey(emulatorClient); - const identity = yield* target.newIdentity(); - const session = mcp.session(identity); - const client = yield* makeApiClient(api, identity); + const identity = yield* target.newIdentity(); + const session = mcp.session(identity); + const client = yield* makeApiClient(api, identity); - // The bound org's slug, read from the same account surface the console - // shell reads — the handoff URL must canonicalize onto exactly this. - const accountClient = yield* makeApiClient(AccountHttpApi, identity); - const me = yield* accountClient.account.me(); - const orgSlug = me.organization?.slug; - expect(orgSlug, "the bound organization advertises a URL slug").toBeTruthy(); + // The bound org's slug, read from the same account surface the console + // shell reads — the handoff URL must canonicalize onto exactly this. + const accountClient = yield* makeApiClient(AccountHttpApi, identity); + const me = yield* accountClient.account.me(); + const orgSlug = me.organization?.slug; + expect(orgSlug, "the bound organization advertises a URL slug").toBeTruthy(); - yield* runScenario({ - target, - browser, - session, - identity, - integration, - emailSubject, - apiKey, - orgSlug: orgSlug!, - emulatorClient, - }).pipe( - // Best-effort cleanup even on failure: drop the created connection(s) - // over MCP, then the integration over the API. `connections.remove` is - // approval-gated, so the cleanup execute pauses per connection; - // `executeJson` auto-approves each pause so the removes actually run. - Effect.ensuring( - Effect.gen(function* () { - yield* executeJson(session, removeConnectionsCode(integration)); - yield* client.openapi.removeSpec({ params: { slug: integration } }); - }).pipe(Effect.ignore), - ), - ); - }), + yield* runScenario({ + target, + browser, + session, + identity, + integration, + emailSubject, + apiKey, + orgSlug: orgSlug!, + emulatorClient, + }).pipe( + // Best-effort cleanup even on failure: drop the created connection(s) + // over MCP, then the integration over the API. `connections.remove` is + // approval-gated, so the cleanup execute pauses per connection; + // `executeJson` auto-approves each pause so the removes actually run. + Effect.ensuring( + Effect.gen(function* () { + yield* executeJson(session, removeConnectionsCode(integration)); + yield* client.openapi.removeSpec({ params: { slug: integration } }); + }).pipe(Effect.ignore), + ), + ); + }), + ), ); const runScenario = (input: { diff --git a/e2e/scenarios/google-health-checks.test.ts b/e2e/scenarios/google-health-checks.test.ts index 8cdb602a1..2d5c8dcbf 100644 --- a/e2e/scenarios/google-health-checks.test.ts +++ b/e2e/scenarios/google-health-checks.test.ts @@ -190,236 +190,242 @@ const runGoogleOAuthFlow = ( scenario( "Google · Calendar and Gmail catalog health checks run against the emulator", { timeout: 300_000 }, - Effect.gen(function* () { - const target = yield* Target; - const browser = yield* Browser; - const { client: makeClient } = yield* Api; - const identity = yield* target.newIdentity(); - const client = yield* makeClient(api, identity); - const emulator = yield* createGoogleEmulator; - - const rows = [ - { - presetName: "Google Calendar", - slug: IntegrationSlug.make("google_calendar"), - oauthClient: OAuthClientSlug.make(unique("google_calendar_oauth")), - expectedHealthOperation: "calendar.calendarList.list", - expectedLedgerOperation: "calendar.calendarList.list", - emulatorPathPrefix: "/calendar/v3", - }, - { - presetName: "Gmail", - slug: IntegrationSlug.make("google_gmail"), - oauthClient: OAuthClientSlug.make(unique("google_gmail_oauth")), - expectedHealthOperation: "gmail.users.labels.list", - expectedLedgerOperation: "gmail.users.labels.list", - emulatorPathPrefix: "", - }, - ] as const; - - yield* Effect.ensuring( - Effect.gen(function* () { - for (const row of rows) { - yield* addGooglePresetFromCatalog(browser, identity, row.presetName, String(row.slug)); + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const browser = yield* Browser; + const { client: makeClient } = yield* Api; + const identity = yield* target.newIdentity(); + const client = yield* makeClient(api, identity); + const emulator = yield* createGoogleEmulator; + + const rows = [ + { + presetName: "Google Calendar", + slug: IntegrationSlug.make("google_calendar"), + oauthClient: OAuthClientSlug.make(unique("google_calendar_oauth")), + expectedHealthOperation: "calendar.calendarList.list", + expectedLedgerOperation: "calendar.calendarList.list", + emulatorPathPrefix: "/calendar/v3", + }, + { + presetName: "Gmail", + slug: IntegrationSlug.make("google_gmail"), + oauthClient: OAuthClientSlug.make(unique("google_gmail_oauth")), + expectedHealthOperation: "gmail.users.labels.list", + expectedLedgerOperation: "gmail.users.labels.list", + emulatorPathPrefix: "", + }, + ] as const; + + yield* Effect.ensuring( + Effect.gen(function* () { + for (const row of rows) { + yield* addGooglePresetFromCatalog(browser, identity, row.presetName, String(row.slug)); + + const stored = yield* client.integrations.healthCheckGet({ + params: { slug: row.slug }, + }); + expect(stored?.operation, `${row.presetName} stored health check`).toBe( + row.expectedHealthOperation, + ); + + yield* connectGoogleAccount({ + client, + emulator: emulator.client, + emulatorBaseUrl: emulator.baseUrl, + integrationBaseUrl: `${emulator.baseUrl}${row.emulatorPathPrefix ?? ""}`, + target, + integration: row.slug, + oauthClient: row.oauthClient, + }); + + const connections = yield* client.connections.list({ + query: { owner: "org", integration: row.slug }, + }); + const connected = connections.find((connection) => connection.name === CONNECTION); + expect( + connected?.identityLabel, + `${row.presetName} stores OAuth identity from the id_token before health checks`, + ).toBe(GOOGLE_EMULATOR_ACCOUNT_EMAIL); + expect( + connected?.lastHealth, + `${row.presetName} has not run a health check before the explicit probe`, + ).toBeNull(); + + const tools = yield* client.tools.list({ + query: { integration: row.slug, connection: CONNECTION }, + }); + expect( + tools.length, + `${row.presetName} exposes tools for the connection`, + ).toBeGreaterThan(0); + + const health = yield* client.connections.checkHealth({ + params: { owner: "org", integration: row.slug, name: CONNECTION }, + query: { ifStaleMs: 0 }, + }); + expect( + health.status, + `${row.presetName} health check is healthy: ${JSON.stringify(health)}`, + ).toBe("healthy"); + + // Check this row's ledger entry HERE, not after both rows have run. + // `ledger.list(n)` is the last n entries, and connecting the second + // account is easily a hundred emulator requests, so the first row's + // health check can be evicted from the window before a combined + // assertion at the end ever looks for it — which reads as "Calendar's + // health check never reached the emulator" when it plainly did (the + // probe above came back healthy, and only the emulator can answer + // that). The hosted emulator also acknowledges a request before its + // ledger entry is readable, so poll rather than read once. + const reached = yield* Effect.promise(() => emulator.client.ledger.list(50)).pipe( + Effect.map((ledger) => + ledger.some((entry) => entry.operationId === row.expectedLedgerOperation), + ), + Effect.repeat({ + schedule: Schedule.spaced("500 millis"), + until: (seen) => seen, + times: 19, + }), + ); + expect( + reached, + `${row.presetName}'s health check reached the Google emulator as ${row.expectedLedgerOperation}`, + ).toBe(true); + } + }), + Effect.gen(function* () { + for (const row of rows) { + yield* client.connections + .remove({ + params: { owner: "org", integration: row.slug, name: CONNECTION }, + }) + .pipe(Effect.ignore); + yield* client.oauth + .removeClient({ params: { slug: row.oauthClient }, payload: { owner: "org" } }) + .pipe(Effect.ignore); + yield* client.openapi.removeSpec({ params: { slug: row.slug } }).pipe(Effect.ignore); + } + }), + ); + }), + ), +); - const stored = yield* client.integrations.healthCheckGet({ params: { slug: row.slug } }); - expect(stored?.operation, `${row.presetName} stored health check`).toBe( - row.expectedHealthOperation, - ); +scenario( + "Google · OAuth catalog connection without a health check is healthy from the grant", + { timeout: 300_000 }, + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const browser = yield* Browser; + const { client: makeClient } = yield* Api; + const identity = yield* target.newIdentity(); + const client = yield* makeClient(api, identity); + const emulator = yield* createGoogleEmulator; + const slug = IntegrationSlug.make("google_sheets"); + const oauthClient = OAuthClientSlug.make(unique("google_sheets_oauth")); + + yield* Effect.ensuring( + Effect.gen(function* () { + yield* addGooglePresetFromCatalog(browser, identity, "Google Sheets", String(slug)); + + const stored = yield* client.integrations.healthCheckGet({ params: { slug } }); + expect(stored, "Google Sheets catalog preset declares no health check").toBeNull(); yield* connectGoogleAccount({ client, emulator: emulator.client, emulatorBaseUrl: emulator.baseUrl, - integrationBaseUrl: `${emulator.baseUrl}${row.emulatorPathPrefix ?? ""}`, target, - integration: row.slug, - oauthClient: row.oauthClient, + integration: slug, + oauthClient, }); const connections = yield* client.connections.list({ - query: { owner: "org", integration: row.slug }, + query: { owner: "org", integration: slug }, }); const connected = connections.find((connection) => connection.name === CONNECTION); expect( connected?.identityLabel, - `${row.presetName} stores OAuth identity from the id_token before health checks`, + "Google Sheets stores grant identity before any probe is configured", ).toBe(GOOGLE_EMULATOR_ACCOUNT_EMAIL); expect( connected?.lastHealth, - `${row.presetName} has not run a health check before the explicit probe`, + "Google Sheets has not run a health check before the explicit check", ).toBeNull(); - const tools = yield* client.tools.list({ - query: { integration: row.slug, connection: CONNECTION }, - }); - expect( - tools.length, - `${row.presetName} exposes tools for the connection`, - ).toBeGreaterThan(0); - const health = yield* client.connections.checkHealth({ - params: { owner: "org", integration: row.slug, name: CONNECTION }, + params: { owner: "org", integration: slug, name: CONNECTION }, query: { ifStaleMs: 0 }, }); expect( health.status, - `${row.presetName} health check is healthy: ${JSON.stringify(health)}`, + `Google Sheets no-probe health is healthy: ${JSON.stringify(health)}`, ).toBe("healthy"); + expect(health.detail).toBe("Credential resolved (no probe configured)."); + + const refreshed = yield* client.connections.list({ + query: { owner: "org", integration: slug }, + }); + expect( + refreshed.find((connection) => connection.name === CONNECTION)?.identityLabel, + "Google Sheets still shows the OAuth grant identity after no-probe health", + ).toBe(GOOGLE_EMULATOR_ACCOUNT_EMAIL); + + // Reconnecting the SAME connection must not clobber a curated label + // with the grant identity. + yield* client.connections.update({ + params: { owner: "org", integration: slug, name: CONNECTION }, + payload: { identityLabel: "Finance account" }, + }); + yield* runGoogleOAuthFlow({ client, target, integration: slug, oauthClient }); + const reconnected = yield* client.connections.list({ + query: { owner: "org", integration: slug }, + }); + expect( + reconnected.find((connection) => connection.name === CONNECTION)?.identityLabel, + "reconnect keeps the curated label over the grant identity", + ).toBe("Finance account"); - // Check this row's ledger entry HERE, not after both rows have run. - // `ledger.list(n)` is the last n entries, and connecting the second - // account is easily a hundred emulator requests, so the first row's - // health check can be evicted from the window before a combined - // assertion at the end ever looks for it — which reads as "Calendar's - // health check never reached the emulator" when it plainly did (the - // probe above came back healthy, and only the emulator can answer - // that). The hosted emulator also acknowledges a request before its - // ledger entry is readable, so poll rather than read once. - const reached = yield* Effect.promise(() => emulator.client.ledger.list(50)).pipe( - Effect.map((ledger) => - ledger.some((entry) => entry.operationId === row.expectedLedgerOperation), - ), - Effect.repeat({ - schedule: Schedule.spaced("500 millis"), - until: (seen) => seen, - times: 19, - }), + // A `newConnection` connect under a taken name mints a SECOND + // connection with a suffixed name instead of replacing the first. + const second = yield* runGoogleOAuthFlow({ + client, + target, + integration: slug, + oauthClient, + newConnection: true, + }); + expect(String(second.name), "second account mints a suffixed connection").toBe( + `${String(CONNECTION)}2`, ); + expect(second.identityLabel, "second account label comes from the grant identity").toBe( + GOOGLE_EMULATOR_ACCOUNT_EMAIL, + ); + const both = yield* client.connections.list({ + query: { owner: "org", integration: slug }, + }); expect( - reached, - `${row.presetName}'s health check reached the Google emulator as ${row.expectedLedgerOperation}`, - ).toBe(true); - } - }), - Effect.gen(function* () { - for (const row of rows) { - yield* client.connections - .remove({ - params: { owner: "org", integration: row.slug, name: CONNECTION }, - }) - .pipe(Effect.ignore); + both.map((connection) => String(connection.name)).sort(), + "both accounts coexist", + ).toEqual([String(CONNECTION), `${String(CONNECTION)}2`]); + }), + Effect.gen(function* () { + for (const name of [CONNECTION, ConnectionName.make(`${String(CONNECTION)}2`)]) { + yield* client.connections + .remove({ + params: { owner: "org", integration: slug, name }, + }) + .pipe(Effect.ignore); + } yield* client.oauth - .removeClient({ params: { slug: row.oauthClient }, payload: { owner: "org" } }) - .pipe(Effect.ignore); - yield* client.openapi.removeSpec({ params: { slug: row.slug } }).pipe(Effect.ignore); - } - }), - ); - }), -); - -scenario( - "Google · OAuth catalog connection without a health check is healthy from the grant", - { timeout: 300_000 }, - Effect.gen(function* () { - const target = yield* Target; - const browser = yield* Browser; - const { client: makeClient } = yield* Api; - const identity = yield* target.newIdentity(); - const client = yield* makeClient(api, identity); - const emulator = yield* createGoogleEmulator; - const slug = IntegrationSlug.make("google_sheets"); - const oauthClient = OAuthClientSlug.make(unique("google_sheets_oauth")); - - yield* Effect.ensuring( - Effect.gen(function* () { - yield* addGooglePresetFromCatalog(browser, identity, "Google Sheets", String(slug)); - - const stored = yield* client.integrations.healthCheckGet({ params: { slug } }); - expect(stored, "Google Sheets catalog preset declares no health check").toBeNull(); - - yield* connectGoogleAccount({ - client, - emulator: emulator.client, - emulatorBaseUrl: emulator.baseUrl, - target, - integration: slug, - oauthClient, - }); - - const connections = yield* client.connections.list({ - query: { owner: "org", integration: slug }, - }); - const connected = connections.find((connection) => connection.name === CONNECTION); - expect( - connected?.identityLabel, - "Google Sheets stores grant identity before any probe is configured", - ).toBe(GOOGLE_EMULATOR_ACCOUNT_EMAIL); - expect( - connected?.lastHealth, - "Google Sheets has not run a health check before the explicit check", - ).toBeNull(); - - const health = yield* client.connections.checkHealth({ - params: { owner: "org", integration: slug, name: CONNECTION }, - query: { ifStaleMs: 0 }, - }); - expect( - health.status, - `Google Sheets no-probe health is healthy: ${JSON.stringify(health)}`, - ).toBe("healthy"); - expect(health.detail).toBe("Credential resolved (no probe configured)."); - - const refreshed = yield* client.connections.list({ - query: { owner: "org", integration: slug }, - }); - expect( - refreshed.find((connection) => connection.name === CONNECTION)?.identityLabel, - "Google Sheets still shows the OAuth grant identity after no-probe health", - ).toBe(GOOGLE_EMULATOR_ACCOUNT_EMAIL); - - // Reconnecting the SAME connection must not clobber a curated label - // with the grant identity. - yield* client.connections.update({ - params: { owner: "org", integration: slug, name: CONNECTION }, - payload: { identityLabel: "Finance account" }, - }); - yield* runGoogleOAuthFlow({ client, target, integration: slug, oauthClient }); - const reconnected = yield* client.connections.list({ - query: { owner: "org", integration: slug }, - }); - expect( - reconnected.find((connection) => connection.name === CONNECTION)?.identityLabel, - "reconnect keeps the curated label over the grant identity", - ).toBe("Finance account"); - - // A `newConnection` connect under a taken name mints a SECOND - // connection with a suffixed name instead of replacing the first. - const second = yield* runGoogleOAuthFlow({ - client, - target, - integration: slug, - oauthClient, - newConnection: true, - }); - expect(String(second.name), "second account mints a suffixed connection").toBe( - `${String(CONNECTION)}2`, - ); - expect(second.identityLabel, "second account label comes from the grant identity").toBe( - GOOGLE_EMULATOR_ACCOUNT_EMAIL, - ); - const both = yield* client.connections.list({ - query: { owner: "org", integration: slug }, - }); - expect( - both.map((connection) => String(connection.name)).sort(), - "both accounts coexist", - ).toEqual([String(CONNECTION), `${String(CONNECTION)}2`]); - }), - Effect.gen(function* () { - for (const name of [CONNECTION, ConnectionName.make(`${String(CONNECTION)}2`)]) { - yield* client.connections - .remove({ - params: { owner: "org", integration: slug, name }, - }) + .removeClient({ params: { slug: oauthClient }, payload: { owner: "org" } }) .pipe(Effect.ignore); - } - yield* client.oauth - .removeClient({ params: { slug: oauthClient }, payload: { owner: "org" } }) - .pipe(Effect.ignore); - yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore); - }), - ); - }), + yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore); + }), + ); + }), + ), ); diff --git a/e2e/scenarios/graphql-introspection-health.test.ts b/e2e/scenarios/graphql-introspection-health.test.ts index b0ca487fb..5ddca2578 100644 --- a/e2e/scenarios/graphql-introspection-health.test.ts +++ b/e2e/scenarios/graphql-introspection-health.test.ts @@ -19,122 +19,124 @@ const unique = (prefix: string): string => `${prefix}_${randomBytes(4).toString( scenario( "GraphQL · failed introspection blocks connection creation with an actionable error", {}, - Effect.gen(function* () { - const target = yield* Target; - const browser = yield* Browser; - const { client: makeApiClient } = yield* Api; - const identity = yield* target.newIdentity(); - const client = yield* makeApiClient(api, identity); - const slug = unique("graphql_health"); - const emulatorBaseUrl = yield* createEmulatorInstance("github", "graphql-health"); - const emulator = yield* Effect.promise(() => - connectEmulator({ baseUrl: emulatorBaseUrl, service: "github" }), - ); + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const browser = yield* Browser; + const { client: makeApiClient } = yield* Api; + const identity = yield* target.newIdentity(); + const client = yield* makeApiClient(api, identity); + const slug = unique("graphql_health"); + const emulatorBaseUrl = yield* createEmulatorInstance("github", "graphql-health"); + const emulator = yield* Effect.promise(() => + connectEmulator({ baseUrl: emulatorBaseUrl, service: "github" }), + ); - // A budget, not a count: nothing in this scenario depends on how many - // times the connect flow introspects, and the emulator's answer when the - // budget runs out is not a neutral pass-through — an unauthenticated - // GraphQL POST to the real handler is GitHub-shaped, so it comes back 403 - // "API rate limit exceeded". The UI then honestly reports HTTP 403 and the - // assertion below fails on a message that has nothing to do with the - // product. Arm enough that one connect attempt cannot exhaust it. - yield* Effect.promise(() => - emulator.faults.arm({ - match: { method: "POST", pathPattern: "/graphql" }, - response: { status: 401, body: { message: "Bad credentials" } }, - times: 100, - }), - ); + // A budget, not a count: nothing in this scenario depends on how many + // times the connect flow introspects, and the emulator's answer when the + // budget runs out is not a neutral pass-through — an unauthenticated + // GraphQL POST to the real handler is GitHub-shaped, so it comes back 403 + // "API rate limit exceeded". The UI then honestly reports HTTP 403 and the + // assertion below fails on a message that has nothing to do with the + // product. Arm enough that one connect attempt cannot exhaust it. + yield* Effect.promise(() => + emulator.faults.arm({ + match: { method: "POST", pathPattern: "/graphql" }, + response: { status: 401, body: { message: "Bad credentials" } }, + times: 100, + }), + ); - yield* client.graphql.addIntegration({ - payload: { - endpoint: `${emulatorBaseUrl}/graphql`, - slug, - name: "GraphQL health", - authenticationTemplate: [ - { - slug: "header", - type: "apiKey", - headers: { Authorization: [variable("token")] }, - }, - ], - }, - }); - - yield* Effect.gen(function* () { - yield* browser.session(identity, async ({ page, step }) => { - await step("Open the connection flow", async () => { - await visit(page, `/integrations/${slug}?addAccount=1&owner=org&template=header`); - await page.getByRole("heading", { name: /Add connection · GraphQL health/ }).waitFor(); - }); + yield* client.graphql.addIntegration({ + payload: { + endpoint: `${emulatorBaseUrl}/graphql`, + slug, + name: "GraphQL health", + authenticationTemplate: [ + { + slug: "header", + type: "apiKey", + headers: { Authorization: [variable("token")] }, + }, + ], + }, + }); - await step("Submit a credential rejected during schema introspection", async () => { - const dialog = page.getByRole("dialog", { - name: /Add connection · GraphQL health/, + yield* Effect.gen(function* () { + yield* browser.session(identity, async ({ page, step }) => { + await step("Open the connection flow", async () => { + await visit(page, `/integrations/${slug}?addAccount=1&owner=org&template=header`); + await page.getByRole("heading", { name: /Add connection · GraphQL health/ }).waitFor(); }); - await dialog.getByRole("textbox", { name: "Authorization" }).fill("invalid-token"); - await dialog.getByRole("button", { name: "Continue" }).click(); - const alert = dialog.getByRole("alert"); - await alert.waitFor(); - const message = await alert.textContent(); - expect(message).toContain("The endpoint rejected the credential with HTTP 401."); - expect(message).toContain("Check the credential and selected authentication method."); - await dialog.getByText("Step 1 of 2").waitFor(); - expect( - await page.getByText("No connections yet").count(), - "the rejected credential is not saved", - ).toBe(1); - }); - }); + await step("Submit a credential rejected during schema introspection", async () => { + const dialog = page.getByRole("dialog", { + name: /Add connection · GraphQL health/, + }); + await dialog.getByRole("textbox", { name: "Authorization" }).fill("invalid-token"); + await dialog.getByRole("button", { name: "Continue" }).click(); - // The low-level API can still import an existing credential reference - // without the browser's preflight. This models connections created before - // the fix and proves their failed tool sync is no longer a silent zero. - yield* client.connections.create({ - payload: { - owner: "org", - name: ConnectionName.make("legacy"), - integration: IntegrationSlug.make(slug), - template: AuthTemplateSlug.make("header"), - value: "invalid-token", - }, - }); + const alert = dialog.getByRole("alert"); + await alert.waitFor(); + const message = await alert.textContent(); + expect(message).toContain("The endpoint rejected the credential with HTTP 401."); + expect(message).toContain("Check the credential and selected authentication method."); + await dialog.getByText("Step 1 of 2").waitFor(); + expect( + await page.getByText("No connections yet").count(), + "the rejected credential is not saved", + ).toBe(1); + }); + }); - yield* browser.session(identity, async ({ page, step }) => { - await step("A failed existing connection explains the empty tool catalogue", async () => { - await visit(page, `/integrations/${slug}?tab=tools`); - await page.getByText("Connection rejected", { exact: true }).first().waitFor(); - await page - .getByText("The endpoint rejected the credential with HTTP 401.", { - exact: false, - }) - .waitFor(); - await page.getByRole("button", { name: "Check and sync tools" }).waitFor(); + // The low-level API can still import an existing credential reference + // without the browser's preflight. This models connections created before + // the fix and proves their failed tool sync is no longer a silent zero. + yield* client.connections.create({ + payload: { + owner: "org", + name: ConnectionName.make("legacy"), + integration: IntegrationSlug.make(slug), + template: AuthTemplateSlug.make("header"), + value: "invalid-token", + }, }); - await step("The account row carries the same actionable health verdict", async () => { - await page.getByRole("tab", { name: "Accounts" }).click(); - await page.getByText("Expired", { exact: true }).waitFor(); - await page - .getByText("Check the credential and selected authentication method.", { - exact: false, - }) - .waitFor(); + yield* browser.session(identity, async ({ page, step }) => { + await step("A failed existing connection explains the empty tool catalogue", async () => { + await visit(page, `/integrations/${slug}?tab=tools`); + await page.getByText("Connection rejected", { exact: true }).first().waitFor(); + await page + .getByText("The endpoint rejected the credential with HTTP 401.", { + exact: false, + }) + .waitFor(); + await page.getByRole("button", { name: "Check and sync tools" }).waitFor(); + }); + + await step("The account row carries the same actionable health verdict", async () => { + await page.getByRole("tab", { name: "Accounts" }).click(); + await page.getByText("Expired", { exact: true }).waitFor(); + await page + .getByText("Check the credential and selected authentication method.", { + exact: false, + }) + .waitFor(); + }); }); - }); - }).pipe( - Effect.ensuring( - Effect.all( - [ - client.integrations - .remove({ params: { slug: IntegrationSlug.make(slug) } }) - .pipe(Effect.ignore), - Effect.promise(() => emulator.faults.clear()).pipe(Effect.ignore), - ], - { concurrency: "unbounded" }, + }).pipe( + Effect.ensuring( + Effect.all( + [ + client.integrations + .remove({ params: { slug: IntegrationSlug.make(slug) } }) + .pipe(Effect.ignore), + Effect.promise(() => emulator.faults.clear()).pipe(Effect.ignore), + ], + { concurrency: "unbounded" }, + ), ), - ), - ); - }), + ); + }), + ), ); diff --git a/e2e/scenarios/oauth-client-handoff.test.ts b/e2e/scenarios/oauth-client-handoff.test.ts index a9575c2e7..865c3289c 100644 --- a/e2e/scenarios/oauth-client-handoff.test.ts +++ b/e2e/scenarios/oauth-client-handoff.test.ts @@ -317,176 +317,178 @@ const requireOAuthClientCredential = (credential: IssuedCredential) => scenario( "OAuth client · agent hands off, the human enters the secret in the browser, and the app connects", { timeout: 240_000 }, - Effect.gen(function* () { - const target = yield* Target; - const { client: makeApiClient } = yield* Api; - const mcp = yield* Mcp; - const browser = yield* Browser; - const identity = yield* target.newIdentity(); - const session = mcp.session(identity); - const client = yield* makeApiClient(microsoftApi, identity); - - const accountClient = yield* makeApiClient(AccountHttpApi, identity); - const me = yield* accountClient.account.me(); - const orgSlug = me.organization?.slug; - expect(orgSlug, "the bound organization advertises a URL slug").toBeTruthy(); - - const integration = unique("msgraph"); - const clientSlug = unique("msgraph_app"); - const connection = "machine"; - const template = MICROSOFT_CLIENT_CREDENTIALS_AUTH_TEMPLATE_SLUG; - - // A per-run hosted emulator instance mints a real-shaped client-credentials - // app and records every token exchange in its own isolated ledger. - const emulatorBase = yield* createEmulatorInstance("microsoft", "oauth-handoff"); - const emulator: EmulatorClient = yield* Effect.promise(() => - connectEmulator({ baseUrl: emulatorBase, service: "microsoft" }), - ); - const minted = yield* Effect.promise(() => - emulator.credentials.mint({ type: "oauth-client-credentials", name: "Executor E2E Graph" }), - ); - const oauth = yield* requireOAuthClientCredential(minted); - - yield* Effect.ensuring( - Effect.gen(function* () { - // Register the Microsoft Graph integration so the console has an OAuth - // method to register a client against. - yield* client.openapi.addSpec({ - payload: { - spec: { kind: "url", url: emulator.openapiUrl }, - slug: integration, - name: "Microsoft Graph Emulator", - baseUrl: emulator.baseUrl, - family: "microsoft", - authenticationTemplate: [ - { - slug: template, - kind: "oauth2", - authorizationUrl: oauth.authorizationUrl, - tokenUrl: oauth.tokenUrl, - scopes: ["https://graph.microsoft.com/.default"], - }, - ], - }, - }); - - // 1. The agent asks for a browser handoff URL — it has the client id and - // endpoints (discovered/known), but never the secret. - const handoff = yield* executeJson( - session, - handoffForBrowserCode({ - integration, - slug: clientSlug, - clientId: oauth.clientId, - authorizationUrl: oauth.authorizationUrl, - tokenUrl: oauth.tokenUrl, - }), - ); - expect(handoff.ok, `createHandoff succeeded: ${JSON.stringify(handoff)}`).toBe(true); - const handoffUrl = String(handoff.url); - - const parsed = new URL(handoffUrl); - expect(parsed.origin, `handoff URL (${handoffUrl}) targets this deployment`).toBe( - new URL(target.baseUrl).origin, - ); - expect(parsed.pathname).toBe(`/${orgSlug}/integrations/${integration}`); - // The agent's URL carries the client id but NOT the secret. - expect(handoffUrl).toContain(oauth.clientId); - expect( - handoffUrl.includes(oauth.clientSecret), - "the handoff URL never carries the client secret", - ).toBe(false); - - // 2. The human opens the URL: the Register-OAuth-app form is open and - // pre-filled from the handoff. They type ONLY the secret. - yield* browser.session(identity, async ({ page, step }) => { - await step("Open the agent's handoff URL", async () => { - await visit(page, handoffUrl); - }); - - await step("The Register-OAuth-app form auto-opens, pre-filled", async () => { - await page - .getByRole("heading", { name: "Register OAuth app" }) - .waitFor({ timeout: 20_000 }); - // The agent's non-secret fields pre-filled — this is the whole point - // of the handoff: the human verifies, they don't re-type. - await expect - .poll(() => page.locator("#oauth-client-id").inputValue()) - .toBe(oauth.clientId); - await expect - .poll(() => page.locator("#grant-client_credentials").isChecked()) - .toBe(true); + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const { client: makeApiClient } = yield* Api; + const mcp = yield* Mcp; + const browser = yield* Browser; + const identity = yield* target.newIdentity(); + const session = mcp.session(identity); + const client = yield* makeApiClient(microsoftApi, identity); + + const accountClient = yield* makeApiClient(AccountHttpApi, identity); + const me = yield* accountClient.account.me(); + const orgSlug = me.organization?.slug; + expect(orgSlug, "the bound organization advertises a URL slug").toBeTruthy(); + + const integration = unique("msgraph"); + const clientSlug = unique("msgraph_app"); + const connection = "machine"; + const template = MICROSOFT_CLIENT_CREDENTIALS_AUTH_TEMPLATE_SLUG; + + // A per-run hosted emulator instance mints a real-shaped client-credentials + // app and records every token exchange in its own isolated ledger. + const emulatorBase = yield* createEmulatorInstance("microsoft", "oauth-handoff"); + const emulator: EmulatorClient = yield* Effect.promise(() => + connectEmulator({ baseUrl: emulatorBase, service: "microsoft" }), + ); + const minted = yield* Effect.promise(() => + emulator.credentials.mint({ type: "oauth-client-credentials", name: "Executor E2E Graph" }), + ); + const oauth = yield* requireOAuthClientCredential(minted); + + yield* Effect.ensuring( + Effect.gen(function* () { + // Register the Microsoft Graph integration so the console has an OAuth + // method to register a client against. + yield* client.openapi.addSpec({ + payload: { + spec: { kind: "url", url: emulator.openapiUrl }, + slug: integration, + name: "Microsoft Graph Emulator", + baseUrl: emulator.baseUrl, + family: "microsoft", + authenticationTemplate: [ + { + slug: template, + kind: "oauth2", + authorizationUrl: oauth.authorizationUrl, + tokenUrl: oauth.tokenUrl, + scopes: ["https://graph.microsoft.com/.default"], + }, + ], + }, }); - await step("The human types the client secret (only the secret)", async () => { - const secret = page.locator("#oauth-client-secret"); - await secret.waitFor({ timeout: 15_000 }); - await secret.fill(oauth.clientSecret); + // 1. The agent asks for a browser handoff URL — it has the client id and + // endpoints (discovered/known), but never the secret. + const handoff = yield* executeJson( + session, + handoffForBrowserCode({ + integration, + slug: clientSlug, + clientId: oauth.clientId, + authorizationUrl: oauth.authorizationUrl, + tokenUrl: oauth.tokenUrl, + }), + ); + expect(handoff.ok, `createHandoff succeeded: ${JSON.stringify(handoff)}`).toBe(true); + const handoffUrl = String(handoff.url); + + const parsed = new URL(handoffUrl); + expect(parsed.origin, `handoff URL (${handoffUrl}) targets this deployment`).toBe( + new URL(target.baseUrl).origin, + ); + expect(parsed.pathname).toBe(`/${orgSlug}/integrations/${integration}`); + // The agent's URL carries the client id but NOT the secret. + expect(handoffUrl).toContain(oauth.clientId); + expect( + handoffUrl.includes(oauth.clientSecret), + "the handoff URL never carries the client secret", + ).toBe(false); + + // 2. The human opens the URL: the Register-OAuth-app form is open and + // pre-filled from the handoff. They type ONLY the secret. + yield* browser.session(identity, async ({ page, step }) => { + await step("Open the agent's handoff URL", async () => { + await visit(page, handoffUrl); + }); + + await step("The Register-OAuth-app form auto-opens, pre-filled", async () => { + await page + .getByRole("heading", { name: "Register OAuth app" }) + .waitFor({ timeout: 20_000 }); + // The agent's non-secret fields pre-filled — this is the whole point + // of the handoff: the human verifies, they don't re-type. + await expect + .poll(() => page.locator("#oauth-client-id").inputValue()) + .toBe(oauth.clientId); + await expect + .poll(() => page.locator("#grant-client_credentials").isChecked()) + .toBe(true); + }); + + await step("The human types the client secret (only the secret)", async () => { + const secret = page.locator("#oauth-client-secret"); + await secret.waitFor({ timeout: 15_000 }); + await secret.fill(oauth.clientSecret); + }); + + await step("Register the app", async () => { + await page.getByRole("button", { name: "Register app", exact: true }).click(); + // onCreated returns to the Add-connection view — the register form closes. + await page + .getByRole("heading", { name: "Register OAuth app" }) + .waitFor({ state: "hidden", timeout: 20_000 }); + }); }); - await step("Register the app", async () => { - await page.getByRole("button", { name: "Register app", exact: true }).click(); - // onCreated returns to the Add-connection view — the register form closes. - await page - .getByRole("heading", { name: "Register OAuth app" }) - .waitFor({ state: "hidden", timeout: 20_000 }); - }); - }); - - // 3. The agent discovers the browser-registered client and completes the - // connection — client credentials need no user consent. - const listed = yield* executeJson(session, listClientSlugsCode); - expect( - (listed.slugs as ReadonlyArray | undefined)?.includes(clientSlug), - `the agent sees the human-registered client: ${JSON.stringify(listed)}`, - ).toBe(true); - - const started = yield* executeJson( - session, - startConnectionCode({ - slug: clientSlug, - integration, - connection, - template: String(template), - }), - ); - expect(started.ok, `oauth.start succeeded: ${JSON.stringify(started)}`).toBe(true); - expect(started.status, "client-credentials OAuth connected without browser consent").toBe( - "connected", - ); - - // 4. The emulator ledger proves Executor exchanged THIS app's credentials. - const ledger = yield* Effect.promise(() => emulator.ledger.list()); - const tokenRequest = ledger.find( - (entry) => - entry.path === "/oauth2/v2.0/token" && - JSON.stringify(entry.request.body ?? "").includes(oauth.clientId), - ); - expect( - tokenRequest?.response.status, - "the emulator recorded a client-credentials token exchange for this app", - ).toBe(200); - expect(tokenRequest?.request.body).toMatchObject({ grant_type: "client_credentials" }); - }), - // Best-effort teardown: selfhost shares one workspace, so remove everything. - Effect.gen(function* () { - yield* client.connections - .remove({ - params: { - owner: "org", - integration: IntegrationSlug.make(integration), - name: ConnectionName.make(connection), - }, - }) - .pipe(Effect.ignore); - yield* client.oauth - .removeClient({ - params: { slug: OAuthClientSlug.make(clientSlug) }, - payload: { owner: "org" }, - }) - .pipe(Effect.ignore); - yield* client.openapi.removeSpec({ params: { slug: integration } }).pipe(Effect.ignore); - }).pipe(Effect.ignore), - ); - }), + // 3. The agent discovers the browser-registered client and completes the + // connection — client credentials need no user consent. + const listed = yield* executeJson(session, listClientSlugsCode); + expect( + (listed.slugs as ReadonlyArray | undefined)?.includes(clientSlug), + `the agent sees the human-registered client: ${JSON.stringify(listed)}`, + ).toBe(true); + + const started = yield* executeJson( + session, + startConnectionCode({ + slug: clientSlug, + integration, + connection, + template: String(template), + }), + ); + expect(started.ok, `oauth.start succeeded: ${JSON.stringify(started)}`).toBe(true); + expect(started.status, "client-credentials OAuth connected without browser consent").toBe( + "connected", + ); + + // 4. The emulator ledger proves Executor exchanged THIS app's credentials. + const ledger = yield* Effect.promise(() => emulator.ledger.list()); + const tokenRequest = ledger.find( + (entry) => + entry.path === "/oauth2/v2.0/token" && + JSON.stringify(entry.request.body ?? "").includes(oauth.clientId), + ); + expect( + tokenRequest?.response.status, + "the emulator recorded a client-credentials token exchange for this app", + ).toBe(200); + expect(tokenRequest?.request.body).toMatchObject({ grant_type: "client_credentials" }); + }), + // Best-effort teardown: selfhost shares one workspace, so remove everything. + Effect.gen(function* () { + yield* client.connections + .remove({ + params: { + owner: "org", + integration: IntegrationSlug.make(integration), + name: ConnectionName.make(connection), + }, + }) + .pipe(Effect.ignore); + yield* client.oauth + .removeClient({ + params: { slug: OAuthClientSlug.make(clientSlug) }, + payload: { owner: "org" }, + }) + .pipe(Effect.ignore); + yield* client.openapi.removeSpec({ params: { slug: integration } }).pipe(Effect.ignore); + }).pipe(Effect.ignore), + ); + }), + ), ); diff --git a/e2e/src/emulator-instance.ts b/e2e/src/emulator-instance.ts index d17ebc711..169662673 100644 --- a/e2e/src/emulator-instance.ts +++ b/e2e/src/emulator-instance.ts @@ -1,6 +1,32 @@ -import { Effect, Schedule } from "effect"; +// Per-scenario service emulators, spawned IN THIS PROCESS. +// +// These used to be hosted instances created through +// `https://.emulators.dev/_emulate/instances`. That put the public +// internet on the critical path of a scenario that is otherwise entirely +// local, and it showed: in the two weeks to 2026-08-20, `connect ETIMEDOUT` +// and bare 502s reaching the edge failed 17 shards. Worse, by 2026-08-20 +// `graphql-introspection-health` failed on every CI run because the hosted +// GitHub emulator answered `403 API rate limit exceeded` — GitHub's real +// unauthenticated-rate-limit shape — instead of the 401 the scenario had +// armed a fault for. Raising the fault budget from 10 to 100 changed nothing, +// so whatever produced that 403 sat outside the instance's own fault +// accounting; from a shared CI egress IP there is no version of the scenario +// that can stay under it. +// +// `@executor-js/emulate` describes itself as "local drop-in replacement +// services for CI and no-network sandboxes", and the suite already boots +// WorkOS and Autumn this way in `setup/cloud.boot.ts`. This is the same thing +// per scenario: same package, same wire behaviour, same per-run isolation (a +// fresh process-local instance, so ledger assertions stay clean), minus the +// network. The app under test reaches it over loopback, which cloud already +// allows for the WorkOS and Autumn emulators (`ALLOW_LOCAL_NETWORK`). +import { createServer } from "node:net"; + +import { Effect, Schedule, type Scope } from "effect"; -/** The suite could not get an instance out of the hosted control plane. */ +import { createEmulator, type ServiceName } from "@executor-js/emulate"; + +/** The emulator could not be started, or never answered its control plane. */ export class EmulatorInstanceError extends Error { readonly _tag = "EmulatorInstanceError"; @@ -8,64 +34,88 @@ export class EmulatorInstanceError extends Error { readonly service: string, readonly reason: string, ) { - super(`${service} emulator instance creation failed: ${reason}`); + super(`${service} emulator did not start: ${reason}`); this.name = "EmulatorInstanceError"; } } -// Bound each attempt: a hung connection to the edge must not eat the -// scenario's whole timeout before the first retry. -const ATTEMPT_TIMEOUT = "10 seconds"; -const RETRIES = 3; - -const requestInstance = (service: string, label: string) => - Effect.tryPromise({ - try: async (): Promise => { - const response = await fetch(`https://${service}.emulators.dev/_emulate/instances`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ instance: label }), - }); - if (!response.ok) { - throw new EmulatorInstanceError(service, `HTTP ${response.status}`); - } - const instance = (await response.json()) as { readonly providerBaseUrl: string }; - return instance.providerBaseUrl; - }, - catch: (cause) => - cause instanceof EmulatorInstanceError - ? cause - : new EmulatorInstanceError(service, String(cause)), +// Ask the OS for a port and hand it straight to the emulator. There is a +// window between the probe closing and the emulator binding, which is why +// `spawnEmulator` retries: on Linux CI this whole range is ephemeral, so an +// outbound socket really can take the port in between (the same race +// `src/ports.ts` documents for the target's own stack). +const freePort = (): Promise => + new Promise((resolve) => { + const probe = createServer(); + probe.once("error", () => resolve(null)); + probe.listen(0, "0.0.0.0", () => { + const address = probe.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + probe.close(() => resolve(port === 0 ? null : port)); + }); }); -// Hosted service hosts (e.g. resend.emulators.dev) are control plane only — -// there is no shared default instance behind them. Every scenario creates its -// own isolated instance and works against the returned providerBaseUrl, which -// also keeps ledger assertions free of cross-run pollution. The server -// generates an unguessable instance name; the label is a readable prefix. -// -// This is also the one request in a scenario that leaves the runner, so it is -// the one place where a CI runner's transient network trouble fails a scenario -// that has nothing to do with the network: `connect ETIMEDOUT` reaching the -// edge, plus the occasional bare 502, accounted for 17 shard failures in the -// two weeks to 2026-08-20. Asking for an instance is idempotent (a spare -// instance is nobody's business but the control plane's), so bound the attempt -// and retry with backoff. Nothing below this line retries anything the -// scenario is actually asserting on. -export const createEmulatorInstance = (service: string, label = "e2e"): Effect.Effect => - requestInstance(service, label).pipe( - Effect.timeoutOrElse({ - duration: ATTEMPT_TIMEOUT, - orElse: () => - Effect.fail(new EmulatorInstanceError(service, `no response in ${ATTEMPT_TIMEOUT}`)), - }), - Effect.retry( - Schedule.both( - Schedule.exponential("500 millis").pipe(Schedule.jittered), - Schedule.recurs(RETRIES), +const READY_TIMEOUT = "10 seconds"; + +const spawnEmulator = (service: ServiceName, label: string) => + Effect.gen(function* () { + const port = yield* Effect.promise(freePort).pipe( + Effect.flatMap((found) => + found === null + ? Effect.fail(new EmulatorInstanceError(service, "the OS offered no free port")) + : Effect.succeed(found), ), + ); + // 127.0.0.1, not localhost: the emulator stamps this base URL into its own + // OAuth metadata and discovery documents, and `localhost` resolves to ::1 + // first on some hosts. Pinning the family keeps what the app is told + // byte-identical to what the emulator is listening on. + const baseUrl = `http://127.0.0.1:${port}`; + const emulator = yield* Effect.tryPromise({ + try: () => createEmulator({ service, port, baseUrl }), + catch: (cause) => new EmulatorInstanceError(service, String(cause)), + }); + // `createEmulator` returns as soon as the server is handed off, so prove + // the control plane answers before a scenario arms a fault against it. + yield* Effect.tryPromise({ + try: async () => { + const response = await fetch(`${baseUrl}/_emulate/manifest`); + if (!response.ok) throw new Error(`manifest HTTP ${response.status}`); + }, + catch: (cause) => new EmulatorInstanceError(service, String(cause)), + }).pipe( + Effect.retry(Schedule.both(Schedule.spaced("100 millis"), Schedule.recurs(50))), + Effect.timeoutOrElse({ + duration: READY_TIMEOUT, + orElse: () => + Effect.fail( + new EmulatorInstanceError(service, `control plane silent for ${READY_TIMEOUT}`), + ), + }), + // The half-started server must not outlive the failure. + Effect.tapError(() => Effect.promise(() => emulator.close())), + ); + yield* Effect.logDebug(`[e2e] ${label} ${service} emulator at ${baseUrl}`); + return emulator; + }); + +/** + * Start a `service` emulator for the calling scenario and return its base URL. + * + * Scoped: the emulator is closed when the scenario's scope closes, so wrap the + * body in `Effect.scoped`. `label` names the instance in debug output. + */ +export const createEmulatorInstance = ( + service: ServiceName, + label = "e2e", +): Effect.Effect => + Effect.acquireRelease( + spawnEmulator(service, label).pipe( + // A port lost between probe and bind is worth one more try; anything + // else is a defect in the run, not a product failure the scenario + // should be asked to model. + Effect.retry(Schedule.both(Schedule.spaced("200 millis"), Schedule.recurs(2))), + Effect.orDie, ), - // An emulator the suite cannot reach at all is a defect in the run, not a - // product failure the scenario should be asked to model. - Effect.orDie, - ); + (emulator) => Effect.promise(() => emulator.close()), + ).pipe(Effect.map((emulator) => emulator.url)); From 6cfb96bbb128aef5eb616616eabb3332203a0e36 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:01:28 -0700 Subject: [PATCH 2/4] Point the emulator policy at in-process instances --- AGENTS.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 244b88678..1f113577d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,8 +23,14 @@ Stable contracts live here. Current setup and server mechanics are in Tests and demos that need an upstream API, OAuth/OIDC provider, or webhook use the published `@executor-js/emulate` emulators rather than stubs. They provide wire-level state, real-shaped credentials, OpenAPI descriptions, and request -ledgers. Create per-run hosted instances through the service's -`/_emulate/instances` control route. +ledgers. Spawn a per-scenario instance IN PROCESS — `createEmulatorInstance` +(`e2e/src/emulator-instance.ts`) for a scenario, `createEmulator` for a target's +own stack. A scenario that runs entirely on this machine must not need the +public internet to pass: a hosted instance makes the runner's network, and its +egress IP's rate limits, part of every assertion. Reach for the hosted +`/_emulate/instances` control route only when exercising the deployed service +IS the point of the scenario, and keep an in-process counterpart for the +contract itself. Emulate is a separate project. Make emulator changes there and consume the published package here; never re-vendor or create a parallel fake inside From 40521a00938f16db1f3560c24123bf08f94840a4 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:07:17 -0700 Subject: [PATCH 3/4] Give a paused execution long enough for a human to approve it The browser approval scenario measures what that costs: 8.7s from page load to the Approve click on a loaded runner, against a 6s ceiling, so the POST came back 404 and the page said the execution was no longer available. 12 shards in two weeks. Production allows 9 minutes; 6s was a test-speed knob borrowed from the expiry scenario, which now sleeps correspondingly longer - the one place the raise is paid for. --- .github/workflows/ci.yml | 3 ++- e2e/setup/mcp-session-timeouts.ts | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49b9c446d..607098d1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -304,7 +304,8 @@ jobs: if: matrix.target == 'cloud' env: MCP_SESSION_TIMEOUT_MS: "3000" - MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS: "6000" + # Must outlive a browser approval; see e2e/setup/mcp-session-timeouts.ts. + MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS: "30000" run: bun scripts/run-ci-shard.ts cloud ${{ matrix['shard-index'] }} working-directory: e2e diff --git a/e2e/setup/mcp-session-timeouts.ts b/e2e/setup/mcp-session-timeouts.ts index dfd478297..047c2e421 100644 --- a/e2e/setup/mcp-session-timeouts.ts +++ b/e2e/setup/mcp-session-timeouts.ts @@ -1,5 +1,18 @@ const DEFAULT_E2E_MCP_SESSION_TIMEOUT_MS = 3_000; -const DEFAULT_E2E_MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS = 6_000; +// A paused execution must outlive a HUMAN deciding on it, and the browser +// approval scenario is the one that measures what that costs: on a loaded CI +// runner, page load to Approve click was 8.7s (the resume page reads the +// session once on load, so nothing keeps the pause warm in between). At 6s the +// approval POST came back 404 and the scenario failed on "Approve sent" — 12 +// shards in the two weeks to 2026-08-20 — with the page showing "this paused +// execution is no longer available", which is a fixture too tight for the +// journey, not a product fault. Production allows 9 minutes. +// +// The cost of raising it is paid in ONE place: cloud/mcp-client-sessions +// derives its teardown wait from this value, so the expiry scenario now sleeps +// this much longer. That is the trade — seconds on one scenario against a +// 30-second timeout plus a re-run of the whole matrix. +const DEFAULT_E2E_MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS = 30_000; const PRODUCTION_MCP_SESSION_TIMEOUT_MS = 5 * 60 * 1000; const PRODUCTION_MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS = 9 * 60 * 1000; From 2e1fdaca0dc1a0da3a656a5e0fa07bab976c3be8 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:19:52 -0700 Subject: [PATCH 4/4] Re-trigger the Cloudflare preview build