diff --git a/packages/contracts/src/runtime/browser-state.mjs b/packages/contracts/src/runtime/browser-state.mjs index 917a918..fbc93b7 100644 --- a/packages/contracts/src/runtime/browser-state.mjs +++ b/packages/contracts/src/runtime/browser-state.mjs @@ -4,12 +4,11 @@ // (partitionKey and the real sameSite included; none is invented), and each // origin keeps its localStorage, sessionStorage and IndexedDB. // -// The document is a superset of agent-browser's state file: `state load` -// (0.33.1, serde without deny_unknown_fields) reads cookies, localStorage and -// sessionStorage from it and ignores the rest, so one file serves both the -// CLI's restore and the IndexedDB restore that follows it. A document -// written before sessionStorage and IndexedDB were captured reads as one -// whose origins hold neither. +// The document keeps agent-browser's state-file shape (cookies, then origins +// with localStorage and sessionStorage) and adds IndexedDB; the runner puts +// all of it back over CDP (runner/src/runtime/browser-state-restore.mjs). A +// document written before sessionStorage and IndexedDB were captured reads as +// one whose origins hold neither. // // Dependencies: the contract-values.mjs leaf only. This is reachable from the // console's workflow bundle. diff --git a/packages/contracts/src/runtime/browser-storage-script.mjs b/packages/contracts/src/runtime/browser-storage-script.mjs index a2660ae..26ab235 100644 --- a/packages/contracts/src/runtime/browser-storage-script.mjs +++ b/packages/contracts/src/runtime/browser-storage-script.mjs @@ -2,8 +2,8 @@ // localStorage, sessionStorage and IndexedDB, read inside the page and // serialized so it can be put back into a fresh profile, and the restore // that puts IndexedDB back (cookies and the two key-value storages go back -// through agent-browser's `state load`; IndexedDB has no CDP write command, -// so it is written by a page script as Playwright's storageScript.ts does). +// in runner/src/runtime/browser-state-restore.mjs; IndexedDB has no CDP write +// command, so it is written by a page script as Playwright's storageScript.ts does). // // Every function below is embedded in the page by source // (Function.prototype.toString) and runs there as a plain script, so each @@ -326,9 +326,8 @@ async function restoreDatabase(indexedDB, database) { /** * Puts the captured databases back on this origin. Each one is deleted - * first: `state load` navigated the real origin a moment ago, and a page - * that creates its database on load (Firebase does) left an empty one - * behind. A database that will not go back is named in `failed`, and the + * first: a page that creates its database on load (Firebase does) may have + * left an empty one behind. A database that will not go back is named in `failed`, and the * rest still go. * * @returns {Promise<{ restored: string[], failed: string[] }>} diff --git a/packages/control-plane/src/human-auth/candidate.ts b/packages/control-plane/src/human-auth/candidate.ts index c857944..a226818 100644 --- a/packages/control-plane/src/human-auth/candidate.ts +++ b/packages/control-plane/src/human-auth/candidate.ts @@ -171,7 +171,7 @@ async function containerScope(sealedPath: string): Promise { * cookies cannot decide: analytics, CSRF and bot-management cookies land on * the product's site before anyone signs in, and plenty of products set no * cookie at all (Supabase, Auth0 on `cacheLocation: "localstorage"`, Amplify - * keep the session in local storage, which `state load` restores the same + * keep the session in local storage, which the restore puts back the same * way). Bytes this host cannot open are taken at the container's word: it * seals a candidate ready only on the same verdict. */ diff --git a/packages/runner/src/runtime/activity-runner/browser.mjs b/packages/runner/src/runtime/activity-runner/browser.mjs index 247aa6a..adb0da1 100644 --- a/packages/runner/src/runtime/activity-runner/browser.mjs +++ b/packages/runner/src/runtime/activity-runner/browser.mjs @@ -66,7 +66,7 @@ export function activityControlLeaseReader() { // The outcome of this leg's sign-in state restore, kept for the first-frame // judgment: a state that did not load is a harness fault, not a wall. let lastSignInStateLoad = null; -/** @returns {{ loaded: boolean, restoreStatus: string | null } | null} */ +/** @returns {{ loaded: boolean } | null} */ export function signInStateRestore() { return lastSignInStateLoad; } @@ -85,19 +85,9 @@ const agentBrowserCli = createAgentBrowserCli({ openFlow: { // The restore's outcome on the record, where an expired or missing // sign-in used to be indistinguishable from a swapped browser. - onStateLoaded: (response) => { - const lifecycle = response?.data?.lifecycle; - lastSignInStateLoad = { - loaded: response?.data?.loaded === true, - restoreStatus: typeof lifecycle?.restoreStatus === "string" ? lifecycle.restoreStatus : null, - }; - emit("evidence", "Sign-in state loaded", { - safeData: { - event: "sign_in_state_loaded", - loaded: response?.data?.loaded === true, - restoreStatus: typeof lifecycle?.restoreStatus === "string" ? lifecycle.restoreStatus : null, - }, - }); + onStateLoaded: () => { + lastSignInStateLoad = { loaded: true }; + emit("evidence", "Sign-in state loaded", { safeData: { event: "sign_in_state_loaded", loaded: true } }); }, // What of the state's IndexedDB went back, on the record beside the // load: a Firebase sign-in that did not is the first frame's wall. diff --git a/packages/runner/src/runtime/agent-browser-arg-policy.mjs b/packages/runner/src/runtime/agent-browser-arg-policy.mjs index 48e75ec..43317cf 100644 --- a/packages/runner/src/runtime/agent-browser-arg-policy.mjs +++ b/packages/runner/src/runtime/agent-browser-arg-policy.mjs @@ -13,8 +13,8 @@ * first-class field, needing no extraArgs at all. * * None of these are the agent's to set (a repo-wide search finds no harness - * use; the harness applies its own sign-in state through the CLI's `state load` - * subcommand, never these fields), so the proxy erases them from every + * use; the harness restores its own sign-in state over CDP in + * browser-state-restore.mjs, never through these fields), so the proxy erases them from every * forwarded call and every advertised schema. A stripped call carries one * factual note so the agent learns the field is not its own. */ diff --git a/packages/runner/src/runtime/agent-browser-cli.mjs b/packages/runner/src/runtime/agent-browser-cli.mjs index 54ab0c4..78c3ffe 100644 --- a/packages/runner/src/runtime/agent-browser-cli.mjs +++ b/packages/runner/src/runtime/agent-browser-cli.mjs @@ -8,35 +8,22 @@ import { prepareDesktopBrowser } from "./desktop-browser.mjs"; * launch a browser of its own. Activity unwraps the JSON reply while the * suite runner keeps the full process result. */ -import { restoreIndexedDbFromStateFile } from "./browser-state-restore.mjs"; +import { restoreStateFile } from "./browser-state-restore.mjs"; import { CLOUDFLARE_CHALLENGE_SCRIPT, cloudflareChallengeFromBrowserResult, startTrustedAccessNavigation, } from "./trusted-access.mjs"; -/** Hands the state-load reply to the runner's observer in its parsed shape. */ -function announceStateLoad(onStateLoaded, unwrapJson, stateLoad) { - if (!onStateLoaded) return; - onStateLoaded(unwrapJson ? stateLoad : stateLoad?.json); -} - /** - * What a state file adds to an open: agent-browser's `state load` (cookies, - * localStorage, sessionStorage), announced, then the IndexedDB restore over - * CDP — not agent-browser's to do, state load ignores it — before the - * product opens. The restore never blocks the open; its outcome is handed - * to the runner's observer. + * What a state file adds to an open: the sign-in restored over CDP + * (browser-state-restore.mjs), announced, before the product opens. A state + * that cannot be read or set throws; what of its storage did not go back is + * handed to the runner's observer. */ -async function loadStateIntoBrowser(flow, statePath, env, options) { - const { agentBrowser, unwrapJson, stateLoadTimeoutMs, onStateLoaded, restoreIndexedDb, onIndexedDbRestored } = flow; - const stateLoad = await agentBrowser( - ["state", "load", statePath], - env, - stateLoadTimeoutMs === null ? options : { ...options, timeoutMs: stateLoadTimeoutMs }, - ); - announceStateLoad(onStateLoaded, unwrapJson, stateLoad); - const restored = await restoreIndexedDb({ agentBrowser, env, statePath, options, unwrapJson }); +async function loadStateIntoBrowser({ onStateLoaded, onIndexedDbRestored }, statePath, env) { + const restored = await restoreStateFile({ env, statePath }); + onStateLoaded?.(); if (restored && onIndexedDbRestored) onIndexedDbRestored(restored); } @@ -77,9 +64,6 @@ export function createAgentBrowserCli({ unwrapJson = false, openFlow = {}, }) { - const { - stateLoadTimeoutMs = null, onStateLoaded = null, onIndexedDbRestored = null, restoreIndexedDb = restoreIndexedDbFromStateFile, - } = openFlow; /** Redacted, bounded post-mortem for a failed agent-browser invocation. */ function agentBrowserDiagnostics(args, result) { @@ -131,10 +115,7 @@ export function createAgentBrowserCli({ */ async function openProductionBrowser(productionUrl, statePath, env, trustedAccess, { timing = UNTIMED, ...options } = {}) { if (desktop) await timing.time("desktop_probe", () => prepareDesktopBrowser(agentBrowser, env, options)); - if (statePath) { - const flow = { agentBrowser, unwrapJson, stateLoadTimeoutMs, onStateLoaded, restoreIndexedDb, onIndexedDbRestored }; - await timing.time("sign_in_load", () => loadStateIntoBrowser(flow, statePath, env, options)); - } + if (statePath) await timing.time("sign_in_load", () => loadStateIntoBrowser(openFlow, statePath, env)); if (!trustedAccess) { await timing.time("page_open", () => agentBrowser(["open", productionUrl], env, options)); return null; diff --git a/packages/runner/src/runtime/browser-state-restore.mjs b/packages/runner/src/runtime/browser-state-restore.mjs index 15e9138..53e85f1 100644 --- a/packages/runner/src/runtime/browser-state-restore.mjs +++ b/packages/runner/src/runtime/browser-state-restore.mjs @@ -1,20 +1,22 @@ -// The IndexedDB half of restoring a sign-in state: agent-browser's `state -// load` puts cookies, localStorage and sessionStorage back and ignores the -// rest, so after it, and before the product opens, every origin that -// carries IndexedDB records gets them back over CDP. Each origin is -// navigated to, with the request answered by an empty document from Fetch -// interception so nothing reaches the product — a service worker the -// product registered during state load is bypassed, or it would answer the -// navigation itself, past the interception — and the page script from -// contracts/runtime/browser-storage-script.mjs rebuilds the databases -// there, once the document on the origin is ShipBash's own and not the -// product page state load left the tab on. Nothing here throws to its -// caller: what did not go back is named in the outcome, and the leg goes +// Restoring a sign-in state into the harness's browser over CDP, in place of +// agent-browser's `state load`: the cookies in one Network.setCookies, then +// every origin that holds storage is navigated to, with the request answered +// by an empty document from Fetch interception so nothing reaches the +// product, and its localStorage, sessionStorage and IndexedDB are written +// there by page scripts (the IndexedDB one from +// contracts/runtime/browser-storage-script.mjs), once the document on the +// origin is ShipBash's own. `state load` visited each origin's real root, +// waited a fixed 500 ms and wrote one key per evaluate, and the product then +// loaded a second time on open; here it loads once, on open. A service +// worker the product registered is bypassed, or it would answer the +// navigation itself, past the interception. Cookies go back as `state load` +// set them (without a partition key). A state that cannot be read, or whose +// cookies the browser refuses, throws, as `state load` did; what of an +// origin's storage did not go back is named in the outcome, and the leg goes // on — the first frame will say whether the sign-in held. import { isRecord } from "@shipbash/contracts/runtime/contract-values.mjs"; import { indexedDbRestoreExpression } from "@shipbash/contracts/runtime/browser-storage-script.mjs"; -import { cdpBaseUrl } from "./browser-state-capture.mjs"; import { readBrowserStateFile } from "./browser-state-seal.mjs"; import { openCdpChannel } from "./cdp-channel.mjs"; @@ -28,11 +30,27 @@ const RESPONSE_HEADERS = [ { name: "Cache-Control", value: "no-store" }, ]; -/** The origins of a state that carry IndexedDB records to put back. */ -export function indexedDbOrigins(state) { +function hasIndexedDbRecords(origin) { + return origin.indexedDB.some((database) => + database.unsupported === undefined && database.stores.some((store) => store.records.length > 0)); +} + +/** The origins of a state with anything to write in a document on them. */ +function originsToRestore(state) { return state.origins.filter((origin) => - origin.indexedDB.some((database) => - database.unsupported === undefined && database.stores.some((store) => store.records.length > 0))); + origin.localStorage.length > 0 || origin.sessionStorage.length > 0 || hasIndexedDbRecords(origin)); +} + +/** The cookies as `state load` set them: Chromium's fields, without the partition key it never passed. */ +function cookieParams(cookies) { + return cookies.map(({ name, value, domain, path, expires, httpOnly, secure, sameSite }) => + ({ name, value, domain, path, expires, httpOnly, secure, ...(sameSite ? { sameSite } : {}) })); +} + +/** Writes both storages of the document's origin in one evaluate. */ +function storageExpression(origin) { + const areas = `[[localStorage, ${JSON.stringify(origin.localStorage)}], [sessionStorage, ${JSON.stringify(origin.sessionStorage)}]]`; + return `(() => { for (const [area, entries] of ${areas}) for (const { name, value } of entries) area.setItem(name, value); })()`; } async function pageSocketUrl(baseUrl) { @@ -96,7 +114,14 @@ function outcomeFrom(evaluated) { return { restored: parsed.restored.map(String), failed: parsed.failed.map(String) }; } -/** One origin: intercept, navigate to its empty document, run the restore script there. */ +/** The storages' write on the origin's document: nothing, or the failure it threw. */ +async function writeStorage(channel, origin) { + if (origin.localStorage.length === 0 && origin.sessionStorage.length === 0) return []; + const written = await channel.command("Runtime.evaluate", { expression: storageExpression(origin), returnByValue: true }); + return isRecord(written?.exceptionDetails) ? [`storage: ${String(written.exceptionDetails.text ?? "the write threw")}`] : []; +} + +/** One origin: intercept, navigate to its empty document, write its storages and databases there. */ async function restoreOrigin(channel, origin, timeoutMs) { await channel.command("Fetch.enable", { patterns: [{ urlPattern: `${origin.origin}/*`, requestStage: "Request" }] }); try { @@ -104,35 +129,42 @@ async function restoreOrigin(channel, origin, timeoutMs) { loaded.catch(() => {}); await channel.command("Page.navigate", { url: `${origin.origin}/` }); await loaded; + const failed = await writeStorage(channel, origin); + if (!hasIndexedDbRecords(origin)) return { restored: [], failed }; const evaluated = await channel.command( "Runtime.evaluate", { expression: indexedDbRestoreExpression(origin.indexedDB), awaitPromise: true, returnByValue: true }, { timeoutMs }, ); - return outcomeFrom(evaluated); + const databases = outcomeFrom(evaluated); + return { restored: databases.restored, failed: [...failed, ...databases.failed] }; } finally { await channel.command("Fetch.disable").catch(() => {}); } } /** - * Puts the IndexedDB of these origins back into the browser behind the CDP - * endpoint, one origin after another on its first page, and leaves that - * page blank for the product to open on. + * Puts a sign-in state back into the browser behind the CDP endpoint: its + * cookies, then each origin's storages and databases, one origin after + * another on its first page, and leaves that page blank for the product to + * open on. * * @param {string} baseUrl `http://host:port` of the CDP endpoint. - * @param {import("@shipbash/contracts/runtime/browser-state.mjs").BrowserOrigin[]} origins + * @param {import("@shipbash/contracts/runtime/browser-state.mjs").BrowserState} state * @param {{ timeoutMs?: number }} [options] * @returns {Promise<{ origins: number, restored: string[], failed: string[] }>} - * `restored` names each database as `name@origin`; `failed` names what did - * not go back and why. + * `origins` counts those with IndexedDB records, `restored` names each + * database as `name@origin`, and `failed` names what did not go back and + * why. */ -export async function restoreIndexedDb(baseUrl, origins, options = {}) { +export async function restoreBrowserState(baseUrl, state, options = {}) { const { timeoutMs = 15_000 } = options; - const outcome = { origins: origins.length, restored: [], failed: [] }; - if (origins.length === 0) return outcome; + const origins = originsToRestore(state); + const outcome = { origins: origins.filter(hasIndexedDbRecords).length, restored: [], failed: [] }; const channel = await openCdpChannel(await pageSocketUrl(baseUrl), { timeoutMs, label: LABEL }); try { + if (state.cookies.length > 0) await channel.command("Network.setCookies", { cookies: cookieParams(state.cookies) }); + if (origins.length === 0) return outcome; channel.on("Fetch.requestPaused", fulfillWithEmptyPage(channel)); await channel.command("Page.enable"); await channel.command("Network.setBypassServiceWorker", { bypass: true }); @@ -153,30 +185,18 @@ export async function restoreIndexedDb(baseUrl, origins, options = {}) { } /** - * The restore for a state file agent-browser just loaded: when any origin - * in it carries IndexedDB records, asks agent-browser for its CDP endpoint - * and puts them back. Null when the file carries none; an outcome naming - * the failure, never a throw, when it could not be read or reached. + * Restores the state file at `statePath` into the browser `env` joins + * (AGENT_BROWSER_CDP). Throws when the file cannot be read or the browser + * refuses its cookies; answers the outcome when an origin carried IndexedDB + * records or something did not go back, and null otherwise. * - * @param {{ - * agentBrowser: Function, - * env: Record, - * statePath: string, - * options?: Record, - * unwrapJson?: boolean, - * }} input `options` are passed to the agent-browser call; `unwrapJson` says - * whether that call answers with the parsed reply or `{ json }`. + * @param {{ env: Record, statePath: string, timeoutMs?: number }} input */ -export async function restoreIndexedDbFromStateFile({ agentBrowser, env, statePath, options = {}, unwrapJson = false }) { - try { - const state = await readBrowserStateFile(statePath, env.AGENT_BROWSER_ENCRYPTION_KEY); - const origins = indexedDbOrigins(state); - if (origins.length === 0) return null; - const reply = await agentBrowser(["get", "cdp-url"], env, options); - return await restoreIndexedDb(cdpBaseUrl(unwrapJson ? reply : reply?.json), origins); - } catch (error) { - return { origins: 0, restored: [], failed: [error instanceof Error ? error.message : String(error)] }; - } +export async function restoreStateFile({ env, statePath, timeoutMs }) { + if (!env.AGENT_BROWSER_CDP) throw new Error(`The ${LABEL} CDP endpoint is unknown (AGENT_BROWSER_CDP).`); + const state = await readBrowserStateFile(statePath, env.AGENT_BROWSER_ENCRYPTION_KEY); + const outcome = await restoreBrowserState(`http://${new URL(env.AGENT_BROWSER_CDP).host}`, state, { timeoutMs }); + return outcome.origins > 0 || outcome.failed.length > 0 ? outcome : null; } /** The outcome as an event carries it: counts and the first failure, never a value. */ diff --git a/packages/runner/src/runtime/suite-runner/browser-idle.mjs b/packages/runner/src/runtime/suite-runner/browser-idle.mjs index 9453d30..079210a 100644 --- a/packages/runner/src/runtime/suite-runner/browser-idle.mjs +++ b/packages/runner/src/runtime/suite-runner/browser-idle.mjs @@ -10,7 +10,7 @@ import { setTimeout as delay } from "node:timers/promises"; import { LANE_BROWSER_CDP_URL_PATH } from "@shipbash/contracts/runtime/browser-lane.mjs"; -import { indexedDbRestoreSummary, restoreIndexedDbFromStateFile } from "../browser-state-restore.mjs"; +import { indexedDbRestoreSummary, restoreStateFile } from "../browser-state-restore.mjs"; import { createControlLeaseReader } from "../control-lease-reader.mjs"; import { prepareDesktopBrowser } from "../desktop-browser.mjs"; import { launchHarnessBrowser } from "../harness-browser.mjs"; @@ -55,11 +55,7 @@ export async function runBrowserIdle() { // arrives from the canonical state file, never from profile bytes. const preloadStatePath = existingAuthStatePath(); if (preloadStatePath) { - await agentBrowser(["state", "load", preloadStatePath], env, { timeoutMs: 30_000 }); - // state load ignores IndexedDB; it goes back over CDP, and never blocks the lane. - const restored = await restoreIndexedDbFromStateFile({ - agentBrowser, env, statePath: preloadStatePath, options: { timeoutMs: 30_000 }, - }); + const restored = await restoreStateFile({ env, statePath: preloadStatePath, timeoutMs: 30_000 }); if (restored) emit("browser", "sign_in_indexeddb_restored", { mode: "browser_idle", ...indexedDbRestoreSummary(restored) }); } await agentBrowser(["open", startUrl], env); diff --git a/packages/runner/src/runtime/suite-runner/browser.mjs b/packages/runner/src/runtime/suite-runner/browser.mjs index 9361c35..6591867 100644 --- a/packages/runner/src/runtime/suite-runner/browser.mjs +++ b/packages/runner/src/runtime/suite-runner/browser.mjs @@ -70,7 +70,6 @@ const agentBrowserCli = createAgentBrowserCli({ ? `agent-browser ${args[0] ?? "command"} timed out.` : `agent-browser ${args[0] ?? "command"} exited with code ${result.code}.`), openFlow: { - stateLoadTimeoutMs: 30_000, onIndexedDbRestored: (outcome) => emit("browser", "sign_in_indexeddb_restored", { mode: "case", ...indexedDbRestoreSummary(outcome) }), }, diff --git a/packages/runner/src/runtime/suite-runner/sign-in-verify.mjs b/packages/runner/src/runtime/suite-runner/sign-in-verify.mjs index 9ee8220..19127c4 100644 --- a/packages/runner/src/runtime/suite-runner/sign-in-verify.mjs +++ b/packages/runner/src/runtime/suite-runner/sign-in-verify.mjs @@ -7,7 +7,7 @@ import { join } from "node:path"; import { isRecord } from "@shipbash/contracts/runtime/contract-values.mjs"; import { captureFirstFrame, settleSignIn } from "../activity-runner/first-frame.mjs"; -import { indexedDbRestoreSummary, restoreIndexedDbFromStateFile } from "../browser-state-restore.mjs"; +import { indexedDbRestoreSummary, restoreStateFile } from "../browser-state-restore.mjs"; import { captureBrowserState, cdpBaseUrl, cdpEndpoints } from "../browser-state-capture.mjs"; import { readBrowserStateFile } from "../browser-state-seal.mjs"; import { validateBrowserConnection } from "../browser-connection.mjs"; @@ -76,8 +76,8 @@ async function clearPreviousSignIn(env, config, statePath) { /** * Loads the requested saved session into the existing browser: the sealed bytes - * onto the container's own tmpfs, `state load`, then the IndexedDB the load - * ignores, then the product. The file goes as soon as the browser holds it. + * onto the container's own tmpfs, restored over CDP (browser-state-restore.mjs), + * then the product. The file goes as soon as the browser holds it. */ export async function openProductWithSession(env, config) { if (config.useCurrentSession) { @@ -94,10 +94,7 @@ export async function openProductWithSession(env, config) { await writeFile(statePath, config.state, { mode: 0o600 }); try { await clearPreviousSignIn(env, config, statePath); - await agentBrowser(["state", "load", statePath], env, { timeoutMs: 30_000 }); - const restored = await restoreIndexedDbFromStateFile({ - agentBrowser, env, statePath, options: { timeoutMs: 30_000 }, - }); + const restored = await restoreStateFile({ env, statePath, timeoutMs: 30_000 }); if (restored) emit("browser", "sign_in_indexeddb_restored", { mode: "sign_in_verify", ...indexedDbRestoreSummary(restored) }); } finally { await rm(statePath, { force: true }).catch(() => {});