diff --git a/.github/workflows/keystone-native.yaml b/.github/workflows/keystone-native.yaml new file mode 100644 index 000000000..8e958468f --- /dev/null +++ b/.github/workflows/keystone-native.yaml @@ -0,0 +1,113 @@ +name: Keystone native smoke + +on: + pull_request: + paths: + - 'src/**' + - 'frontend/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - 'build.rs' + - 'build_support/**' + - 'scripts/**' + - '.github/workflows/keystone-native.yaml' + workflow_dispatch: + # Warm reusable base-branch Rust caches; PR merge-ref caches cannot serve + # subsequent PRs. Keep the same relevant-file scope as the automatic PR gate. + push: + branches: ['main', 'feat/native-only-ui'] + paths: + - 'src/**' + - 'frontend/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - 'build.rs' + - 'build_support/**' + - 'scripts/**' + - '.github/workflows/keystone-native.yaml' + +permissions: + contents: read + +concurrency: + group: keystone-native-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + native-smoke: + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + # DataFusion debug symbols and incremental objects can exhaust runner disk. + CARGO_PROFILE_DEV_DEBUG: '0' + CARGO_PROFILE_TEST_DEBUG: '0' + CARGO_INCREMENTAL: '0' + CARGO_BUILD_JOBS: '4' + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Read repository Rust version + id: rust-version + run: | + python3 - <<'PY' + import os + import tomllib + from pathlib import Path + + channel = tomllib.loads(Path('rust-toolchain.toml').read_text())['toolchain']['channel'] + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'channel={channel}\n') + PY + + - name: Set up pinned Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.rust-version.outputs.channel }} + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + shared-key: keystone-native + cache-on-failure: true + + - name: Install native build dependencies + # Default features exercise the native server without Kafka dependencies. + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential pkg-config cmake clang libssl-dev + + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + + - name: Install Chromium + working-directory: frontend + run: npx playwright install --with-deps chromium + + - name: Exercise Keystone against a fresh native server + # The same local command builds the UI and Rust server, runs focused + # browser tests, and cleans up ephemeral ports and storage. Synthetic + # provider responses cover the product boundary without paid calls or + # provider secrets; this gate does not establish real-provider readiness. + working-directory: frontend + run: npm run test:keystone-native + + - name: Upload native smoke diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: keystone-native-${{ github.run_id }}-${{ github.run_attempt }} + path: frontend/test-results/keystone-native/ + retention-days: 7 + # Dependency/preflight failures can happen before a run directory exists. + if-no-files-found: ignore diff --git a/frontend/docs/keystone-browser-validation.md b/frontend/docs/keystone-browser-validation.md index ec1e00e52..647622de2 100644 --- a/frontend/docs/keystone-browser-validation.md +++ b/frontend/docs/keystone-browser-validation.md @@ -193,3 +193,73 @@ screenshot specs and all **60** Keystone cases. Results are in Screenshot comparisons are only meaningful inside the CI container. On a developer machine the screenshot specs may fail because of local font and rendering differences; that is not a regression signal, and baselines must not be refreshed from such a run. + +## Native socket workflow + +From `frontend`, run: + +```sh +npm ci +# One-time prerequisite; the launcher does not install browsers: +npx playwright install chromium +npm run test:keystone-native +``` + +The launcher requires Node 22.12 or later, this repository's Rust toolchain and its native build +prerequisites. The launcher uses Linux/macOS process groups; Linux has been exercised locally, +macOS remains unverified, and Windows process cleanup is not supported. Use +`npm run test:keystone-native -- --help` for environment options. `TMPDIR` selects temporary +storage, `CARGO_TARGET_DIR` permits reuse of compilation dependencies, and +`PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH` selects an explicitly installed Chromium. + +This command typechecks and builds the actual frontend into a private temporary directory, +builds the Rust library test harness and selects its executable from Cargo JSON output. It +launches only the ignored native socket fixture and only `keystone-native-readiness.spec.ts`. +Both HTTP listeners bind fresh OS-assigned loopback ports. A ready-file handshake and unique +fixture response header are checked before browser mutations. There is no external-server URL, +server reuse, HMR, existing `dist` dependency, native API interception or provider credential. +The ordinary `test:live` suite excludes these cases. + +The fixture exercises actual Actix native routes, durable repository commits, coordinator, +projections, native adapter and SSE. Authentication/current grants, user and dataset metadata, +scoped sample rows and model answers are synthetic. Production login/RBAC middleware, DataFusion, +encrypted credential storage, real provider transport and production deployment are outside this +workflow. The explicit `[diagnostic:hold]` prompt token keeps generation composing for the cancel +case. The cancellation response acknowledges a still-running run; terminal cancellation arrives +through persisted progress after execution unwinds. + +The checks cover draft save/navigation with zero implicit runs, one explicit completion, run-link +navigation/reload, explicit selection of one direct-parent source for a durable child draft, +composing reload/resubscription, cancellation and terminal reload. Cursor replay is compared with +the committed stream's tail. Reload/resubscription does not simulate packet loss or prove every +network-disconnect recovery path. + +Each invocation retains stage logs, its runtime manifest, request/status logs, cancellation +acknowledgement, SSE replay, Playwright JSON/HTML report, screenshots, videos and traces under +`test-results/keystone-native/`. `KEYSTONE_NATIVE_ARTIFACTS` changes the parent folder. +On failure, read the named stage log and open `/report/index.html`; traces can be opened +with `npx playwright show-trace `. Artifacts contain synthetic fixture data. Runtime +storage and the temporary frontend build are removed on success, failure and SIGINT/SIGTERM; +only process groups started by this invocation are stopped. No existing services are killed. + +The `Keystone native smoke` workflow invokes the same command automatically for relevant pull +requests and pushes to `main` or `feat/native-only-ui`, and also supports manual dispatch. It +uploads `test-results/keystone-native/` on success or failure for seven days. The new hosted CI +workflow has not yet been exercised; local execution and static workflow review are separate +validation evidence. + +### Live validation boundary + +On 2026-10-14, a separately authorized paid run using the configured default model +`xiaomi/mimo-v2.6-flash` completed against a local production runtime with synthetic telemetry. +The request explicitly supplied `zdr: true`, `data_collection: "deny"`, +`allow_fallbacks: false`, `require_parameters: true`, +`max_price: { "prompt": 0.2, "completion": 1 }`, `max_tokens: 2048`, and disabled/excluded +reasoning. Success establishes that this request completed; it does not independently verify +provider retention practices or identify the serving endpoint. The adapter did not expose the +actual billed cost. This observation is +separate from the repeatable fixture command and does not establish production readiness or +future endpoint availability. Two independently reproduced P2 findings remain unfixed: reopening +the encrypted credential store with a different master key can still advertise readiness, and +execution failures can present generic messages rather than an actionable cause. Credential, +account and private runtime details are deliberately excluded from these documents. diff --git a/frontend/docs/keystone-foundations.md b/frontend/docs/keystone-foundations.md index 0cf3a78ce..d28ddfb85 100644 --- a/frontend/docs/keystone-foundations.md +++ b/frontend/docs/keystone-foundations.md @@ -8,8 +8,8 @@ initialized. > **Status: merge-readiness fixes implemented and validated.** The frontend CI pipeline and the > clean backend validation both passed on the final source, subject to the baseline and > host-toolchain notes in [validation status](#validation-status). This document makes no -> production-readiness claim: none of these fixes has been deployed or exercised against a running -> server or a paid provider. +> production-readiness claim. Later local runtime and paid-provider observations, with remaining +> readiness findings, are recorded in [native browser validation](keystone-browser-validation.md#live-validation-boundary). ## Implemented backend @@ -496,11 +496,11 @@ These limits come from the merge-readiness review and remain after its fixes. running. Quarantined files are kept until an operator removes them, and a full quarantine leaves unreferenced files counting toward the store's entry cap. Request-body copies of a credential are not zeroized. -- **Provider.** No paid live call has validated the configured model against the required privacy - routing. See [provider foundations](keystone-providers.md#privacy-routing-requirements). -- **Not yet exercised on a running server.** The fixes in this change are covered by automated - tests only. They have not been deployed, no existing installation has been restarted with them, - and no paid inference has run against them. +- **Provider/runtime validation.** One authorized paid MiMo run completed on 2026-10-14 against + synthetic telemetry in a local runtime. This does not establish future endpoint availability or + production readiness; billed cost was not exposed. Wrong-master-key readiness and generic + failure messages remain independently reproduced P2 findings. See + [live validation boundary](keystone-browser-validation.md#live-validation-boundary). - Distributed coordinators, broader tools/SQL, sharing, dashboard promotion, retention/archival, encrypted-store rotation, real token streaming and advisory-provider activation remain follow-up work, not capabilities silently enabled by this increment. diff --git a/frontend/docs/keystone-local-smoke.md b/frontend/docs/keystone-local-smoke.md index 59dd61ae2..bc09b94b5 100644 --- a/frontend/docs/keystone-local-smoke.md +++ b/frontend/docs/keystone-local-smoke.md @@ -4,6 +4,11 @@ A recipe for running native Keystone on one development machine with synthetic d from any other Parseable installation. It is a local test, not a release deployment. Replace every `` with a value for your machine; nothing here is a working credential. +For repeatable browser checks without credentials, use +[`npm run test:keystone-native`](keystone-browser-validation.md#native-socket-workflow). It owns +a fresh synthetic socket fixture and frontend build. The recipe below instead exercises the +production runtime, authentication and query engine. + ## Runtime layout Create one private runtime directory and keep everything for this instance inside it: @@ -130,3 +135,7 @@ internal datasets such as `pstats` are never offered, and the backend rejects th Smoke artifacts (logs, screenshots, browser results) belong outside the repository, for example under `$RUNTIME` or a private scratch directory. They are not part of this change. + +Local production-runtime and paid-provider observations from 2026-10-14, including remaining +unfixed readiness findings, are recorded in the +[live validation boundary](keystone-browser-validation.md#live-validation-boundary). diff --git a/frontend/docs/keystone-providers.md b/frontend/docs/keystone-providers.md index d97075d5e..516f79bfc 100644 --- a/frontend/docs/keystone-providers.md +++ b/frontend/docs/keystone-providers.md @@ -2,8 +2,9 @@ The native runtime now includes optional live generation and encrypted credential entry. They remain unavailable until operator deployment inputs and an admin-entered credential -are present. Automated verification uses fixtures and local TLS only. No paid call has -validated the configured model against the privacy routing requirements below. +are present. Automated verification uses fixtures and local TLS only. A separately authorized +paid MiMo run completed on 2026-10-14; actual billed cost was not exposed. See the +[live validation boundary](keystone-browser-validation.md#live-validation-boundary). `ModelProvider` and the separate advisory `DecisionProvider` receive a `FrozenProviderContext` verified from the server-built `PinnedContext`. The wire @@ -116,8 +117,8 @@ that meets both requirements within the price caps. The documentation does not state the `zdr: true` behavior when no ZDR endpoint exists; Keystone assumes the request fails and never falls back. Such a run fails with the upstream rejection rather than sending data to a less restricted endpoint. The default model was chosen -for price; its availability under these requirements has not been tested with a -paid call, so no call has confirmed which endpoints serve it. Before +for price; one controlled paid MiMo run completed on 2026-10-14. This does not +guarantee future availability or identify every qualifying endpoint. Before enabling Keystone, check the model's endpoint list and run one controlled call; choose another model with `KEYSTONE_GENERATION_MODEL` if none qualifies. diff --git a/frontend/e2e-live/keystone-native-readiness.spec.ts b/frontend/e2e-live/keystone-native-readiness.spec.ts new file mode 100644 index 000000000..50488ffe8 --- /dev/null +++ b/frontend/e2e-live/keystone-native-readiness.spec.ts @@ -0,0 +1,309 @@ +// Actual native HTTP/SSE, coordinator and durable repository; synthetic auth/rows/model. +// The launcher proves fixture identity before this suite performs any mutations. +import { test, expect, type Page, type TestInfo } from '@playwright/test'; +import { writeFile } from 'node:fs/promises'; + +const base = '/api/prism/v1/keystone/native'; +type NativeCall = { method: string; url: string; body: unknown; status?: number }; +type StartedRun = { id: string; scope: unknown }; +type EventData = { + runId: string; + nodeId: string; + sequence: number; + kind: string; + payload: unknown; +}; +type EventFrame = { id: number; event: string; data: EventData }; +const pageErrors = new Map(); +test.beforeEach(async ({ request }) => { + const response = await request.get(`${base}/capabilities`); + expect(response.status()).toBe(200); + expect(process.env.KEYSTONE_NATIVE_ID).toBeTruthy(); + expect(response.headers()['x-keystone-native-fixture']).toBe(process.env.KEYSTONE_NATIVE_ID); +}); +async function ready(page: Page) { + await expect(page.getByTestId('ks-workspace')).toHaveAttribute('data-state', 'ready'); +} +async function create(page: Page, prompt: string) { + await page.goto('/keystone'); + await page.getByTestId('ks-new-prompt').fill(prompt); + await page.getByRole('checkbox', { name: 'logs', exact: true }).check(); + await page.getByTestId('ks-new-save').click(); + await ready(page); + const node = await page.getByTestId('ks-inspector').getAttribute('data-node-id'); + expect(node).toBeTruthy(); + return node!; +} +function readRequestBody(body: string | null): unknown { + if (!body) return null; + try { + return JSON.parse(body); + } catch { + return { unparsedBody: body }; + } +} +function observe(page: Page, info: TestInfo) { + const calls: NativeCall[] = []; + page.on('request', (request) => { + if (request.url().includes(base)) + calls.push({ + method: request.method(), + url: request.url(), + body: readRequestBody(request.postData()), + }); + }); + // afterEach writes even if a browser assertion fails. + const errors: string[] = []; + page.on('pageerror', (error) => errors.push(error.message)); + page.on('response', (response) => { + const call = [...calls] + .reverse() + .find( + (entry) => entry.url === response.url() && entry.method === response.request().method(), + ); + if (call) call.status = response.status(); + }); + pageErrors.set(info.testId, errors); + logs.set(info.testId, calls); + return calls; +} +const logs = new Map(); +test.afterEach(async ({}, info) => { + await writeFile( + info.outputPath('native-requests.json'), + JSON.stringify(logs.get(info.testId) ?? [], null, 2), + ); + await info.attach('native-requests', { + path: info.outputPath('native-requests.json'), + contentType: 'application/json', + }); + await info.attach('page-errors', { + body: JSON.stringify(pageErrors.get(info.testId) ?? []), + contentType: 'application/json', + }); + expect(pageErrors.get(info.testId) ?? []).toEqual([]); + logs.delete(info.testId); + pageErrors.delete(info.testId); +}); +const runs = (calls: NativeCall[]) => + calls.filter((call) => call.method === 'POST' && call.url.endsWith('/runs')); +async function start(page: Page): Promise { + const accepted = page.waitForResponse( + (response) => response.request().method() === 'POST' && response.url().endsWith('/runs'), + ); + await page.getByTestId('ks-run').click(); + const response = await accepted; + expect(response.status()).toBe(202); + const body = (await response.json()) as { run: StartedRun }; + expect(body.run.id).toBeTruthy(); + return body.run; +} +function frames(body: string): EventFrame[] { + return body + .split(/\r?\n\r?\n/) + .filter((block) => block.trim() && !block.startsWith(':')) + .map((block) => { + const lines = block.split(/\r?\n/); + const id = Number(lines.find((line) => line.startsWith('id: '))?.slice(4)); + const event = lines.find((line) => line.startsWith('event: '))?.slice(7); + const data = JSON.parse( + lines + .filter((line) => line.startsWith('data: ')) + .map((line) => line.slice(6)) + .join('\n'), + ) as EventData; + expect(Number.isSafeInteger(id)).toBe(true); + expect(event).toBe('keystone'); + return { id, event: event!, data }; + }); +} +async function saveArtifact(info: TestInfo, name: string, body: string, contentType: string) { + const path = info.outputPath(name); + await writeFile(path, body); + await info.attach(name, { path, contentType }); +} +async function replay( + page: Page, + calls: NativeCall[], + run: StartedRun, + node: string, + terminal: string, + info: TestInfo, +) { + const events = calls.find((call) => + new URL(call.url).pathname.endsWith(`/runs/${run.id}/events`), + ); + expect(events).toBeTruthy(); + const url = events!.url.split('?')[0]; + const response = await page.request.get(`${url}?after=0`); + expect(response.status()).toBe(200); + expect(response.headers()['content-type']).toContain('text/event-stream'); + const body = await response.text(); + await saveArtifact(info, 'native-sse.txt', body, 'text/plain'); + const committed = frames(body); + expect(committed.length).toBeGreaterThan(4); + expect(committed.map((frame) => frame.id)).toEqual(committed.map((_, index) => index + 1)); + for (const frame of committed) + expect(frame.data).toMatchObject({ runId: run.id, nodeId: node, sequence: frame.id }); + expect(committed[committed.length - 1].data.kind).toBe(terminal); + const after = committed[Math.floor(committed.length / 2) - 1].id; + const resumed = await page.request.get(`${url}?after=${after}`); + expect(resumed.status()).toBe(200); + expect(resumed.headers()['content-type']).toContain('text/event-stream'); + const resumedBody = await resumed.text(); + await saveArtifact(info, 'native-sse-replay.txt', resumedBody, 'text/plain'); + expect(frames(resumedBody)).toEqual(committed.filter((frame) => frame.id > after)); +} + +test('native save, explicit run, run link/reload, selected direct-parent evidence', async ({ + page, +}, info) => { + const calls = observe(page, info); + let node = ''; + let run: StartedRun; + await test.step('Save a durable draft and navigate without implicit inference', async () => { + node = await create(page, `Browser diagnostic ${info.testId}`); + await expect(page.getByTestId(`ks-node-${node}`)).toHaveAttribute('data-status', 'draft'); + const url = page.url(); + await page.reload(); + await ready(page); + await page.goto(url); + await ready(page); + expect(runs(calls)).toHaveLength(0); + }); + await test.step('Explicit Run completes through real native SSE; run link survives reload', async () => { + run = await start(page); + await expect(page.getByTestId(`ks-node-${node}`)).toHaveAttribute('data-status', 'completed', { + timeout: 20_000, + }); + await expect(page.getByTestId('ks-inspector-answer')).toContainText( + 'Synthetic diagnostic answer', + ); + expect(runs(calls)).toHaveLength(1); + const runLink = page.url(); + expect(new URL(runLink).searchParams.get('run')).toBe(run.id); + await page.goto('/keystone'); + await page.goto(runLink); + await ready(page); + await page.reload(); + await ready(page); + await expect(page.getByTestId(`ks-node-${node}`)).toHaveAttribute('data-status', 'completed'); + expect(runs(calls)).toHaveLength(1); + }); + await test.step('Select exactly one source for a draft child; never implicitly run it', async () => { + await page.getByTestId('ks-follow-up').click(); + const sources = page.getByTestId('ks-context-sources').getByRole('checkbox'); + await expect(sources.first()).toBeVisible(); + for (const source of await sources.all()) await expect(source).not.toBeChecked(); + await sources.first().check(); + const selected = (await sources.first().getAttribute('data-testid'))!.replace( + 'ks-context-source-', + '', + ); + await page.getByTestId('ks-composer-prompt').fill('Child synthetic diagnostic'); + const storedResponse = page.waitForResponse( + (response) => response.request().method() === 'POST' && response.url().endsWith('/nodes'), + ); + await page.getByTestId('ks-composer-submit').click(); + const stored = await storedResponse; + expect(stored.status()).toBe(201); + const storedBody = (await stored.json()) as { + node: { + id: string; + scope: unknown; + parent: { nodeId: string; version: number }; + selectedRunId: string | null; + }; + context: { + manifest: { + evidence: { id: string }[]; + scope: unknown; + directParent: { nodeId: string; version: number }; + parentExportHash: string; + contextPolicy: string; + }; + }; + }; + expect(storedBody.context.manifest.evidence.map((evidence) => evidence.id)).toEqual([selected]); + await expect(page.getByTestId(`ks-node-${storedBody.node.id}`)).toHaveAttribute( + 'data-status', + 'draft', + ); + await expect(page.getByTestId('ks-inspector')).not.toHaveAttribute('data-node-id', node); + const child: unknown = stored.request().postDataJSON(); + expect(child).toMatchObject({ + parentNodeId: node, + contextPolicy: 'direct_parent_v1', + exportedEvidenceIds: [selected], + requestedScope: null, + parentExportSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + expect(storedBody.node.scope).toEqual(run.scope); + expect(storedBody.node.selectedRunId).toBeNull(); + expect(storedBody.node.parent).toEqual({ nodeId: node, version: 1 }); + expect(storedBody.context.manifest).toMatchObject({ + contextPolicy: 'direct_parent_v1', + scope: run.scope, + directParent: { nodeId: node, version: 1 }, + parentExportHash: (child as { parentExportSha256: string }).parentExportSha256, + }); + expect(runs(calls)).toHaveLength(1); + await page.reload(); + await ready(page); + await expect(page.getByTestId(`ks-node-${storedBody.node.id}`)).toHaveAttribute( + 'data-status', + 'draft', + ); + expect(runs(calls)).toHaveLength(1); + }); + await test.step('Replay committed SSE strictly after the supplied cursor', async () => + replay(page, calls, run, node, 'completed', info)); +}); + +test('native composing reload/resubscribe, actual cancellation acknowledgement and terminal reload', async ({ + page, +}, info) => { + const calls = observe(page, info); + const node = await create(page, `[diagnostic:hold] browser diagnostic ${info.testId}`); + let run: StartedRun; + await test.step('Reload while composing and resubscribe without another Run', async () => { + run = await start(page); + await expect(page.getByTestId(`ks-node-${node}`)).toHaveAttribute('data-status', 'composing', { + timeout: 15_000, + }); + const before = calls.filter((call) => call.url.includes('/events?')).length; + await page.reload(); + await ready(page); + await expect(page.getByTestId('ks-run-live')).toContainText('Live. Steps update'); + await expect(page.getByTestId(`ks-node-${node}`)).toHaveAttribute('data-status', 'composing'); + await expect + .poll(() => calls.filter((call) => call.url.includes('/events?')).length) + .toBeGreaterThan(before); + expect(runs(calls)).toHaveLength(1); + }); + await test.step('Cancel acknowledges a nonterminal run before terminal cancelled replay', async () => { + const acknowledgement = page.waitForResponse( + (response) => response.request().method() === 'POST' && response.url().endsWith('/cancel'), + ); + await page.getByTestId('ks-cancel-run').click(); + const response = await acknowledgement; + expect(response.status()).toBe(200); + const body: unknown = await response.json(); + await saveArtifact( + info, + 'native-cancel-ack.json', + JSON.stringify(body, null, 2), + 'application/json', + ); + expect(body).toMatchObject({ run: { id: run.id, status: 'running' } }); + await expect(page.getByTestId(`ks-node-${node}`)).toHaveAttribute('data-status', 'cancelled', { + timeout: 15_000, + }); + await page.reload(); + await ready(page); + await expect(page.getByTestId(`ks-node-${node}`)).toHaveAttribute('data-status', 'cancelled'); + expect(runs(calls)).toHaveLength(1); + }); + await test.step('Replay committed cancellation SSE after a cursor', async () => + replay(page, calls, run, node, 'cancelled', info)); +}); diff --git a/frontend/package.json b/frontend/package.json index 689c9c55b..ce844a027 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,9 +17,10 @@ "storybook": "storybook dev -p 6006 --no-open --disable-telemetry", "build-storybook": "storybook build --disable-telemetry", "check": "npm run format:check && npm run build && npm test && npm run test:ui && npm run test:prod && npm run test:stories", - "format": "prettier --write src e2e e2e-live e2e-prod story-tests .storybook *.ts *.json docs README.md", - "format:check": "prettier --check src e2e e2e-live e2e-prod story-tests .storybook *.ts *.json docs README.md", + "format": "prettier --write scripts src e2e e2e-live e2e-prod story-tests .storybook *.ts *.json docs README.md", + "format:check": "prettier --check scripts src e2e e2e-live e2e-prod story-tests .storybook *.ts *.json docs README.md", "test:stories": "npm run build-storybook && playwright test -c playwright.storybook.config.ts", + "test:keystone-native": "node scripts/keystone-native.mjs", "test:live": "playwright test -c playwright.live.config.ts" }, "dependencies": { diff --git a/frontend/playwright.live.config.ts b/frontend/playwright.live.config.ts index 4185ecaae..e7648bf67 100644 --- a/frontend/playwright.live.config.ts +++ b/frontend/playwright.live.config.ts @@ -4,6 +4,7 @@ import { defineConfig, devices } from '@playwright/test'; export default defineConfig({ testDir: './e2e-live', testMatch: '**/*.spec.ts', + testIgnore: 'keystone-native-readiness.spec.ts', workers: 1, fullyParallel: false, timeout: 45_000, diff --git a/frontend/playwright.native.config.ts b/frontend/playwright.native.config.ts new file mode 100644 index 000000000..ea6a43a27 --- /dev/null +++ b/frontend/playwright.native.config.ts @@ -0,0 +1,35 @@ +import { defineConfig, devices } from '@playwright/test'; + +const artifacts = process.env.KEYSTONE_NATIVE_RUN_ARTIFACTS; +if (!artifacts || !process.env.KEYSTONE_NATIVE_URL || !process.env.KEYSTONE_NATIVE_ID) { + throw new Error( + 'Use npm run test:keystone-native: the launcher owns the isolated fixture and build.', + ); +} +if (new URL(process.env.KEYSTONE_NATIVE_URL).hostname !== '127.0.0.1') { + throw new Error('Native fixture URL must be loopback.'); +} +export default defineConfig({ + testDir: './e2e-live', + testMatch: 'keystone-native-readiness.spec.ts', + workers: 1, + fullyParallel: false, + retries: 0, + timeout: 60_000, + outputDir: `${artifacts}/browser`, + reporter: [ + ['list'], + ['json', { outputFile: `${artifacts}/results.json` }], + ['html', { open: 'never', outputFolder: `${artifacts}/report` }], + ], + use: { + ...devices['Desktop Chrome'], + baseURL: process.env.KEYSTONE_NATIVE_URL, + trace: 'on', + video: 'on', + screenshot: 'on', + launchOptions: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH + ? { executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH } + : {}, + }, +}); diff --git a/frontend/scripts/keystone-native.mjs b/frontend/scripts/keystone-native.mjs new file mode 100644 index 000000000..d909e7fe6 --- /dev/null +++ b/frontend/scripts/keystone-native.mjs @@ -0,0 +1,354 @@ +// Own every process and byte served: never attach to a developer or production server. +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { createReadStream, createWriteStream } from 'node:fs'; +import { access, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { createServer, request } from 'node:http'; +import { tmpdir } from 'node:os'; +import { dirname, extname, join, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const frontend = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const repository = resolve(frontend, '..'); +if (process.argv.includes('--help')) { + console.log(`Usage: npm run test:keystone-native [-- --help] +Requires Node >=22.12, npm ci, the repository Rust toolchain, and Playwright Chromium. +Install Chromium explicitly with: npx playwright install chromium +Linux/macOS only. Builds this frontend and Rust harness; uses fresh loopback ports/storage. +Optional: TMPDIR, CARGO_TARGET_DIR, PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, +KEYSTONE_NATIVE_ARTIFACTS (parent directory for retained per-run logs and reports). +No credentials or real provider calls. No external server URL is accepted.`); + process.exit(0); +} +if (process.platform === 'win32') + throw new Error('This process-group launcher supports Linux/macOS only.'); +const id = randomUUID(); +const artifacts = join( + resolve(process.env.KEYSTONE_NATIVE_ARTIFACTS || join(frontend, 'test-results/keystone-native')), + id, +); +await mkdir(artifacts, { recursive: true }); +console.log(`Native browser artifacts: ${artifacts}`); +const runtime = await mkdtemp(join(tmpdir(), 'keystone-native-')); +// Keep group ownership after a leader exits, so grandchildren are still cleaned. +const children = new Set(); +const childEnv = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => + [ + 'PATH', + 'HOME', + 'TMPDIR', + 'LANG', + 'LC_ALL', + 'CARGO_HOME', + 'RUSTUP_HOME', + 'RUSTUP_TOOLCHAIN', + 'RUSTC', + 'RUSTFLAGS', + 'RUSTDOCFLAGS', + 'CC', + 'CXX', + 'AR', + 'PKG_CONFIG_PATH', + 'CFLAGS', + 'CXXFLAGS', + 'LDFLAGS', + 'RUSTC_WRAPPER', + 'CMAKE', + 'CMAKE_GENERATOR', + 'SDKROOT', + 'DEVELOPER_DIR', + 'MACOSX_DEPLOYMENT_TARGET', + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'ALL_PROXY', + 'NO_PROXY', + 'http_proxy', + 'https_proxy', + 'all_proxy', + 'no_proxy', + 'SSL_CERT_FILE', + 'SSL_CERT_DIR', + 'PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH', + 'PLAYWRIGHT_BROWSERS_PATH', + ].includes(key) || key.startsWith('CARGO_'), + ), +); +childEnv.VITE_ENABLE_DEMO = 'false'; +// Proxy policies must never send our synthetic loopback traffic off this machine. +childEnv.NO_PROXY = `${childEnv.NO_PROXY || ''},127.0.0.1,localhost,::1`; +childEnv.no_proxy = `${childEnv.no_proxy || ''},127.0.0.1,localhost,::1`; +let server; +let interrupted = false; +const stop = (child, signal = 'SIGTERM') => { + if (child.pid) { + try { + process.kill(-child.pid, signal); + } catch (error) { + if (error.code !== 'ESRCH') throw error; + } + } +}; +let interruptionTimer; +for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) + process.on(signal, () => { + interrupted = true; + for (const child of children) stop(child); + interruptionTimer ??= setTimeout(() => { + for (const child of children) stop(child, 'SIGKILL'); + }, 3000); + }); +function launch(label, command, args, options = {}) { + if (interrupted) throw new Error('Interrupted'); + console.log(`Stage: ${label}`); + const output = createWriteStream(join(artifacts, `${label}.log`)); + const child = spawn(command, args, { + cwd: frontend, + env: childEnv, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + ...options, + }); + children.add(child); + child.stdout.pipe(output, { end: false }); + child.stderr.pipe(output, { end: false }); + if (label === 'browser') child.stdout.pipe(process.stdout, { end: false }); + const started = Date.now(); + const progress = setInterval( + () => + console.log( + `Stage: ${label} (${Math.round((Date.now() - started) / 1000)}s); ${join(artifacts, `${label}.log`)}`, + ), + 30_000, + ); + const done = new Promise((resolveDone, reject) => { + child.once('error', (error) => { + clearInterval(progress); + output.end(); + reject( + new Error( + `${label}: ${error.message}. Check prerequisites with npm run test:keystone-native -- --help`, + ), + ); + }); + child.once('close', (code, signal) => { + clearInterval(progress); + output.end(); + if (code === 0) resolveDone(); + else + reject( + new Error(`${label} exited ${signal || code}; see ${join(artifacts, `${label}.log`)}`), + ); + }); + }); + // Servers can fail before readiness; preserve the rejection for the awaited lifecycle. + done.catch(() => {}); + return { child, done }; +} +async function run(label, command, args, options) { + await launch(label, command, args, options).done; +} +async function waitReady(path, processDone) { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + if (interrupted) throw new Error('Interrupted'); + try { + return JSON.parse(await readFile(path, 'utf8')); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + await Promise.race([ + new Promise((r) => setTimeout(r, 100)), + processDone.then(() => { + throw new Error('Harness exited before readiness'); + }), + ]); + } + throw new Error('Harness did not publish readiness within 30 seconds; see harness.log'); +} +try { + const { chromium } = await import('@playwright/test').catch(() => { + throw new Error('Frontend dependencies missing: run npm ci from frontend first.'); + }); + await access(process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || chromium.executablePath()).catch( + () => { + throw new Error( + 'Chromium missing: run npx playwright install chromium, or set PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH.', + ); + }, + ); + await run('rust-preflight', 'cargo', ['--version'], { cwd: repository }); + await run('frontend-typecheck', process.execPath, [ + 'node_modules/typescript/bin/tsc', + '--noEmit', + ]); + const build = join(runtime, 'build'); + await run('frontend-build', process.execPath, [ + 'node_modules/vite/bin/vite.js', + 'build', + '--base=/', + '--outDir', + build, + ]); + const classic = join(runtime, 'classic'); + await mkdir(classic); + await writeFile( + join(classic, 'index.html'), + 'Classic UI excluded from native fixture', + ); + const cargo = launch( + 'rust-build', + 'cargo', + ['test', '--locked', '--lib', '--no-run', '--message-format=json'], + { cwd: repository, env: { ...childEnv, LOCAL_ASSETS_PATH: classic, NEXT_ASSETS_PATH: build } }, + ); + cargo.child.stdout.setEncoding('utf8'); + let buffer = ''; + let executable; + cargo.child.stdout.on('data', (chunk) => { + buffer += chunk; + const lines = buffer.split('\n'); + buffer = lines.pop(); + for (const line of lines) { + try { + const message = JSON.parse(line); + if ( + message.reason === 'compiler-artifact' && + message.target.name === 'parseable' && + message.target.kind.includes('lib') && + message.profile.test && + message.executable + ) + executable = message.executable; + } catch { + /* Cargo diagnostics are retained verbatim. */ + } + } + }); + await cargo.done; + if (!executable) throw new Error('Cargo did not report the parseable library test executable.'); + const readyFile = join(runtime, 'ready.json'); + const harness = launch( + 'harness', + executable, + [ + 'keystone::api::live_tests::native_browser_diagnostic_server', + '--ignored', + '--exact', + '--nocapture', + ], + { + cwd: repository, + env: { + ...childEnv, + TMPDIR: runtime, + KEYSTONE_DIAGNOSTIC_PORT: '0', + KEYSTONE_DIAGNOSTIC_DIRECTORY: join(runtime, 'repository'), + KEYSTONE_DIAGNOSTIC_ID: id, + KEYSTONE_DIAGNOSTIC_READY_FILE: readyFile, + KEYSTONE_DIAGNOSTIC_SECONDS: '600', + }, + }, + ); + const ready = await waitReady(readyFile, harness.done); + if (ready.identity !== id) throw new Error('Harness readiness identity mismatch'); + const probe = await fetch(`${ready.url}/api/prism/v1/keystone/native/capabilities`, { + signal: AbortSignal.timeout(10_000), + }); + if (!probe.ok || probe.headers.get('x-keystone-native-fixture') !== id) + throw new Error('Harness capabilities/identity readiness check failed'); + const types = { + '.html': 'text/html', + '.js': 'text/javascript', + '.css': 'text/css', + '.svg': 'image/svg+xml', + '.json': 'application/json', + '.png': 'image/png', + '.woff2': 'font/woff2', + }; + server = createServer(async (incoming, outgoing) => { + if (incoming.url.startsWith('/api/')) { + const upstream = request( + new URL(incoming.url, ready.url), + { method: incoming.method, headers: incoming.headers }, + (response) => { + outgoing.writeHead(response.statusCode, response.headers); + response.pipe(outgoing); + }, + ); + upstream.on('error', () => { + if (!outgoing.headersSent) outgoing.writeHead(502); + outgoing.end(); + }); + outgoing.on('close', () => upstream.destroy()); + incoming.pipe(upstream); + return; + } + try { + const pathname = decodeURIComponent(new URL(incoming.url, 'http://localhost').pathname); + let path = resolve(build, `.${pathname}`); + if (!path.startsWith(`${build}${sep}`)) path = join(build, 'index.html'); + try { + if (!(await stat(path)).isFile()) path = join(build, 'index.html'); + } catch { + path = join(build, 'index.html'); + } + outgoing.writeHead(200, { + 'content-type': types[extname(path)] || 'application/octet-stream', + 'cache-control': 'no-store', + }); + createReadStream(path).pipe(outgoing); + } catch { + outgoing.writeHead(400); + outgoing.end(); + } + }); + await new Promise((r, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', r); + }); + const url = `http://127.0.0.1:${server.address().port}`; + await writeFile( + join(artifacts, 'run.json'), + JSON.stringify( + { identity: id, frontend: url, harness: ready.url, executable, synthetic: true }, + null, + 2, + ), + ); + await Promise.race([ + run( + 'browser', + process.execPath, + ['node_modules/@playwright/test/cli.js', 'test', '-c', 'playwright.native.config.ts'], + { + env: { + ...childEnv, + KEYSTONE_NATIVE_URL: url, + KEYSTONE_NATIVE_ID: id, + KEYSTONE_NATIVE_RUN_ARTIFACTS: artifacts, + }, + }, + ), + harness.done.then(() => { + throw new Error('Harness exited during browser checks'); + }), + ]); + console.log('Native browser checks passed.'); +} catch (error) { + console.error(error.message); + console.error(`Inspect stage logs and Playwright artifacts: ${artifacts}`); + process.exitCode = interrupted ? 130 : 1; +} finally { + clearTimeout(interruptionTimer); + const owned = [...children]; + for (const child of owned) stop(child); + const deadline = Date.now() + 3000; + while (owned.some((c) => c.exitCode === null && c.signalCode === null) && Date.now() < deadline) + await new Promise((r) => setTimeout(r, 50)); + for (const child of owned) stop(child, 'SIGKILL'); + server?.closeAllConnections(); + if (server) await new Promise((r) => server.close(r)); + await rm(runtime, { recursive: true, force: true }); +} diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index 7922e47c3..0eb55e826 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -20,6 +20,7 @@ ".storybook", "e2e", "playwright.live.config.ts", + "playwright.native.config.ts", "e2e-live", "playwright.prod.config.ts", "e2e-prod" diff --git a/src/keystone/api_live_tests.rs b/src/keystone/api_live_tests.rs index dfd4786ce..86f1d82b2 100644 --- a/src/keystone/api_live_tests.rs +++ b/src/keystone/api_live_tests.rs @@ -152,6 +152,131 @@ fn access(_: &HttpRequest) -> std::result::Result { Ok(BoundAccess::unchanged(owner1())) } +/// Diagnostic model: `[diagnostic:hold]` in the frozen context delays the answer long +/// enough to exercise real native cancellation. Dropping the future stops the wait. +struct BrowserDiagnosticModel; +#[async_trait] +impl crate::keystone::providers::ModelProvider for BrowserDiagnosticModel { + async fn propose( + &self, + context: &crate::keystone::providers::FrozenProviderContext, + ) -> std::result::Result { + let delay = if context + .serialized() + .to_ascii_lowercase() + .contains("[diagnostic:hold]") + { + 90 + } else { + 2 + }; + tokio::time::sleep(std::time::Duration::from_secs(delay)).await; + Ok(ModelProposal { + explanation: "Synthetic diagnostic answer from a real native run; no provider call." + .into(), + }) + } +} + +/// Opt-in socket harness, deliberately excluded from ordinary library tests. +/// Real native routes, projections, SSE, coordinator and durable repository; +/// synthetic fixed owner/grants, dataset metadata, rows and model. No production +/// runtime, login middleware, query engine, credentials or provider transport. +/// `frontend/npm run test:keystone-native` owns the frontend, socket and lifecycle. +#[actix_web::test] +#[ignore = "isolated browser diagnostic; launched by npm run test:keystone-native"] +async fn native_browser_diagnostic_server() { + let port: u16 = std::env::var("KEYSTONE_DIAGNOSTIC_PORT") + .unwrap_or_else(|_| "0".into()) + .parse() + .expect("diagnostic port must be a u16"); + let lifetime: u64 = std::env::var("KEYSTONE_DIAGNOSTIC_SECONDS") + .unwrap_or_else(|_| "600".into()) + .parse() + .expect("diagnostic lifetime must be seconds"); + assert!((1..=1800).contains(&lifetime)); + let temporary = tempfile::Builder::new() + .prefix("keystone-native-browser-") + .tempdir() + .unwrap(); + let directory = std::env::var_os("KEYSTONE_DIAGNOSTIC_DIRECTORY") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| temporary.path().to_owned()); + std::fs::create_dir_all(&directory).unwrap(); + let repo = Arc::new(NativeRepository::open(&directory).unwrap()); + let allowed = Arc::new(AtomicBool::new(true)); + let coordinator = Arc::new(Coordinator::new_with_reads( + repo.clone(), + Arc::new(BrowserDiagnosticModel), + Arc::new(Authority(allowed.clone())), + Arc::new(Rows), + )); + let execution: Arc = Arc::new(Execution { + repo: repo.clone(), + coordinator, + allowed, + }); + let identity = std::env::var("KEYSTONE_DIAGNOSTIC_ID") + .expect("launcher must supply a unique fixture identity"); + let response_identity = identity.clone(); + let server = actix_web::HttpServer::new(move || { + App::new() + .wrap( + actix_web::middleware::DefaultHeaders::new() + .add(("x-keystone-native-fixture", response_identity.clone())), + ) + .service(web::scope("/api/prism/v1").service(scope( + web::Data::new(ApiState { + scope_preparer: None, + repository: Some(repo.clone()), + authorize: access, + mode: "all", + execution: Some(execution.clone()), + }), + |route| route, + ))) + .route( + "/api/v1/logstream", + web::get().to(|| async { HttpResponse::Ok().json(json!([{"name":"logs"}])) }), + ) + .route( + "/api/v1/logstream/logs/info", + web::get().to(|| async { + HttpResponse::Ok() + .json(json!({"streamType":"UserDefined","telemetryType":"logs"})) + }), + ) + .route( + "/api/v1/users/{id}", + web::get().to(|| async { + HttpResponse::Ok() + .json(json!({"id":"owner1","username":"synthetic-native-diagnostic"})) + }), + ) + }) + .workers(1) + .bind(("127.0.0.1", port)) + .expect("diagnostic loopback socket must bind"); + let address = server.addrs()[0]; + let server = server.run(); + let handle = server.handle(); + let task = actix_web::rt::spawn(server); + if let Some(path) = std::env::var_os("KEYSTONE_DIAGNOSTIC_READY_FILE") { + let path = std::path::PathBuf::from(path); + let pending = path.with_extension("tmp"); + std::fs::write( + &pending, + json!({"url":format!("http://{address}"),"identity":identity}).to_string(), + ) + .unwrap(); + std::fs::rename(pending, path).unwrap(); + } + eprintln!("Synthetic native browser diagnostic listening on {address}; expires in {lifetime}s"); + tokio::time::sleep(std::time::Duration::from_secs(lifetime)).await; + handle.stop(true).await; + task.await.unwrap().unwrap(); +} + #[actix_web::test] async fn real_routes_run_replay_branch_projection_layout_and_restart() { let directory = tempfile::tempdir().unwrap();