From a77ca3f63f8f05d77b1626af5dab3f9e708c8ae3 Mon Sep 17 00:00:00 2001 From: Diego Carlino Date: Sat, 1 Aug 2026 04:21:57 +0200 Subject: [PATCH 1/6] feat(mobile): add live preview and markup --- apps/desktop/src/ipc/methods/preview.test.ts | 37 + apps/desktop/src/ipc/methods/preview.ts | 7 +- apps/desktop/src/preload.ts | 7 +- apps/desktop/src/preview/Manager.test.ts | 345 +++++ apps/desktop/src/preview/Manager.ts | 212 ++- .../src/preview/PickedElementPayload.test.ts | 145 ++ .../src/preview/PickedElementPayload.ts | 111 ++ apps/mobile/package.json | 1 + apps/mobile/src/Stack.tsx | 6 + apps/mobile/src/components/AppSymbol.tsx | 1 + .../components/ComposerAttachmentStrip.tsx | 138 +- .../src/components/FullscreenImageViewer.tsx | 115 ++ .../features/annotations/ImageMarkupModal.tsx | 1222 +++++++++++++++++ .../annotations/attachmentMarkup.test.ts | 244 ++++ .../features/annotations/attachmentMarkup.ts | 135 ++ .../src/features/annotations/model.test.ts | 203 +++ apps/mobile/src/features/annotations/model.ts | 528 +++++++ .../features/files/ThreadFilesRouteScreen.tsx | 3 +- .../files/WorkspaceFileImagePreview.tsx | 10 +- .../layout/AdaptiveWorkspaceLayout.tsx | 100 +- .../layout/workspace-inspector-pane.tsx | 15 +- .../features/preview/MobileLivePreview.tsx | 334 +++++ .../features/preview/ThreadPreviewPane.tsx | 1087 +++++++++++++++ .../preview/ThreadPreviewRouteScreen.tsx | 114 ++ .../features/preview/mobilePreviewCapture.ts | 78 ++ .../preview/mobilePreviewCaptureModel.test.ts | 110 ++ .../preview/mobilePreviewCaptureModel.ts | 76 + .../preview/mobilePreviewDomBridge.test.ts | 148 ++ .../preview/mobilePreviewDomBridge.ts | 267 ++++ .../preview/mobilePreviewGatewayModel.test.ts | 111 ++ .../preview/mobilePreviewGatewayModel.ts | 62 + .../preview/mobilePreviewSourceUrl.test.ts | 64 + .../preview/mobilePreviewSourceUrl.ts | 28 + .../preview/previewLiveTarget.test.ts | 158 +++ .../src/features/preview/previewLiveTarget.ts | 171 +++ .../features/preview/previewPaneModel.test.ts | 251 ++++ .../src/features/preview/previewPaneModel.ts | 146 ++ .../preview/previewReviewModel.test.ts | 196 +++ .../features/preview/previewReviewModel.ts | 275 ++++ .../review/ReviewCommentComposerSheet.tsx | 7 + .../features/threads/NewTaskDraftScreen.tsx | 54 +- .../src/features/threads/ThreadComposer.tsx | 11 +- .../features/threads/ThreadDetailScreen.tsx | 2 + .../src/features/threads/ThreadFeed.tsx | 135 +- .../features/threads/ThreadGitControls.tsx | 37 +- .../features/threads/ThreadRouteScreen.tsx | 76 +- .../threads/new-task-flow-provider.tsx | 2 +- .../projectThreadCreationValidation.ts | 3 +- .../threads/threadAuxiliaryRoute.test.ts | 66 + .../features/threads/threadAuxiliaryRoute.ts | 20 + .../threadUserMessagePresentation.test.ts | 107 ++ .../threads/threadUserMessagePresentation.ts | 33 + .../features/threads/use-project-actions.ts | 1 + apps/mobile/src/lib/composer-image-schema.ts | 93 +- .../src/lib/composerImageAnnotations.ts | 21 + apps/mobile/src/lib/composerImages.test.ts | 144 +- apps/mobile/src/lib/composerImages.ts | 29 + apps/mobile/src/lib/layout.test.ts | 80 ++ apps/mobile/src/lib/layout.ts | 69 +- .../src/lib/projectThreadStartTurn.test.ts | 142 ++ apps/mobile/src/lib/projectThreadStartTurn.ts | 46 +- apps/mobile/src/state/preview.ts | 5 + apps/mobile/src/state/thread-outbox-model.ts | 9 +- apps/mobile/src/state/thread-outbox.test.ts | 31 + .../src/state/use-composer-drafts.test.ts | 51 + apps/mobile/src/state/use-composer-drafts.ts | 43 +- .../mobile/src/state/use-pending-new-tasks.ts | 2 +- .../src/state/use-thread-composer-state.ts | 23 + .../src/state/use-thread-outbox-drain.ts | 10 +- apps/server/src/auth/RpcAuthorization.ts | 2 + .../src/mcp/PreviewAutomationBroker.test.ts | 102 ++ .../server/src/mcp/PreviewAutomationBroker.ts | 49 +- apps/server/src/persistence/Migrations.ts | 4 + .../036_RepairAuthAuthorizationScopes.test.ts | 85 ++ .../036_RepairAuthAuthorizationScopes.ts | 25 + ...epairAuthPairingProofKeyThumbprint.test.ts | 36 + ...037_RepairAuthPairingProofKeyThumbprint.ts | 3 + apps/server/src/preview/LiveGateway.test.ts | 235 ++++ apps/server/src/preview/LiveGateway.ts | 376 +++++ .../src/preview/LiveGatewayHttp.test.ts | 266 ++++ apps/server/src/preview/LiveGatewayHttp.ts | 627 +++++++++ .../server/src/preview/ReviewSnapshot.test.ts | 206 +++ apps/server/src/preview/ReviewSnapshot.ts | 222 +++ apps/server/src/server.test.ts | 438 ++++++ apps/server/src/server.ts | 61 +- apps/server/src/ws.ts | 230 +++- .../components/chat/MessagesTimeline.test.tsx | 62 +- .../src/components/chat/MessagesTimeline.tsx | 106 +- .../chat/userMessagePreviewAnnotation.test.ts | 67 + .../chat/userMessagePreviewAnnotation.ts | 41 + .../preview/PreviewAutomationHosts.tsx | 9 +- .../previewAutomationReviewSnapshot.test.ts | 138 ++ .../previewAutomationReviewSnapshot.ts | 29 + apps/web/src/lib/previewAnnotation.ts | 110 +- docs/README.md | 3 +- docs/internals/remote.md | 63 + docs/user/mobile.md | 103 ++ packages/client-runtime/README.md | 1 + packages/client-runtime/package.json | 4 + .../src/annotations/elementContext.ts | 140 ++ .../client-runtime/src/annotations/index.ts | 11 + .../src/annotations/previewAnnotation.test.ts | 353 +++++ .../src/annotations/previewAnnotation.ts | 368 +++++ .../client-runtime/src/state/preview.test.ts | 47 +- packages/client-runtime/src/state/preview.ts | 36 + packages/contracts/src/index.ts | 1 + packages/contracts/src/ipc.ts | 208 +-- packages/contracts/src/preview.ts | 51 +- .../contracts/src/previewAnnotation.test.ts | 211 +++ packages/contracts/src/previewAnnotation.ts | 380 +++++ packages/contracts/src/previewAutomation.ts | 140 +- .../contracts/src/previewLiveGateway.test.ts | 71 + packages/contracts/src/previewReview.test.ts | 124 ++ packages/contracts/src/rpc.ts | 33 + pnpm-lock.yaml | 52 + 115 files changed, 13951 insertions(+), 591 deletions(-) create mode 100644 apps/mobile/src/components/FullscreenImageViewer.tsx create mode 100644 apps/mobile/src/features/annotations/ImageMarkupModal.tsx create mode 100644 apps/mobile/src/features/annotations/attachmentMarkup.test.ts create mode 100644 apps/mobile/src/features/annotations/attachmentMarkup.ts create mode 100644 apps/mobile/src/features/annotations/model.test.ts create mode 100644 apps/mobile/src/features/annotations/model.ts create mode 100644 apps/mobile/src/features/preview/MobileLivePreview.tsx create mode 100644 apps/mobile/src/features/preview/ThreadPreviewPane.tsx create mode 100644 apps/mobile/src/features/preview/ThreadPreviewRouteScreen.tsx create mode 100644 apps/mobile/src/features/preview/mobilePreviewCapture.ts create mode 100644 apps/mobile/src/features/preview/mobilePreviewCaptureModel.test.ts create mode 100644 apps/mobile/src/features/preview/mobilePreviewCaptureModel.ts create mode 100644 apps/mobile/src/features/preview/mobilePreviewDomBridge.test.ts create mode 100644 apps/mobile/src/features/preview/mobilePreviewDomBridge.ts create mode 100644 apps/mobile/src/features/preview/mobilePreviewGatewayModel.test.ts create mode 100644 apps/mobile/src/features/preview/mobilePreviewGatewayModel.ts create mode 100644 apps/mobile/src/features/preview/mobilePreviewSourceUrl.test.ts create mode 100644 apps/mobile/src/features/preview/mobilePreviewSourceUrl.ts create mode 100644 apps/mobile/src/features/preview/previewLiveTarget.test.ts create mode 100644 apps/mobile/src/features/preview/previewLiveTarget.ts create mode 100644 apps/mobile/src/features/preview/previewPaneModel.test.ts create mode 100644 apps/mobile/src/features/preview/previewPaneModel.ts create mode 100644 apps/mobile/src/features/preview/previewReviewModel.test.ts create mode 100644 apps/mobile/src/features/preview/previewReviewModel.ts create mode 100644 apps/mobile/src/features/threads/threadAuxiliaryRoute.test.ts create mode 100644 apps/mobile/src/features/threads/threadAuxiliaryRoute.ts create mode 100644 apps/mobile/src/features/threads/threadUserMessagePresentation.test.ts create mode 100644 apps/mobile/src/features/threads/threadUserMessagePresentation.ts create mode 100644 apps/mobile/src/lib/composerImageAnnotations.ts create mode 100644 apps/mobile/src/lib/projectThreadStartTurn.test.ts create mode 100644 apps/mobile/src/state/preview.ts create mode 100644 apps/server/src/persistence/Migrations/036_RepairAuthAuthorizationScopes.test.ts create mode 100644 apps/server/src/persistence/Migrations/036_RepairAuthAuthorizationScopes.ts create mode 100644 apps/server/src/persistence/Migrations/037_RepairAuthPairingProofKeyThumbprint.test.ts create mode 100644 apps/server/src/persistence/Migrations/037_RepairAuthPairingProofKeyThumbprint.ts create mode 100644 apps/server/src/preview/LiveGateway.test.ts create mode 100644 apps/server/src/preview/LiveGateway.ts create mode 100644 apps/server/src/preview/LiveGatewayHttp.test.ts create mode 100644 apps/server/src/preview/LiveGatewayHttp.ts create mode 100644 apps/server/src/preview/ReviewSnapshot.test.ts create mode 100644 apps/server/src/preview/ReviewSnapshot.ts create mode 100644 apps/web/src/components/chat/userMessagePreviewAnnotation.test.ts create mode 100644 apps/web/src/components/chat/userMessagePreviewAnnotation.ts create mode 100644 apps/web/src/components/preview/previewAutomationReviewSnapshot.test.ts create mode 100644 apps/web/src/components/preview/previewAutomationReviewSnapshot.ts create mode 100644 docs/user/mobile.md create mode 100644 packages/client-runtime/src/annotations/elementContext.ts create mode 100644 packages/client-runtime/src/annotations/index.ts create mode 100644 packages/client-runtime/src/annotations/previewAnnotation.test.ts create mode 100644 packages/client-runtime/src/annotations/previewAnnotation.ts create mode 100644 packages/contracts/src/previewAnnotation.test.ts create mode 100644 packages/contracts/src/previewAnnotation.ts create mode 100644 packages/contracts/src/previewLiveGateway.test.ts create mode 100644 packages/contracts/src/previewReview.test.ts diff --git a/apps/desktop/src/ipc/methods/preview.test.ts b/apps/desktop/src/ipc/methods/preview.test.ts index 92336cc7362..9dade738b87 100644 --- a/apps/desktop/src/ipc/methods/preview.test.ts +++ b/apps/desktop/src/ipc/methods/preview.test.ts @@ -51,4 +51,41 @@ describe("preview IPC methods", () => { }, ), ); + + effectIt.effect("forwards review mode while legacy snapshot calls stay full", () => + Effect.gen(function* () { + const automationSnapshot = vi.fn(() => + Effect.succeed({ + url: "https://example.com", + title: "Example", + loading: false, + visibleText: "", + interactiveElements: [], + accessibilityTree: null, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + screenshot: { + mimeType: "image/png" as const, + data: "cG5n", + width: 1, + height: 1, + }, + }), + ); + const manager = { automationSnapshot } as never; + + yield* PreviewIpc.automationSnapshot + .handler({ tabId: "tab-1", mode: "review" }) + .pipe(Effect.provideService(PreviewManager.PreviewManager, manager)); + yield* PreviewIpc.automationSnapshot + .handler({ tabId: "tab-1" }) + .pipe(Effect.provideService(PreviewManager.PreviewManager, manager)); + + expect(automationSnapshot.mock.calls).toEqual([ + ["tab-1", "review"], + ["tab-1", "full"], + ]); + }), + ); }); diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 4d50ad8d665..0297a1c5df9 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -5,6 +5,7 @@ import { DesktopPreviewAutomationEvaluateInputSchema, DesktopPreviewAutomationPressInputSchema, DesktopPreviewAutomationScrollInputSchema, + DesktopPreviewAutomationSnapshotInputSchema, DesktopPreviewAutomationTypeInputSchema, DesktopPreviewAutomationWaitForInputSchema, DesktopPreviewConfigInputSchema, @@ -276,11 +277,11 @@ export const automationStatus = DesktopIpc.makeIpcMethod({ export const automationSnapshot = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, - payload: DesktopPreviewTabInputSchema, + payload: DesktopPreviewAutomationSnapshotInputSchema, result: PreviewAutomationSnapshot, - handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId }) { + handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId, mode }) { const manager = yield* PreviewManager.PreviewManager; - return yield* manager.automationSnapshot(tabId); + return yield* manager.automationSnapshot(tabId, mode === "review" ? "review" : "full"); }), }); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 9f01baeed90..95aeed98d0e 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -209,8 +209,11 @@ contextBridge.exposeInMainWorld("desktopBridge", { automation: { status: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, { tabId }), - snapshot: (tabId) => - ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { tabId }), + snapshot: (tabId, options) => + ipcRenderer.invoke( + IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, + options?.mode === "review" ? { tabId, mode: "review" } : { tabId }, + ), click: (tabId, input) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, { tabId, input }), type: (tabId, input) => diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 684d6655da5..0c69d4714f1 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -36,6 +36,122 @@ describe("fitPictureInPictureContentSize", () => { }); }); +describe("compactPreviewAutomationElements", () => { + it("clips semantics to the viewport and drops invalid or oversized entries", () => { + const oversizedSelector = `#${"x".repeat(2_048)}`; + const result = PreviewManager.compactPreviewAutomationElements( + [ + { + tag: "BUTTON", + role: " button ", + name: ` Save\n${"now ".repeat(200)}`, + selector: "#save", + x: -10, + y: 20, + width: 30, + height: 40, + }, + { + tag: "a", + role: null, + name: "Outside", + selector: "#outside", + x: 900, + y: 20, + width: 30, + height: 40, + }, + { + tag: "input", + role: null, + name: "Invalid geometry", + selector: "#invalid", + x: 20, + y: 20, + width: Number.NaN, + height: 40, + }, + { + tag: "div", + role: "button", + name: "Oversized selector", + selector: oversizedSelector, + x: 20, + y: 20, + width: 40, + height: 40, + }, + ], + { width: 800, height: 600 }, + ); + + expect(result).toEqual([ + { + tag: "button", + role: "button", + name: expect.stringMatching(/^Save now /), + selector: "#save", + x: 0, + y: 20, + width: 20, + height: 40, + }, + ]); + expect(result[0]?.name.length).toBe(512); + }); + + it("caps the number of semantic elements retained", () => { + const result = PreviewManager.compactPreviewAutomationElements( + Array.from({ length: 250 }, (_, index) => ({ + tag: "button", + role: null, + name: `Button ${index}`, + selector: `#button-${index}`, + x: 0, + y: 0, + width: 10, + height: 10, + })), + { width: 800, height: 600 }, + ); + + expect(result).toHaveLength(200); + expect(result.at(-1)?.selector).toBe("#button-199"); + }); +}); + +describe("previewAutomationSnapshotResizeWidth", () => { + it("keeps the legacy snapshot path width-only", () => { + expect( + PreviewManager.previewAutomationSnapshotResizeWidth({ width: 1_000, height: 3_000 }, "full"), + ).toBeNull(); + expect( + PreviewManager.previewAutomationSnapshotResizeWidth({ width: 2_400, height: 1_200 }, "full"), + ).toBe(1_280); + }); + + it("caps review snapshots by width and approximately two million pixels", () => { + expect( + PreviewManager.previewAutomationSnapshotResizeWidth( + { width: 1_000, height: 3_000 }, + "review", + ), + ).toBe(816); + expect( + PreviewManager.previewAutomationSnapshotResizeWidth( + { width: 2_400, height: 1_200 }, + "review", + ), + ).toBe(1_280); + expect( + PreviewManager.previewAutomationSnapshotResizeWidth( + { width: 1_000, height: 1_000 }, + "review", + ), + ).toBeNull(); + }); +}); + const { browserWindowConstructor, createFromPath, @@ -766,6 +882,235 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("keeps legacy automation snapshots full", () => + withManager((manager) => + Effect.gen(function* () { + const png = Buffer.from("review-snapshot-png"); + const resizedImage = { + getSize: () => ({ width: 1_280, height: 768 }), + toPNG: () => png, + }; + const resize = vi.fn(() => resizedImage); + const capturePage = vi.fn(async () => ({ + getSize: () => ({ width: 1_600, height: 960 }), + resize, + toPNG: () => Buffer.from("unscaled"), + })); + const accessibilityTree = { + nodes: [{ nodeId: "1", role: { value: "button" } }], + }; + const sendCommand = vi.fn(async (method: string) => { + if (method === "Runtime.evaluate") { + return { + result: { + value: { + url: "https://example.com/review", + title: "Review", + loading: false, + visibleText: "Save settings", + viewport: { + width: 1_000, + height: 600, + scrollX: 12, + scrollY: 240, + devicePixelRatio: 2, + }, + interactiveElements: [ + { + tag: "button", + role: "button", + name: "Save", + selector: "#save", + x: 990, + y: 20, + width: 40, + height: 30, + }, + { + tag: "button", + role: "button", + name: "Below the frame", + selector: "#below", + x: 20, + y: 700, + width: 40, + height: 30, + }, + ], + }, + }, + }; + } + if (method === "Accessibility.getFullAXTree") return accessibilityTree; + return undefined; + }); + fromId.mockReturnValue({ + id: 42, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com/review", + getTitle: () => "Review", + isLoading: () => false, + isDevToolsOpened: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + capturePage, + } as never); + + yield* manager.createTab("tab_review"); + yield* manager.registerWebview("tab_review", 42); + const snapshot = yield* manager.automationSnapshot("tab_review"); + + expect(snapshot.viewport).toEqual({ + width: 1_000, + height: 600, + scrollX: 12, + scrollY: 240, + devicePixelRatio: 2, + }); + expect(snapshot.interactiveElements).toEqual([ + { + tag: "button", + role: "button", + name: "Save", + selector: "#save", + x: 990, + y: 20, + width: 10, + height: 30, + }, + ]); + expect(snapshot.visibleText).toBe("Save settings"); + expect(snapshot.accessibilityTree).toEqual(accessibilityTree); + expect(snapshot.screenshot).toEqual({ + mimeType: "image/png", + data: png.toString("base64"), + width: 1_280, + height: 768, + scale: 1.28, + }); + expect(resize).toHaveBeenCalledWith({ width: 1_280 }); + expect(capturePage).toHaveBeenCalledOnce(); + }), + ), + ); + + effectIt.effect("captures compact review snapshots without diagnostic context", () => + withManager((manager) => + Effect.gen(function* () { + const png = Buffer.from("compact-review-snapshot-png"); + const resizedImage = { + getSize: () => ({ width: 816, height: 2_448 }), + toPNG: () => png, + }; + const resize = vi.fn(() => resizedImage); + const capturePage = vi.fn(async () => ({ + getSize: () => ({ width: 1_000, height: 3_000 }), + resize, + toPNG: () => Buffer.from("unscaled"), + })); + const sendCommand = vi.fn( + async (method: string, _commandParams?: Record) => { + if (method !== "Runtime.evaluate") return undefined; + return { + result: { + value: { + url: "https://example.com/review", + title: "Review", + loading: false, + visibleText: "Sensitive page text", + viewport: { + width: 1_000, + height: 3_000, + scrollX: 0, + scrollY: 120, + devicePixelRatio: 2, + }, + interactiveElements: [ + { + tag: "button", + role: "button", + name: "Save", + selector: "#save", + x: 20, + y: 30, + width: 80, + height: 32, + }, + ], + }, + }, + }; + }, + ); + fromId.mockReturnValue({ + id: 42, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com/review", + getTitle: () => "Review", + isLoading: () => false, + isDevToolsOpened: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + capturePage, + } as never); + + yield* manager.createTab("tab_review_compact"); + yield* manager.registerWebview("tab_review_compact", 42); + const snapshot = yield* manager.automationSnapshot("tab_review_compact", "review"); + + expect(snapshot.visibleText).toBe(""); + expect(snapshot.accessibilityTree).toBeNull(); + expect(snapshot.consoleEntries).toEqual([]); + expect(snapshot.networkEntries).toEqual([]); + expect(snapshot.actionTimeline).toEqual([]); + expect(snapshot.interactiveElements).toHaveLength(1); + expect(snapshot.screenshot).toEqual({ + mimeType: "image/png", + data: png.toString("base64"), + width: 816, + height: 2_448, + scale: 0.816, + }); + expect(sendCommand).not.toHaveBeenCalledWith("Accessibility.getFullAXTree"); + const evaluateExpression = sendCommand.mock.calls.find( + ([method]) => method === "Runtime.evaluate", + )?.[1]?.expression; + expect(evaluateExpression).toEqual(expect.any(String)); + expect(evaluateExpression).not.toContain("document.body?.innerText"); + expect(resize).toHaveBeenCalledWith({ width: 816 }); + expect(capturePage).toHaveBeenCalledOnce(); + }), + ), + ); + effectIt.effect("captures hidden preview recordings independently for concurrent tabs", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index d7e8376cec7..8ea0fba7599 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -22,6 +22,7 @@ import type { PreviewAutomationNetworkEntry, PreviewAutomationScrollInput, PreviewAutomationSnapshot, + PreviewAutomationSnapshotViewport, PreviewAutomationStatus, PreviewAutomationTypeInput, PreviewAutomationWaitForInput, @@ -97,7 +98,12 @@ const ZOOM_EPSILON = 0.001; const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; const MAX_INTERACTIVE_ELEMENTS = 200; +const MAX_INTERACTIVE_ELEMENT_NAME_LENGTH = 512; +const MAX_INTERACTIVE_ELEMENT_ROLE_LENGTH = 128; +const MAX_INTERACTIVE_ELEMENT_SELECTOR_LENGTH = 2_048; +const MAX_INTERACTIVE_ELEMENT_TAG_LENGTH = 64; const MAX_SCREENSHOT_WIDTH = 1280; +const MAX_REVIEW_SCREENSHOT_PIXELS = 2_000_000; const RECORDING_FRAME_INTERVAL_MS = Math.ceil(1_000 / 12); const RECORDING_JPEG_QUALITY = 80; const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480; @@ -204,6 +210,99 @@ interface CdpEvaluationResult { }; } +type PreviewAutomationSnapshotElement = PreviewAutomationSnapshot["interactiveElements"][number]; +export type PreviewAutomationSnapshotMode = "full" | "review"; + +const compactSemanticText = (value: string, maximumLength: number): string => + value.replace(/\s+/g, " ").trim().slice(0, maximumLength); + +export function previewAutomationSnapshotResizeWidth( + size: Readonly<{ width: number; height: number }>, + mode: PreviewAutomationSnapshotMode, +): number | null { + if ( + !Number.isFinite(size.width) || + size.width <= 0 || + !Number.isFinite(size.height) || + size.height <= 0 + ) { + return null; + } + + const widthScale = MAX_SCREENSHOT_WIDTH / size.width; + const pixelScale = + mode === "review" ? Math.sqrt(MAX_REVIEW_SCREENSHOT_PIXELS / (size.width * size.height)) : 1; + const scale = Math.min(1, widthScale, pixelScale); + return scale < 1 ? Math.max(1, Math.floor(size.width * scale)) : null; +} + +/** + * Keeps snapshot semantics in the same coordinate space as the captured + * viewport. Page scripts are not a trust boundary, so apply the clipping and + * payload limits again in the main process even though the injected collector + * performs the same viewport filtering before crossing CDP. + */ +export function compactPreviewAutomationElements( + elements: ReadonlyArray, + viewport: Pick, +): ReadonlyArray { + if ( + !Number.isFinite(viewport.width) || + viewport.width <= 0 || + !Number.isFinite(viewport.height) || + viewport.height <= 0 + ) { + return []; + } + + const compacted: PreviewAutomationSnapshotElement[] = []; + for (const element of elements) { + if (compacted.length >= MAX_INTERACTIVE_ELEMENTS) break; + if ( + !Number.isFinite(element.x) || + !Number.isFinite(element.y) || + !Number.isFinite(element.width) || + !Number.isFinite(element.height) || + element.width <= 0 || + element.height <= 0 + ) { + continue; + } + + const left = Math.max(0, element.x); + const top = Math.max(0, element.y); + const right = Math.min(viewport.width, element.x + element.width); + const bottom = Math.min(viewport.height, element.y + element.height); + if (right <= left || bottom <= top) continue; + + const selector = element.selector.trim(); + const tag = compactSemanticText(element.tag, MAX_INTERACTIVE_ELEMENT_TAG_LENGTH).toLowerCase(); + if ( + selector.length === 0 || + selector.length > MAX_INTERACTIVE_ELEMENT_SELECTOR_LENGTH || + tag.length === 0 + ) { + continue; + } + + const role = + element.role === null + ? null + : compactSemanticText(element.role, MAX_INTERACTIVE_ELEMENT_ROLE_LENGTH) || null; + compacted.push({ + tag, + role, + name: compactSemanticText(element.name, MAX_INTERACTIVE_ELEMENT_NAME_LENGTH), + selector, + x: left, + y: top, + width: right - left, + height: bottom - top, + }); + } + return compacted; +} + export const PreviewAutomationSelectorKind = Schema.Literals([ "focused-element", "selector", @@ -2601,16 +2700,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); const captureAutomationSnapshot = Effect.fn("PreviewManager.captureAutomationSnapshot")( - function* (tabId: string, wc: Electron.WebContents, send: SendCommand) { - yield* Effect.all([send("Runtime.enable"), send("Accessibility.enable")], { - concurrency: 2, - discard: true, - }); + function* ( + tabId: string, + wc: Electron.WebContents, + send: SendCommand, + mode: PreviewAutomationSnapshotMode, + ) { + yield* Effect.all( + mode === "review" + ? [send("Runtime.enable")] + : [send("Runtime.enable"), send("Accessibility.enable")], + { + concurrency: 2, + discard: true, + }, + ); const page = yield* evaluateWithDebugger<{ url: string; title: string; loading: boolean; visibleText: string; + viewport: PreviewAutomationSnapshotViewport; interactiveElements: PreviewAutomationSnapshot["interactiveElements"]; }>( tabId, @@ -2641,64 +2751,102 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const visible = (element) => { const style = getComputedStyle(element); const rect = element.getBoundingClientRect(); - return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0; + return style.visibility !== "hidden" + && style.display !== "none" + && style.opacity !== "0" + && rect.width > 0 + && rect.height > 0 + && rect.right > 0 + && rect.bottom > 0 + && rect.left < window.innerWidth + && rect.top < window.innerHeight; }; const elements = Array.from(document.querySelectorAll( "a[href],button,input,textarea,select,[role],[tabindex]" - )).filter(visible).slice(0, ${MAX_INTERACTIVE_ELEMENTS}).map((element) => { + )).filter(visible).map((element) => { const rect = element.getBoundingClientRect(); + const selector = selectorFor(element); + if (!selector || selector.length > ${MAX_INTERACTIVE_ELEMENT_SELECTOR_LENGTH}) { + return null; + } + const rawName = element.getAttribute("aria-label") + || element.innerText + || element.getAttribute("name") + || ""; return { tag: element.tagName.toLowerCase(), role: element.getAttribute("role"), - name: element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "", - selector: selectorFor(element), + name: String(rawName).replace(/\\s+/g, " ").trim().slice(0, ${MAX_INTERACTIVE_ELEMENT_NAME_LENGTH}), + selector, x: rect.x, y: rect.y, width: rect.width, height: rect.height }; - }); + }).filter((element) => element !== null).slice(0, ${MAX_INTERACTIVE_ELEMENTS}); return { url: location.href, title: document.title, loading: document.readyState !== "complete", - visibleText: (document.body?.innerText || "").slice(0, ${MAX_VISIBLE_TEXT_LENGTH}), + visibleText: ${ + mode === "review" + ? '""' + : `(document.body?.innerText || "").slice(0, ${MAX_VISIBLE_TEXT_LENGTH})` + }, + viewport: { + width: Math.max(1, window.innerWidth), + height: Math.max(1, window.innerHeight), + scrollX: Number.isFinite(window.scrollX) ? window.scrollX : 0, + scrollY: Number.isFinite(window.scrollY) ? window.scrollY : 0, + devicePixelRatio: Number.isFinite(window.devicePixelRatio) && window.devicePixelRatio > 0 + ? window.devicePixelRatio + : 1 + }, interactiveElements: elements }; })()`, true, ); - const [accessibility, sourceImage, diagnostics, timelines] = yield* Effect.all([ - send("Accessibility.getFullAXTree"), - attemptPromise( - { - operation: "automationSnapshot.capturePage", - tabId, - webContentsId: wc.id, - }, - () => wc.capturePage(), - ), - Ref.get(diagnosticsRef), - Ref.get(actionTimelineRef), - ]); + const capturePage = attemptPromise( + { + operation: "automationSnapshot.capturePage", + tabId, + webContentsId: wc.id, + }, + () => wc.capturePage(), + ); + const [accessibility, sourceImage, diagnostics, timelines] = + mode === "review" + ? ([null, yield* capturePage, null, null] as const) + : yield* Effect.all([ + send("Accessibility.getFullAXTree"), + capturePage, + Ref.get(diagnosticsRef), + Ref.get(actionTimelineRef), + ]); const sourceSize = sourceImage.getSize(); - const image = - sourceSize.width > MAX_SCREENSHOT_WIDTH - ? sourceImage.resize({ width: MAX_SCREENSHOT_WIDTH }) - : sourceImage; + const resizeWidth = previewAutomationSnapshotResizeWidth(sourceSize, mode); + const image = resizeWidth === null ? sourceImage : sourceImage.resize({ width: resizeWidth }); const size = image.getSize(); - const browserDiagnostics = diagnostics.get(wc.id); + const browserDiagnostics = diagnostics?.get(wc.id); + const interactiveElements = compactPreviewAutomationElements( + page.interactiveElements, + page.viewport, + ); return { ...page, + visibleText: mode === "review" ? "" : page.visibleText, + interactiveElements, accessibilityTree: accessibility, consoleEntries: [...(browserDiagnostics?.consoleEntries ?? [])], networkEntries: [...(browserDiagnostics?.networkEntries ?? [])], - actionTimeline: [...(timelines.get(tabId) ?? [])], + actionTimeline: [...(timelines?.get(tabId) ?? [])], screenshot: { mimeType: "image/png" as const, data: image.toPNG().toString("base64"), width: size.width, height: size.height, + scale: size.width / page.viewport.width, }, }; }, @@ -2706,10 +2854,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const automationSnapshot = Effect.fn("PreviewManager.automationSnapshot")(function* ( tabId: string, + mode: PreviewAutomationSnapshotMode = "full", ) { const wc = yield* requireWebContents(tabId); return yield* withControlSession(tabId, wc, "snapshot", (send) => - captureAutomationSnapshot(tabId, wc, send), + captureAutomationSnapshot(tabId, wc, send, mode), ); }); @@ -3607,6 +3756,7 @@ export class PreviewManager extends Context.Service< ) => Effect.Effect; readonly automationSnapshot: ( tabId: string, + mode?: PreviewAutomationSnapshotMode, ) => Effect.Effect; readonly automationClick: ( tabId: string, diff --git a/apps/desktop/src/preview/PickedElementPayload.test.ts b/apps/desktop/src/preview/PickedElementPayload.test.ts index d7a96732477..0fbf58d02fa 100644 --- a/apps/desktop/src/preview/PickedElementPayload.test.ts +++ b/apps/desktop/src/preview/PickedElementPayload.test.ts @@ -180,6 +180,69 @@ describe("isPreviewAnnotationPayload", () => { expect(isPreviewAnnotationPayload(validAnnotation())).toBe(true); }); + it("accepts valid portable source, callout, and editable-vector extensions", () => { + expect( + isPreviewAnnotationPayload( + validAnnotation({ + schemaVersion: 1, + source: { + kind: "preview", + url: "https://example.com/", + title: "Example", + }, + callouts: [ + { + id: "callout_point", + number: 1, + comment: "Move this.", + anchor: { kind: "point", point: { x: 0.2, y: 0.3 } }, + }, + { + id: "callout_region", + number: 2, + comment: "Reduce this space.", + anchor: { + kind: "region", + rect: { x: 0.4, y: 0.1, width: 0.3, height: 0.2 }, + }, + }, + { + id: "callout_element", + number: 3, + comment: "Use the primary style.", + anchor: { + kind: "element", + targetId: "element_1", + rect: { x: 0.1, y: 0.7, width: 0.4, height: 0.2 }, + }, + }, + ], + editable: { + version: 1, + coordinateSpace: "normalized", + strokes: [ + { + id: "stroke_editable", + color: "#7c3aed", + width: 0.01, + points: [ + { x: 0.1, y: 0.2 }, + { x: 0.2, y: 0.3 }, + ], + bounds: { x: 0.08, y: 0.18, width: 0.14, height: 0.14 }, + }, + ], + }, + }), + ), + ).toBe(true); + expect( + isPreviewAnnotationPayload( + validAnnotation({ source: { kind: "image", name: "checkout.png" }, editable: null }), + ), + ).toBe(true); + }); + it("rejects screenshots supplied by the guest preload", () => { expect(isPreviewAnnotationPayload(validAnnotation({ screenshot: { dataUrl: "bad" } }))).toBe( false, @@ -198,4 +261,86 @@ describe("isPreviewAnnotationPayload", () => { ), ).toBe(false); }); + + it.each<[string, Record]>([ + ["an unknown schema version", validAnnotation({ schemaVersion: 2 })], + ["a malformed image source", validAnnotation({ source: { kind: "image", name: 42 } })], + [ + "a malformed preview source", + validAnnotation({ source: { kind: "preview", url: null, title: "Example" } }), + ], + ["a non-array callout collection", validAnnotation({ callouts: "not-an-array" })], + [ + "an out-of-bounds callout point", + validAnnotation({ + callouts: [ + { + id: "callout_1", + number: 1, + comment: "", + anchor: { kind: "point", point: { x: 1.1, y: 0.5 } }, + }, + ], + }), + ], + [ + "an overflowing callout region", + validAnnotation({ + callouts: [ + { + id: "callout_1", + number: 1, + comment: "", + anchor: { + kind: "region", + rect: { x: 0.8, y: 0.2, width: 0.3, height: 0.2 }, + }, + }, + ], + }), + ], + [ + "a non-positive callout number", + validAnnotation({ + callouts: [ + { + id: "callout_1", + number: 0, + comment: "", + anchor: { kind: "point", point: { x: 0.5, y: 0.5 } }, + }, + ], + }), + ], + [ + "a malformed editable vector document", + validAnnotation({ + editable: { + version: 2, + coordinateSpace: "pixels", + strokes: [], + }, + }), + ], + [ + "an out-of-bounds editable stroke", + validAnnotation({ + editable: { + version: 1, + coordinateSpace: "normalized", + strokes: [ + { + id: "stroke_1", + color: "#7c3aed", + width: 0.01, + points: [{ x: Number.NaN, y: 0.5 }], + bounds: { x: 0.1, y: 0.1, width: 0.2, height: 0.2 }, + }, + ], + }, + }), + ], + ])("rejects %s", (_label, value) => { + expect(isPreviewAnnotationPayload(value)).toBe(false); + }); }); diff --git a/apps/desktop/src/preview/PickedElementPayload.ts b/apps/desktop/src/preview/PickedElementPayload.ts index e2d596120db..e9ce956cb15 100644 --- a/apps/desktop/src/preview/PickedElementPayload.ts +++ b/apps/desktop/src/preview/PickedElementPayload.ts @@ -67,6 +67,97 @@ function isPoint(value: unknown): boolean { ); } +function isNormalizedNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1; +} + +function isNormalizedPoint(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false; + const point = value as Record; + return isNormalizedNumber(point["x"]) && isNormalizedNumber(point["y"]); +} + +function isNormalizedRect(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false; + const rect = value as Record; + if ( + !isNormalizedNumber(rect["x"]) || + !isNormalizedNumber(rect["y"]) || + !isNormalizedNumber(rect["width"]) || + !isNormalizedNumber(rect["height"]) || + rect["width"] <= 0 || + rect["height"] <= 0 + ) { + return false; + } + return rect["x"] + rect["width"] <= 1 && rect["y"] + rect["height"] <= 1; +} + +function isPreviewAnnotationSource(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false; + const source = value as Record; + if (source["kind"] === "image") { + return isStringOrNull(source["name"]); + } + if (source["kind"] === "preview") { + return typeof source["url"] === "string" && isStringOrNull(source["title"]); + } + return false; +} + +function isPreviewAnnotationCallout(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false; + const callout = value as Record; + if ( + typeof callout["id"] !== "string" || + typeof callout["number"] !== "number" || + !Number.isInteger(callout["number"]) || + callout["number"] < 1 || + typeof callout["comment"] !== "string" + ) { + return false; + } + + const anchorValue = callout["anchor"]; + if (typeof anchorValue !== "object" || anchorValue === null) return false; + const anchor = anchorValue as Record; + if (anchor["kind"] === "point") { + return isNormalizedPoint(anchor["point"]); + } + if (anchor["kind"] === "region") { + return isNormalizedRect(anchor["rect"]); + } + if (anchor["kind"] === "element") { + return typeof anchor["targetId"] === "string" && isNormalizedRect(anchor["rect"]); + } + return false; +} + +function isPreviewAnnotationEditableStroke(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false; + const stroke = value as Record; + return ( + typeof stroke["id"] === "string" && + typeof stroke["color"] === "string" && + isNormalizedNumber(stroke["width"]) && + stroke["width"] > 0 && + Array.isArray(stroke["points"]) && + stroke["points"].every(isNormalizedPoint) && + isNormalizedRect(stroke["bounds"]) + ); +} + +function isPreviewAnnotationEditable(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false; + const editable = value as Record; + return ( + editable["version"] === 1 && + editable["coordinateSpace"] === "normalized" && + Array.isArray(editable["strokes"]) && + editable["strokes"].every(isPreviewAnnotationEditableStroke) + ); +} + export function isPreviewAnnotationPayload(value: unknown): value is PreviewAnnotationPayload { if (typeof value !== "object" || value === null) return false; const annotation = value as Record; @@ -75,7 +166,27 @@ export function isPreviewAnnotationPayload(value: unknown): value is PreviewAnno if (!isStringOrNull(annotation["pageTitle"])) return false; if (typeof annotation["comment"] !== "string") return false; if (typeof annotation["createdAt"] !== "string") return false; + // The guest submits structure only. The trusted main process captures and + // attaches the screenshot after this boundary. if (annotation["screenshot"] !== null) return false; + if (annotation["schemaVersion"] !== undefined && annotation["schemaVersion"] !== 1) return false; + if (annotation["source"] !== undefined && !isPreviewAnnotationSource(annotation["source"])) { + return false; + } + if ( + annotation["callouts"] !== undefined && + (!Array.isArray(annotation["callouts"]) || + !annotation["callouts"].every(isPreviewAnnotationCallout)) + ) { + return false; + } + if ( + annotation["editable"] !== undefined && + annotation["editable"] !== null && + !isPreviewAnnotationEditable(annotation["editable"]) + ) { + return false; + } const elements = annotation["elements"]; if (!Array.isArray(elements)) return false; diff --git a/apps/mobile/package.json b/apps/mobile/package.json index c12ca979bf2..348be17ef21 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -114,6 +114,7 @@ "react-native-screens": "4.25.2", "react-native-shiki-engine": "^0.3.12", "react-native-svg": "15.15.4", + "react-native-view-shot": "5.1.0", "react-native-webview": "^13.16.1", "react-native-worklets": "0.8.3", "shiki": "4.2.0", diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index 719a6a4ad59..374309fc641 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -26,6 +26,7 @@ import { AdaptiveWorkspaceLayout } from "./features/layout/AdaptiveWorkspaceLayo import { HardwareKeyboardCommandProvider } from "./features/keyboard/HardwareKeyboardCommandProvider"; import { ReviewCommentComposerSheet } from "./features/review/ReviewCommentComposerSheet"; import { ReviewSheet } from "./features/review/ReviewSheet"; +import { ThreadPreviewRouteScreen } from "./features/preview/ThreadPreviewRouteScreen"; import { ThreadTerminalRouteScreen } from "./features/terminal/ThreadTerminalRouteScreen"; import { GitBranchesSheet } from "./features/threads/git/GitBranchesSheet"; import { GitCommitSheet } from "./features/threads/git/GitCommitSheet"; @@ -422,6 +423,11 @@ export const RootStack = createNativeStackNavigator({ sheetGrabberVisible: Platform.OS !== "android", }, }), + ThreadPreview: createNativeStackScreen({ + screen: ThreadPreviewRouteScreen, + linking: `${THREAD_LINKING_PREFIX}/preview`, + options: SOLID_HEADER_OPTIONS, + }), ThreadFiles: createNativeStackScreen({ screen: ThreadFilesTreeScreen, linking: `${THREAD_LINKING_PREFIX}/files`, diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index d4c7088fe50..4ffadc5a565 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -120,6 +120,7 @@ const ANDROID_ICON_BY_SF_SYMBOL: Partial> = { "line.3.horizontal.decrease.circle.fill": IconFilter, magnifyingglass: IconSearch, paintbrush: IconPalette, + pencil: IconEdit, "person.crop.circle": IconUserCircle, play: IconPlayerPlay, plus: IconPlus, diff --git a/apps/mobile/src/components/ComposerAttachmentStrip.tsx b/apps/mobile/src/components/ComposerAttachmentStrip.tsx index 0621285c03e..4f7b08e08d5 100644 --- a/apps/mobile/src/components/ComposerAttachmentStrip.tsx +++ b/apps/mobile/src/components/ComposerAttachmentStrip.tsx @@ -1,7 +1,9 @@ import { SymbolView } from "../components/AppSymbol"; +import { useState } from "react"; import { Image, Pressable, ScrollView, View } from "react-native"; import { useThemeColor } from "../lib/useThemeColor"; +import { ImageMarkupModal } from "../features/annotations/ImageMarkupModal"; import type { DraftComposerImageAttachment } from "../lib/composerImages"; export interface ComposerAttachmentStripProps { @@ -9,6 +11,8 @@ export interface ComposerAttachmentStripProps { readonly attachments: ReadonlyArray; /** Called when the user taps the remove button on an image. */ readonly onRemove: (imageId: string) => void; + /** Replaces an attachment after adding, editing, or removing image markup. */ + readonly onReplace?: (image: DraftComposerImageAttachment) => void; /** Called when the user taps on an image thumbnail to preview it. */ readonly onPressImage?: (previewUri: string) => void; /** Image thumbnail size in points. Defaults to 72. */ @@ -25,6 +29,9 @@ export interface ComposerAttachmentStripProps { */ export function ComposerAttachmentStrip(props: ComposerAttachmentStripProps) { const subtleBg = useThemeColor("--color-subtle"); + const [markupAttachment, setMarkupAttachment] = useState( + null, + ); const size = props.imageSize ?? 72; const radius = props.imageBorderRadius ?? 16; const removeButtonPlacement = props.removeButtonPlacement ?? "overlay"; @@ -35,56 +42,91 @@ export function ComposerAttachmentStrip(props: ComposerAttachmentStripProps) { } return ( - - - {props.attachments.map((image) => ( - - props.onPressImage!(image.previewUri) : undefined} - > - - - + + + {props.attachments.map((image) => ( + props.onRemove(image.id)} > - - - - ))} - - + props.onPressImage!(image.previewUri) : undefined + } + > + + + props.onRemove(image.id)} + > + + + {props.onReplace ? ( + setMarkupAttachment(image)} + > + + + ) : null} + + ))} + + + setMarkupAttachment(null)} + onDone={(image) => { + props.onReplace?.(image); + setMarkupAttachment(null); + }} + /> + ); } diff --git a/apps/mobile/src/components/FullscreenImageViewer.tsx b/apps/mobile/src/components/FullscreenImageViewer.tsx new file mode 100644 index 00000000000..ced5068fb72 --- /dev/null +++ b/apps/mobile/src/components/FullscreenImageViewer.tsx @@ -0,0 +1,115 @@ +import { + Image, + Modal, + Platform, + Pressable, + ScrollView, + useWindowDimensions, + View, + type ImageURISource, + type NativeScrollEvent, + type NativeSyntheticEvent, +} from "react-native"; +import ImageViewing from "react-native-image-viewing"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { SymbolView } from "./AppSymbol"; + +export type FullscreenImageSource = ImageURISource | number; + +export function FullscreenImageViewer(props: { + readonly source: FullscreenImageSource | null; + readonly visible: boolean; + readonly onRequestClose: () => void; +}) { + const dimensions = useWindowDimensions(); + const insets = useSafeAreaInsets(); + + if (!props.visible || props.source === null) return null; + + if (Platform.OS !== "ios") { + return ( + + ); + } + + const handleScrollEndDrag = (event: NativeSyntheticEvent) => { + if ((event.nativeEvent.zoomScale ?? 1) > 1) return; + if (Math.abs(event.nativeEvent.velocity?.y ?? 0) > 1.55) { + props.onRequestClose(); + } + }; + + return ( + + + + + + + + + + + ); +} diff --git a/apps/mobile/src/features/annotations/ImageMarkupModal.tsx b/apps/mobile/src/features/annotations/ImageMarkupModal.tsx new file mode 100644 index 00000000000..9579377b146 --- /dev/null +++ b/apps/mobile/src/features/annotations/ImageMarkupModal.tsx @@ -0,0 +1,1222 @@ +import { + PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, + type PreviewAnnotationCallout, + type PreviewAnnotationElementTarget, + type PreviewAnnotationNormalizedPoint, + type PreviewAnnotationNormalizedRect, +} from "@t3tools/contracts"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { + ActivityIndicator, + Image, + Modal, + PixelRatio, + Pressable, + ScrollView, + TextInput, + View, + type LayoutChangeEvent, +} from "react-native"; +import { Gesture, GestureDetector } from "react-native-gesture-handler"; +import Animated, { + runOnJS, + useAnimatedProps, + useSharedValue, + type SharedValue, +} from "react-native-reanimated"; +import Svg, { Circle, G, Image as SvgImage, Path, Rect, Text as SvgText } from "react-native-svg"; +import { KeyboardAvoidingView } from "react-native-keyboard-controller"; +import { SafeAreaView } from "react-native-safe-area-context"; + +import { AppText as Text } from "../../components/AppText"; +import { SymbolView, type AppSymbolName } from "../../components/AppSymbol"; +import { estimateBase64ByteSize } from "../../lib/base64"; +import { + restoreComposerImageOriginal, + type DraftComposerImageAttachment, +} from "../../lib/composerImages"; +import { useThemeColor } from "../../lib/useThemeColor"; +import { uuidv4 } from "../../lib/uuid"; +import { + buildAnnotatedImageAttachment, + markupDocumentFromAttachment, + originalImageFromAttachment, +} from "./attachmentMarkup"; +import { + DEFAULT_NORMALIZED_STROKE_WIDTH, + DEFAULT_STROKE_COLOR, + EMPTY_MARKUP_DOCUMENT, + addElementCallout, + addPointCallout, + addRegionCallout, + addStroke, + annotationExportLayoutSize, + annotationExportSize, + aspectFit, + clearMarkupDocument, + commitMarkupDocument, + createMarkupHistory, + deleteMarkupObject, + eraseMarkupObjectAtPoint, + hitTestMarkupObject, + makeStroke, + markupDocumentIsEmpty, + nextSmallerExportSize, + normalizedPoint, + normalizedRectFromPoints, + pathForNormalizedPoints, + redoMarkupHistory, + remainingAnnotationExportByteBudget, + undoMarkupHistory, + updateCalloutComment, + type MarkupDocument, + type MarkupHistory, + type MarkupSelection, + type MarkupSize, + type MarkupTool, +} from "./model"; + +const AnimatedPath = Animated.createAnimatedComponent(Path); +const AnimatedRect = Animated.createAnimatedComponent(Rect); + +const CALLOUT_BADGE_RADIUS = 0.019; +const CALLOUT_SELECTION_RADIUS_SCALE = 1.28; +const CALLOUT_REGION_STROKE_WIDTH = 0.003; +const ACTIVE_REGION_FILL_OPACITY = 0.12; +const ACTIVE_STROKE_CHUNK_COUNT = 16; +const ACTIVE_STROKE_POINTS_PER_CHUNK = 64; +const MAX_ACTIVE_STROKE_POINTS = ACTIVE_STROKE_CHUNK_COUNT * ACTIVE_STROKE_POINTS_PER_CHUNK; +const MIN_ACTIVE_STROKE_DISTANCE = 0.001; + +interface ImageMarkupModalProps { + readonly attachment: DraftComposerImageAttachment | null; + readonly visible: boolean; + readonly semanticElements?: ReadonlyArray; + readonly onCancel: () => void; + readonly onDone: (attachment: DraftComposerImageAttachment) => void; +} + +export interface ImageMarkupSemanticElement { + readonly target: PreviewAnnotationElementTarget; + readonly rect: PreviewAnnotationNormalizedRect; +} + +function MarkupToolbarButton(props: { + readonly label: string; + readonly icon?: AppSymbolName; + readonly active?: boolean; + readonly destructive?: boolean; + readonly disabled?: boolean; + readonly onPress: () => void; +}) { + const foreground = useThemeColor("--color-foreground"); + const muted = useThemeColor("--color-foreground-muted"); + const primary = useThemeColor("--color-primary"); + const danger = useThemeColor("--color-danger-foreground"); + const tint = props.disabled + ? muted + : props.destructive + ? danger + : props.active + ? primary + : foreground; + + return ( + + {props.icon ? ( + + ) : null} + + {props.label} + + + ); +} + +function selectedCallout( + document: MarkupDocument, + selection: MarkupSelection | null, +): PreviewAnnotationCallout | null { + if (selection?.kind !== "callout") return null; + return document.callouts.find((callout) => callout.id === selection.id) ?? null; +} + +function svgPngBase64(svg: Svg | null): Promise { + if (!svg) { + return Promise.reject(new Error("The annotation image is not ready.")); + } + return new Promise((resolve, reject) => { + svg.toDataURL((base64) => { + if (base64.length === 0) { + reject(new Error("The annotation image could not be rendered.")); + return; + } + resolve(base64); + }); + }); +} + +async function exportFlattenedPng( + render: (size: MarkupSize) => Promise, + initialSize: MarkupSize, + maximumSizeBytes: number, +): Promise<{ + readonly base64: string; + readonly size: MarkupSize; + readonly sizeBytes: number; +}> { + if (maximumSizeBytes <= 0) { + throw new Error( + "The original image leaves no room for markup within the 10 MB attachment storage limit.", + ); + } + let size: MarkupSize | null = initialSize; + while (size) { + const base64 = await render(size); + const sizeBytes = estimateBase64ByteSize(base64); + if (sizeBytes > 0 && sizeBytes <= maximumSizeBytes) { + return { base64, size, sizeBytes }; + } + size = nextSmallerExportSize(size); + } + throw new Error( + "The original and annotated PNG cannot both fit within the 10 MB attachment storage limit. Choose a smaller image.", + ); +} + +function dataUrlByteSize(dataUrl: string, fallbackSizeBytes: number): number { + const commaIndex = dataUrl.indexOf(","); + if (commaIndex < 0) return fallbackSizeBytes; + const estimated = estimateBase64ByteSize(dataUrl.slice(commaIndex + 1)); + return estimated > 0 ? estimated : fallbackSizeBytes; +} + +function badgeCenter(rawValue: number, radius: number, imageLength: number): number { + return Math.max(radius, Math.min(imageLength - radius, rawValue)); +} + +function CalloutShape(props: { + readonly callout: PreviewAnnotationCallout; + readonly imageSize: MarkupSize; +}) { + const shortSide = Math.min(props.imageSize.width, props.imageSize.height); + const radius = shortSide * CALLOUT_BADGE_RADIUS; + const badgeInset = radius * CALLOUT_SELECTION_RADIUS_SCALE; + const borderWidth = shortSide * CALLOUT_REGION_STROKE_WIDTH; + const anchor = props.callout.anchor; + const rect = anchor.kind === "point" ? null : anchor.rect; + const badgeX = badgeCenter( + anchor.kind === "point" + ? anchor.point.x * props.imageSize.width + : rect!.x * props.imageSize.width + radius, + badgeInset, + props.imageSize.width, + ); + const badgeY = badgeCenter( + anchor.kind === "point" + ? anchor.point.y * props.imageSize.height + : rect!.y * props.imageSize.height + radius, + badgeInset, + props.imageSize.height, + ); + + return ( + + {rect ? ( + + ) : null} + + + {String(props.callout.number)} + + + ); +} + +function CalloutSelectionShape(props: { + readonly callout: PreviewAnnotationCallout; + readonly imageSize: MarkupSize; +}) { + const shortSide = Math.min(props.imageSize.width, props.imageSize.height); + const radius = shortSide * CALLOUT_BADGE_RADIUS; + const badgeInset = radius * CALLOUT_SELECTION_RADIUS_SCALE; + const borderWidth = shortSide * CALLOUT_REGION_STROKE_WIDTH; + const anchor = props.callout.anchor; + const rect = anchor.kind === "point" ? null : anchor.rect; + const badgeX = badgeCenter( + anchor.kind === "point" + ? anchor.point.x * props.imageSize.width + : rect!.x * props.imageSize.width + radius, + badgeInset, + props.imageSize.width, + ); + const badgeY = badgeCenter( + anchor.kind === "point" + ? anchor.point.y * props.imageSize.height + : rect!.y * props.imageSize.height + radius, + badgeInset, + props.imageSize.height, + ); + + return ( + + {rect ? ( + + ) : null} + + + ); +} + +function CommittedMarkupContent(props: { + readonly backgroundDataUrl: string; + readonly document: MarkupDocument; + readonly imageSize: MarkupSize; + readonly onBackgroundLoad?: () => void; +}) { + const shortSide = Math.min(props.imageSize.width, props.imageSize.height); + return ( + <> + + {props.document.strokes.map((stroke) => ( + + ))} + {props.document.callouts.map((callout) => ( + + ))} + + ); +} + +function ActiveStrokeChunk(props: { + readonly paths: SharedValue>; + readonly index: number; + readonly strokeWidth: number; +}) { + const animatedProps = useAnimatedProps(() => ({ + d: props.paths.value[props.index] ?? "", + })); + return ( + + ); +} + +function emptyActiveStrokePaths(): ReadonlyArray { + return Array.from({ length: ACTIVE_STROKE_CHUNK_COUNT }, () => ""); +} + +interface ExportRenderRequest { + readonly id: number; + readonly document: MarkupDocument; + readonly layoutSize: MarkupSize; +} + +export function ImageMarkupModal(props: ImageMarkupModalProps) { + const attachment = props.attachment; + const semanticElements = props.semanticElements ?? []; + const hasSemanticElements = semanticElements.length > 0; + const muted = useThemeColor("--color-foreground-muted"); + const border = useThemeColor("--color-border"); + const card = useThemeColor("--color-card"); + const screen = useThemeColor("--color-screen"); + const primary = useThemeColor("--color-primary"); + const placeholder = useThemeColor("--color-placeholder"); + + const [tool, setTool] = useState("point"); + const [history, setHistory] = useState(() => + createMarkupHistory( + attachment ? markupDocumentFromAttachment(attachment) : EMPTY_MARKUP_DOCUMENT, + ), + ); + const [selection, setSelection] = useState(null); + const [commentDraft, setCommentDraft] = useState(""); + const [sourceSize, setSourceSize] = useState(null); + const [containerSize, setContainerSize] = useState({ width: 0, height: 0 }); + const [backgroundReady, setBackgroundReady] = useState(false); + const [error, setError] = useState(null); + const [isExporting, setIsExporting] = useState(false); + const [annotationId, setAnnotationId] = useState(""); + const [createdAt, setCreatedAt] = useState(""); + const [exportRenderRequest, setExportRenderRequest] = useState(null); + const exportSvgRef = useRef(null); + const exportRenderIdRef = useRef(0); + const exportRenderPromiseRef = useRef<{ + readonly id: number; + readonly resolve: (base64: string) => void; + readonly reject: (cause: Error) => void; + } | null>(null); + const sessionIdRef = useRef(0); + const activeExportIdRef = useRef(0); + const isSessionActiveRef = useRef(false); + const isExportingRef = useRef(false); + + const activeStrokePaths = useSharedValue>(emptyActiveStrokePaths()); + const activeStrokePoints = useSharedValue>([]); + const activeRegionStartX = useSharedValue(0); + const activeRegionStartY = useSharedValue(0); + const activeRegionX = useSharedValue(0); + const activeRegionY = useSharedValue(0); + const activeRegionWidth = useSharedValue(0); + const activeRegionHeight = useSharedValue(0); + + const original = attachment ? originalImageFromAttachment(attachment) : null; + const fit = useMemo( + () => (sourceSize ? aspectFit(sourceSize, containerSize) : { x: 0, y: 0, width: 0, height: 0 }), + [containerSize, sourceSize], + ); + const currentCallout = selectedCallout(history.present, selection); + + useEffect(() => { + if (!props.visible || !attachment) return; + const sessionId = sessionIdRef.current + 1; + sessionIdRef.current = sessionId; + isSessionActiveRef.current = true; + isExportingRef.current = false; + setHistory(createMarkupHistory(markupDocumentFromAttachment(attachment))); + setSelection(null); + setCommentDraft(""); + setTool(hasSemanticElements ? "element" : "point"); + setSourceSize(null); + setBackgroundReady(false); + setError(null); + setIsExporting(false); + setExportRenderRequest(null); + setAnnotationId(attachment.markup?.annotation.id ?? uuidv4()); + setCreatedAt(attachment.markup?.annotation.createdAt ?? new Date().toISOString()); + activeStrokePaths.value = emptyActiveStrokePaths(); + activeStrokePoints.value = []; + activeRegionWidth.value = 0; + activeRegionHeight.value = 0; + + let cancelled = false; + const durableSource = originalImageFromAttachment(attachment).dataUrl; + void Image.getSize(durableSource) + .then((size) => { + if (!cancelled) { + setSourceSize({ width: size.width, height: size.height }); + } + }) + .catch(() => { + if (!cancelled) { + setError("This image could not be opened for markup."); + } + }); + return () => { + cancelled = true; + exportRenderPromiseRef.current?.reject( + new Error("The annotation image export was cancelled."), + ); + exportRenderPromiseRef.current = null; + if (sessionIdRef.current === sessionId) { + sessionIdRef.current += 1; + isSessionActiveRef.current = false; + } + }; + }, [ + activeRegionHeight, + activeRegionWidth, + activeStrokePaths, + activeStrokePoints, + attachment, + hasSemanticElements, + props.visible, + ]); + + useEffect(() => { + if (!selection) return; + const stillExists = + selection.kind === "callout" + ? history.present.callouts.some((callout) => callout.id === selection.id) + : history.present.strokes.some((stroke) => stroke.id === selection.id); + if (!stillExists) setSelection(null); + }, [history.present, selection]); + + useEffect(() => { + setCommentDraft(currentCallout?.comment ?? ""); + }, [currentCallout?.comment, currentCallout?.id]); + + useEffect(() => { + activeStrokePaths.value = emptyActiveStrokePaths(); + activeStrokePoints.value = []; + activeRegionWidth.value = 0; + activeRegionHeight.value = 0; + }, [activeRegionHeight, activeRegionWidth, activeStrokePaths, activeStrokePoints, tool]); + + const renderExportPng = useCallback( + (size: MarkupSize, document: MarkupDocument) => + new Promise((resolve, reject) => { + exportRenderPromiseRef.current?.reject( + new Error("A newer annotation image export replaced this one."), + ); + const id = exportRenderIdRef.current + 1; + exportRenderIdRef.current = id; + exportRenderPromiseRef.current = { id, resolve, reject }; + setExportRenderRequest({ + id, + document, + layoutSize: annotationExportLayoutSize(size, PixelRatio.get()), + }); + }), + [], + ); + + const completeExportRender = useCallback((id: number) => { + requestAnimationFrame(() => { + const pending = exportRenderPromiseRef.current; + if (!pending || pending.id !== id) return; + void svgPngBase64(exportSvgRef.current) + .then((base64) => { + if (exportRenderPromiseRef.current?.id !== id) return; + exportRenderPromiseRef.current = null; + pending.resolve(base64); + }) + .catch((cause: unknown) => { + if (exportRenderPromiseRef.current?.id !== id) return; + exportRenderPromiseRef.current = null; + pending.reject( + cause instanceof Error + ? cause + : new Error("The annotation image could not be rendered."), + ); + }) + .finally(() => { + setExportRenderRequest((current) => (current?.id === id ? null : current)); + }); + }); + }, []); + + const commitDocument = useCallback((update: (document: MarkupDocument) => MarkupDocument) => { + setHistory((current) => commitMarkupDocument(current, update(current.present))); + }, []); + + const commitCurrentComment = useCallback(() => { + if (!currentCallout) return; + commitDocument((document) => updateCalloutComment(document, currentCallout.id, commentDraft)); + }, [commentDraft, commitDocument, currentCallout]); + + const handleCanvasTap = useCallback( + (x: number, y: number) => { + if (fit.width <= 0 || fit.height <= 0) return; + commitCurrentComment(); + const point = normalizedPoint({ x, y }, fit); + if (tool === "point") { + const id = uuidv4(); + commitDocument((document) => addPointCallout(document, { id, point })); + setSelection({ kind: "callout", id }); + return; + } + if (tool === "element") { + let selected: ImageMarkupSemanticElement | null = null; + let selectedArea = Number.POSITIVE_INFINITY; + for (const candidate of semanticElements) { + const rect = candidate.rect; + const contains = + point.x >= rect.x && + point.x <= rect.x + rect.width && + point.y >= rect.y && + point.y <= rect.y + rect.height; + const area = rect.width * rect.height; + if (contains && area < selectedArea) { + selected = candidate; + selectedArea = area; + } + } + if (!selected) { + setSelection(null); + return; + } + const id = uuidv4(); + commitDocument((document) => + addElementCallout(document, { + id, + targetId: selected!.target.id, + rect: selected!.rect, + }), + ); + setSelection({ kind: "callout", id }); + return; + } + setSelection(hitTestMarkupObject(history.present, point)); + }, + [commitCurrentComment, commitDocument, fit, history.present, semanticElements, tool], + ); + + const commitRegion = useCallback( + (startX: number, startY: number, endX: number, endY: number) => { + const rect = normalizedRectFromPoints({ x: startX, y: startY }, { x: endX, y: endY }); + if (!rect) return; + commitCurrentComment(); + const id = uuidv4(); + commitDocument((document) => addRegionCallout(document, { id, rect })); + setSelection({ kind: "callout", id }); + }, + [commitCurrentComment, commitDocument], + ); + + const commitStroke = useCallback( + (points: ReadonlyArray) => { + const stroke = makeStroke({ + id: uuidv4(), + color: DEFAULT_STROKE_COLOR, + width: DEFAULT_NORMALIZED_STROKE_WIDTH, + points, + }); + if (!stroke) return; + commitCurrentComment(); + commitDocument((document) => addStroke(document, stroke)); + setSelection({ kind: "stroke", id: stroke.id }); + }, + [commitCurrentComment, commitDocument], + ); + + const eraseAt = useCallback( + (x: number, y: number) => { + if (fit.width <= 0 || fit.height <= 0) return; + const point = normalizedPoint({ x, y }, fit); + commitCurrentComment(); + commitDocument((document) => eraseMarkupObjectAtPoint(document, point)); + setSelection(null); + }, + [commitCurrentComment, commitDocument, fit], + ); + + const canvasGesture = useMemo(() => { + if (!sourceSize || fit.width <= 0 || fit.height <= 0) { + return Gesture.Tap().enabled(false); + } + if (tool === "select" || tool === "element" || tool === "point") { + return Gesture.Tap().onEnd((event, success) => { + if (success) runOnJS(handleCanvasTap)(event.x, event.y); + }); + } + if (tool === "region") { + return Gesture.Pan() + .minDistance(0) + .maxPointers(1) + .onBegin((event) => { + const normalizedX = Math.max(0, Math.min(1, event.x / fit.width)); + const normalizedY = Math.max(0, Math.min(1, event.y / fit.height)); + activeRegionStartX.value = normalizedX; + activeRegionStartY.value = normalizedY; + activeRegionX.value = normalizedX * sourceSize.width; + activeRegionY.value = normalizedY * sourceSize.height; + activeRegionWidth.value = 0; + activeRegionHeight.value = 0; + }) + .onUpdate((event) => { + const normalizedX = Math.max(0, Math.min(1, event.x / fit.width)); + const normalizedY = Math.max(0, Math.min(1, event.y / fit.height)); + activeRegionX.value = Math.min(activeRegionStartX.value, normalizedX) * sourceSize.width; + activeRegionY.value = Math.min(activeRegionStartY.value, normalizedY) * sourceSize.height; + activeRegionWidth.value = + Math.abs(normalizedX - activeRegionStartX.value) * sourceSize.width; + activeRegionHeight.value = + Math.abs(normalizedY - activeRegionStartY.value) * sourceSize.height; + }) + .onEnd((event) => { + const endX = Math.max(0, Math.min(1, event.x / fit.width)); + const endY = Math.max(0, Math.min(1, event.y / fit.height)); + runOnJS(commitRegion)(activeRegionStartX.value, activeRegionStartY.value, endX, endY); + activeRegionWidth.value = 0; + activeRegionHeight.value = 0; + }) + .onFinalize(() => { + activeRegionWidth.value = 0; + activeRegionHeight.value = 0; + }); + } + if (tool === "erase") { + return Gesture.Pan() + .minDistance(0) + .maxPointers(1) + .onBegin((event) => { + runOnJS(eraseAt)(event.x, event.y); + }) + .onUpdate((event) => { + runOnJS(eraseAt)(event.x, event.y); + }); + } + return Gesture.Pan() + .minDistance(0) + .maxPointers(1) + .onBegin((event) => { + const x = Math.max(0, Math.min(1, event.x / fit.width)); + const y = Math.max(0, Math.min(1, event.y / fit.height)); + const point = { x, y }; + activeStrokePoints.value = [point]; + const sourcePoint = `${x * sourceSize.width} ${y * sourceSize.height}`; + activeStrokePaths.value = activeStrokePaths.value.map((_, index) => + index === 0 ? `M ${sourcePoint} L ${sourcePoint}` : "", + ); + }) + .onUpdate((event) => { + const x = Math.max(0, Math.min(1, event.x / fit.width)); + const y = Math.max(0, Math.min(1, event.y / fit.height)); + const points = activeStrokePoints.value; + const previous = points.at(-1); + if ( + !previous || + points.length >= MAX_ACTIVE_STROKE_POINTS || + Math.abs(previous.x - x) + Math.abs(previous.y - y) < MIN_ACTIVE_STROKE_DISTANCE + ) { + return; + } + + const segmentIndex = points.length - 1; + const chunkIndex = Math.floor(segmentIndex / ACTIVE_STROKE_POINTS_PER_CHUNK); + const startsChunk = segmentIndex % ACTIVE_STROKE_POINTS_PER_CHUNK === 0; + const previousSourcePoint = `${previous.x * sourceSize.width} ${previous.y * sourceSize.height}`; + const sourcePoint = `${x * sourceSize.width} ${y * sourceSize.height}`; + activeStrokePoints.modify((current) => { + current.push({ x, y }); + return current; + }); + activeStrokePaths.modify((current) => { + const paths = [...current]; + paths[chunkIndex] = startsChunk + ? `M ${previousSourcePoint} L ${sourcePoint}` + : `${paths[chunkIndex]} L ${sourcePoint}`; + return paths; + }); + }) + .onEnd(() => { + const points = [...activeStrokePoints.value]; + if (points.length > 0) runOnJS(commitStroke)(points); + activeStrokePoints.value = []; + activeStrokePaths.value = activeStrokePaths.value.map(() => ""); + }) + .onFinalize(() => { + activeStrokePoints.value = []; + activeStrokePaths.value = activeStrokePaths.value.map(() => ""); + }); + }, [ + activeRegionHeight, + activeRegionStartX, + activeRegionStartY, + activeRegionWidth, + activeRegionX, + activeRegionY, + activeStrokePaths, + activeStrokePoints, + commitRegion, + commitStroke, + eraseAt, + fit.height, + fit.width, + handleCanvasTap, + sourceSize, + tool, + ]); + + const activeRegionAnimatedProps = useAnimatedProps(() => ({ + x: activeRegionX.value, + y: activeRegionY.value, + width: activeRegionWidth.value, + height: activeRegionHeight.value, + })); + + const handleLayout = useCallback((event: LayoutChangeEvent) => { + setContainerSize({ + width: event.nativeEvent.layout.width, + height: event.nativeEvent.layout.height, + }); + }, []); + + const handleCancel = useCallback(() => { + if (isExportingRef.current) return; + sessionIdRef.current += 1; + isSessionActiveRef.current = false; + props.onCancel(); + }, [props.onCancel]); + + const handleDone = useCallback(async () => { + if ( + isExportingRef.current || + !attachment || + !sourceSize || + !backgroundReady || + markupDocumentIsEmpty(history.present) + ) { + return; + } + const document = + currentCallout === null + ? history.present + : updateCalloutComment(history.present, currentCallout.id, commentDraft); + const sessionId = sessionIdRef.current; + const exportId = activeExportIdRef.current + 1; + activeExportIdRef.current = exportId; + isExportingRef.current = true; + setIsExporting(true); + setError(null); + try { + const durableOriginal = originalImageFromAttachment(attachment); + const originalSizeBytes = dataUrlByteSize(durableOriginal.dataUrl, durableOriginal.sizeBytes); + const flattenedByteBudget = remainingAnnotationExportByteBudget( + originalSizeBytes, + PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, + ); + const initialExportSize = annotationExportSize(sourceSize); + const flattened = await exportFlattenedPng( + (size) => renderExportPng(size, document), + initialExportSize, + flattenedByteBudget, + ); + if ( + !isSessionActiveRef.current || + sessionIdRef.current !== sessionId || + activeExportIdRef.current !== exportId + ) { + return; + } + const flattenedDataUrl = `data:image/png;base64,${flattened.base64}`; + props.onDone( + buildAnnotatedImageAttachment({ + attachment, + document, + sourceSize, + exportSize: flattened.size, + annotationId, + createdAt, + flattenedDataUrl, + flattenedSizeBytes: flattened.sizeBytes, + semanticElements: semanticElements.map((candidate) => candidate.target), + }), + ); + } catch (cause) { + if ( + isSessionActiveRef.current && + sessionIdRef.current === sessionId && + activeExportIdRef.current === exportId + ) { + setError( + cause instanceof Error ? cause.message : "The annotated image could not be saved.", + ); + } + } finally { + if (activeExportIdRef.current === exportId) { + isExportingRef.current = false; + if (isSessionActiveRef.current && sessionIdRef.current === sessionId) { + setIsExporting(false); + } + } + } + }, [ + annotationId, + attachment, + backgroundReady, + commentDraft, + createdAt, + currentCallout, + history.present, + props, + renderExportPng, + semanticElements, + sourceSize, + ]); + + if (!attachment || !original) { + return null; + } + + const shortSide = sourceSize ? Math.min(sourceSize.width, sourceSize.height) : 1; + const canDelete = selection !== null; + const canUndo = history.past.length > 0; + const canRedo = history.future.length > 0; + const selectedStroke = + selection?.kind === "stroke" + ? (history.present.strokes.find((stroke) => stroke.id === selection.id) ?? null) + : null; + const canDone = + sourceSize !== null && + backgroundReady && + !markupDocumentIsEmpty(history.present) && + !isExporting; + + return ( + + + + + Cancel + + + + Markup + + + {original.name} + + + void handleDone()} + className="h-11 min-w-16 items-end justify-center pl-4" + > + {isExporting ? ( + + ) : ( + + Done + + )} + + + + + + {sourceSize && original && exportRenderRequest ? ( + + completeExportRender(exportRenderRequest.id)} + /> + + ) : null} + {sourceSize && fit.width > 0 && fit.height > 0 ? ( + + + + + setBackgroundReady(true)} + /> + + + {selectedStroke ? ( + + + + + ) : null} + {tool === "element" + ? semanticElements.map((candidate) => ( + + )) + : null} + {currentCallout ? ( + + ) : null} + {Array.from({ length: ACTIVE_STROKE_CHUNK_COUNT }, (_, index) => ( + + ))} + + + + + + ) : error ? ( + + {error} + + ) : ( + + + + )} + + + {currentCallout ? ( + + + Comment for #{currentCallout.number} + + + commitDocument((document) => + updateCalloutComment(document, currentCallout.id, commentDraft), + ) + } + placeholder="Describe the requested change" + placeholderTextColor={placeholder} + multiline + className="min-h-11 rounded-xl border px-3 py-2 text-base text-foreground" + style={{ borderColor: border }} + /> + + ) : null} + + {error && sourceSize ? ( + + {error} + + ) : null} + + + + setTool("select")} + /> + {hasSemanticElements ? ( + setTool("element")} + /> + ) : null} + setTool("point")} + /> + setTool("region")} + /> + setTool("pen")} + /> + setTool("erase")} + /> + { + setHistory(undoMarkupHistory); + setSelection(null); + }} + /> + { + setHistory(redoMarkupHistory); + setSelection(null); + }} + /> + { + commitDocument((document) => deleteMarkupObject(document, selection)); + setSelection(null); + }} + /> + { + commitDocument(clearMarkupDocument); + setSelection(null); + }} + /> + + {attachment.markup ? ( + + props.onDone(restoreComposerImageOriginal(attachment))} + className="h-9 justify-center px-4" + > + Remove markup + + + ) : null} + + + + + ); +} diff --git a/apps/mobile/src/features/annotations/attachmentMarkup.test.ts b/apps/mobile/src/features/annotations/attachmentMarkup.test.ts new file mode 100644 index 00000000000..63cdaf82c24 --- /dev/null +++ b/apps/mobile/src/features/annotations/attachmentMarkup.test.ts @@ -0,0 +1,244 @@ +import { PreviewAnnotationPayloadSchema } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import type { DraftComposerImageAttachment } from "../../lib/composerImages"; +import { describe, expect, it } from "vite-plus/test"; + +import { + buildAnnotatedImageAttachment, + markupDocumentFromAttachment, + originalImageFromAttachment, +} from "./attachmentMarkup"; +import { EMPTY_MARKUP_DOCUMENT, addElementCallout, addPointCallout } from "./model"; + +const decodeAnnotation = Schema.decodeUnknownSync(PreviewAnnotationPayloadSchema); + +const originalAttachment: DraftComposerImageAttachment = { + id: "image-1", + type: "image", + name: "screen.jpg", + mimeType: "image/jpeg", + sizeBytes: 123, + dataUrl: "data:image/jpeg;base64,b3JpZ2luYWw=", + previewUri: "file:///screen.jpg", +}; + +describe("mobile image annotation attachment adapter", () => { + it("creates a flattened PNG while preserving the durable original", () => { + const document = addPointCallout(EMPTY_MARKUP_DOCUMENT, { + id: "callout-1", + point: { x: 0.25, y: 0.75 }, + comment: "Move this", + }); + const attachment = buildAnnotatedImageAttachment({ + attachment: originalAttachment, + document, + sourceSize: { width: 1_200, height: 800 }, + exportSize: { width: 1_024, height: 683 }, + annotationId: "annotation-1", + createdAt: "2026-07-30T12:00:00.000Z", + flattenedDataUrl: "data:image/png;base64,ZmxhdHRlbmVk", + flattenedSizeBytes: 222, + }); + + expect(attachment).toMatchObject({ + id: "image-1", + name: "preview-annotation-annotation-1.png", + mimeType: "image/png", + sizeBytes: 222, + previewUri: "data:image/png;base64,ZmxhdHRlbmVk", + markup: { + original: { + name: "screen.jpg", + mimeType: "image/jpeg", + sizeBytes: 123, + dataUrl: "data:image/jpeg;base64,b3JpZ2luYWw=", + previewUri: "file:///screen.jpg", + }, + annotation: { + schemaVersion: 1, + source: { kind: "image", name: "screen.jpg" }, + callouts: document.callouts, + editable: { + version: 1, + coordinateSpace: "normalized", + strokes: [], + }, + elements: [], + regions: [], + strokes: [], + styleChanges: [], + screenshot: { + dataUrl: "", + width: 1_024, + height: 683, + cropRect: { x: 0, y: 0, width: 1_200, height: 800 }, + }, + }, + }, + }); + expect(decodeAnnotation(attachment.markup?.annotation)).toEqual(attachment.markup?.annotation); + }); + + it("reopens editable vectors and does not replace the original with a prior flatten", () => { + const first = buildAnnotatedImageAttachment({ + attachment: originalAttachment, + document: EMPTY_MARKUP_DOCUMENT, + sourceSize: { width: 100, height: 100 }, + exportSize: { width: 100, height: 100 }, + annotationId: "annotation-1", + createdAt: "2026-07-30T12:00:00.000Z", + flattenedDataUrl: "data:image/png;base64,Zmlyc3Q=", + flattenedSizeBytes: 50, + }); + const reopenedDocument = addPointCallout(markupDocumentFromAttachment(first), { + id: "callout-2", + point: { x: 0.5, y: 0.5 }, + }); + const second = buildAnnotatedImageAttachment({ + attachment: first, + document: reopenedDocument, + sourceSize: { width: 100, height: 100 }, + exportSize: { width: 100, height: 100 }, + annotationId: "ignored-new-id", + createdAt: "2026-07-30T13:00:00.000Z", + flattenedDataUrl: "data:image/png;base64,c2Vjb25k", + flattenedSizeBytes: 60, + }); + + expect(second.markup?.annotation.id).toBe("annotation-1"); + expect(second.markup?.annotation.createdAt).toBe("2026-07-30T12:00:00.000Z"); + expect(originalImageFromAttachment(second).dataUrl).toBe(originalAttachment.dataUrl); + expect(markupDocumentFromAttachment(second).callouts).toEqual(reopenedDocument.callouts); + expect(second.markup?.annotation.screenshot?.scale).toBe(1); + }); + + it("derives a generic image re-edit scale from the previous flatten", () => { + const first = buildAnnotatedImageAttachment({ + attachment: originalAttachment, + document: EMPTY_MARKUP_DOCUMENT, + sourceSize: { width: 1_200, height: 800 }, + exportSize: { width: 600, height: 400 }, + annotationId: "annotation-1", + createdAt: "2026-07-30T12:00:00.000Z", + flattenedDataUrl: "data:image/png;base64,Zmlyc3Q=", + flattenedSizeBytes: 50, + }); + const second = buildAnnotatedImageAttachment({ + attachment: first, + document: EMPTY_MARKUP_DOCUMENT, + sourceSize: { width: 1_200, height: 800 }, + exportSize: { width: 300, height: 200 }, + annotationId: "ignored-new-id", + createdAt: "2026-07-30T13:00:00.000Z", + flattenedDataUrl: "data:image/png;base64,c2Vjb25k", + flattenedSizeBytes: 60, + }); + + expect(first.markup?.annotation.screenshot?.scale).toBe(0.5); + expect(second.markup?.annotation.screenshot?.scale).toBe(0.25); + }); + + it("preserves preview identity and includes only selected semantic elements", () => { + const candidate = { + id: "element-submit", + rect: { x: 420, y: 180, width: 160, height: 64 }, + element: { + pageUrl: "http://localhost:5173/checkout", + pageTitle: "Checkout", + tagName: "button", + selector: "button.submit", + htmlPreview: '
@@ -960,9 +973,7 @@ function UserTimelineRow({ row }: { row: Extract
{canRevertAgentWork && } - {displayedUserMessage.copyText && ( - - )} + {copyText && }
@@ -1348,8 +1359,17 @@ function UserMessagePreviewAnnotationCard(props: { image: NonNullable[number] | null; }) { const ctx = use(TimelineRowCtx); + const [expanded, setExpanded] = useState(false); + const { callouts, hiddenCalloutCount, canExpand } = deriveUserMessagePreviewAnnotationCardContent( + props.annotation, + expanded, + ); + const expandLabel = + hiddenCalloutCount > 0 + ? `Show all ${props.annotation.calloutCount} callouts` + : "Show full annotation"; return ( -
+
{props.image?.previewUrl ? ( ) : null} -
+
{props.annotation.comment ? ( -
+
{props.annotation.comment}
) : null} + {callouts.length > 0 ? ( +
    + {callouts.map((callout) => ( +
  1. + + Callout {callout.number}: + + {callout.comment || callout.anchorSummary} + +
    2. + ))} +
+ ) : null} + {hiddenCalloutCount > 0 ? ( +
+ +{hiddenCalloutCount} more callout{hiddenCalloutCount === 1 ? "" : "s"} +
+ ) : null}
0 || hiddenCalloutCount > 0) && "mt-1", )} > {props.annotation.targetSummary ? ( @@ -1390,6 +1444,22 @@ function UserMessagePreviewAnnotationCard(props: { ) : null}
+ {canExpand ? ( + + ) : null}
); diff --git a/apps/web/src/components/chat/userMessagePreviewAnnotation.test.ts b/apps/web/src/components/chat/userMessagePreviewAnnotation.test.ts new file mode 100644 index 00000000000..894446eb882 --- /dev/null +++ b/apps/web/src/components/chat/userMessagePreviewAnnotation.test.ts @@ -0,0 +1,67 @@ +import type { ParsedPreviewAnnotation } from "~/lib/previewAnnotation"; +import { describe, expect, it } from "vite-plus/test"; + +import { + deriveUserMessagePreviewAnnotationCardContent, + deriveUserMessagePreviewAnnotationCopyText, +} from "./userMessagePreviewAnnotation"; + +const annotation: ParsedPreviewAnnotation = { + id: "annotation-1", + title: "Checkout screenshot", + comment: "Polish the whole flow.", + targetSummary: "5 numbered callouts.", + styleChanges: [], + hasScreenshot: true, + calloutCount: 5, + callouts: Array.from({ length: 5 }, (_, index) => ({ + number: index + 1, + anchorSummary: `point: x=.${index + 1}, y=.5`, + comment: `Instruction ${index + 1}`, + })), +}; + +describe("deriveUserMessagePreviewAnnotationCardContent", () => { + it("keeps the collapsed card compact and exposes every callout when expanded", () => { + expect(deriveUserMessagePreviewAnnotationCardContent(annotation, false)).toMatchObject({ + callouts: annotation.callouts.slice(0, 3), + hiddenCalloutCount: 2, + canExpand: true, + }); + expect(deriveUserMessagePreviewAnnotationCardContent(annotation, true)).toMatchObject({ + callouts: annotation.callouts, + hiddenCalloutCount: 0, + canExpand: true, + }); + }); + + it("keeps comment-only annotations expandable so truncated text remains recoverable", () => { + expect( + deriveUserMessagePreviewAnnotationCardContent( + { ...annotation, calloutCount: 0, callouts: [] }, + false, + ), + ).toMatchObject({ + callouts: [], + hiddenCalloutCount: 0, + canExpand: true, + }); + }); + + it("replaces raw annotation markup with complete human-readable copy", () => { + const copyText = deriveUserMessagePreviewAnnotationCopyText({ + visibleText: "Update checkout", + annotations: [annotation], + fallbackCopyText: "Update checkout\n\nraw", + }); + + expect(copyText).toContain("Update checkout\n\nAnnotation: Checkout screenshot"); + expect(copyText).toContain("Polish the whole flow."); + for (const callout of annotation.callouts) { + expect(copyText).toContain(`#${callout.number} [${callout.anchorSummary}]`); + expect(copyText).toContain(callout.comment); + } + expect(copyText).not.toContain(""); + expect(copyText).not.toContain(""); + }); +}); diff --git a/apps/web/src/components/chat/userMessagePreviewAnnotation.ts b/apps/web/src/components/chat/userMessagePreviewAnnotation.ts new file mode 100644 index 00000000000..f26eb90c7f7 --- /dev/null +++ b/apps/web/src/components/chat/userMessagePreviewAnnotation.ts @@ -0,0 +1,41 @@ +import { + buildPreviewAnnotationCopyText, + type ParsedPreviewAnnotation, + type ParsedPreviewAnnotationCallout, +} from "~/lib/previewAnnotation"; + +const COLLAPSED_CALLOUT_LIMIT = 3; + +export interface UserMessagePreviewAnnotationCardContent { + readonly callouts: ReadonlyArray; + readonly hiddenCalloutCount: number; + readonly canExpand: boolean; +} + +export function deriveUserMessagePreviewAnnotationCardContent( + annotation: ParsedPreviewAnnotation, + expanded: boolean, +): UserMessagePreviewAnnotationCardContent { + const callouts = expanded + ? annotation.callouts + : annotation.callouts.slice(0, COLLAPSED_CALLOUT_LIMIT); + return { + callouts, + hiddenCalloutCount: Math.max(0, annotation.callouts.length - callouts.length), + canExpand: + annotation.comment.trim().length > 0 || + annotation.callouts.some( + (callout) => callout.comment.trim().length > 0 || callout.anchorSummary.length > 0, + ), + }; +} + +export function deriveUserMessagePreviewAnnotationCopyText(input: { + readonly visibleText: string; + readonly annotations: ReadonlyArray; + readonly fallbackCopyText: string; +}): string { + return input.annotations.length > 0 + ? buildPreviewAnnotationCopyText(input.visibleText, input.annotations) + : input.fallbackCopyText; +} diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 2a007fb4ce5..f2a67c7f991 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -64,6 +64,7 @@ import { waitForNavigationReadiness, } from "./previewNavigationReadiness"; import { createPreviewAutomationRequestConsumerAtom } from "./previewAutomationRequestConsumer"; +import { capturePreviewAutomationSnapshotResponse } from "./previewAutomationReviewSnapshot"; import { createPreviewAutomationClientId } from "./previewAutomationClientId"; import { needsPreviewAutomationSessionSync, @@ -577,7 +578,13 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "snapshot": { const ready = await requireReadyTab(); - return await ready.bridge.automation.snapshot(ready.runtimeTabId); + return await capturePreviewAutomationSnapshotResponse({ + requestInput: request.input, + capture: (options) => + options === undefined + ? ready.bridge.automation.snapshot(ready.runtimeTabId) + : ready.bridge.automation.snapshot(ready.runtimeTabId, options), + }); } case "click": { const ready = await requireReadyTab(); diff --git a/apps/web/src/components/preview/previewAutomationReviewSnapshot.test.ts b/apps/web/src/components/preview/previewAutomationReviewSnapshot.test.ts new file mode 100644 index 00000000000..ace50787285 --- /dev/null +++ b/apps/web/src/components/preview/previewAutomationReviewSnapshot.test.ts @@ -0,0 +1,138 @@ +import type { PreviewAutomationSnapshot } from "@t3tools/contracts"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { capturePreviewAutomationSnapshotResponse } from "./previewAutomationReviewSnapshot"; + +const fullSnapshot: PreviewAutomationSnapshot = { + url: "https://example.com/review", + title: "Review", + loading: false, + viewport: { + width: 1_000, + height: 600, + scrollX: 0, + scrollY: 120, + devicePixelRatio: 2, + }, + pageRevision: "page-1", + capturedAt: "2026-07-30T12:00:00.000Z", + visibleText: "Private diagnostic text", + interactiveElements: [ + { + tag: "button", + role: "button", + name: "Save", + selector: "#save", + x: 10, + y: 20, + width: 80, + height: 32, + }, + ], + accessibilityTree: { nodes: [{ nodeId: "1" }] }, + consoleEntries: [ + { + level: "log", + text: "private console entry", + timestamp: "2026-07-30T12:00:00.000Z", + }, + ], + networkEntries: [ + { + url: "https://example.com/api/private", + method: "GET", + status: 200, + failed: false, + timestamp: "2026-07-30T12:00:00.000Z", + }, + ], + actionTimeline: [ + { + id: "action-1", + action: "click", + status: "succeeded", + startedAt: "2026-07-30T12:00:00.000Z", + }, + ], + screenshot: { + mimeType: "image/png", + data: "cG5n", + width: 1_280, + height: 768, + scale: 1.28, + }, +}; + +describe("capturePreviewAutomationSnapshotResponse", () => { + it("requests review mode and strips diagnostic fields before relay", async () => { + const capture = vi.fn(async () => fullSnapshot); + + const result = await capturePreviewAutomationSnapshotResponse({ + requestInput: { mode: "review" }, + capture, + }); + + expect(capture).toHaveBeenCalledWith({ mode: "review" }); + expect(result).toEqual({ + url: fullSnapshot.url, + title: fullSnapshot.title, + loading: fullSnapshot.loading, + viewport: fullSnapshot.viewport, + pageRevision: fullSnapshot.pageRevision, + capturedAt: fullSnapshot.capturedAt, + interactiveElements: fullSnapshot.interactiveElements, + screenshot: fullSnapshot.screenshot, + }); + expect(result).not.toHaveProperty("visibleText"); + expect(result).not.toHaveProperty("accessibilityTree"); + expect(result).not.toHaveProperty("consoleEntries"); + expect(result).not.toHaveProperty("networkEntries"); + expect(result).not.toHaveProperty("actionTimeline"); + }); + + it("keeps legacy snapshot requests full and calls the bridge without options", async () => { + const capture = vi.fn(async () => fullSnapshot); + + const result = await capturePreviewAutomationSnapshotResponse({ + requestInput: {}, + capture, + }); + + expect(capture.mock.calls).toEqual([[]]); + expect(result).toBe(fullSnapshot); + }); + + it("accepts review responses from legacy hosts without viewport metadata", async () => { + const legacyHostSnapshot: PreviewAutomationSnapshot = { + url: fullSnapshot.url, + title: fullSnapshot.title, + loading: fullSnapshot.loading, + visibleText: fullSnapshot.visibleText, + interactiveElements: fullSnapshot.interactiveElements, + accessibilityTree: fullSnapshot.accessibilityTree, + consoleEntries: fullSnapshot.consoleEntries, + networkEntries: fullSnapshot.networkEntries, + actionTimeline: fullSnapshot.actionTimeline, + screenshot: { + mimeType: "image/png", + data: "cG5n", + width: 1_000, + height: 600, + }, + }; + const capture = vi.fn(async () => legacyHostSnapshot); + + const result = await capturePreviewAutomationSnapshotResponse({ + requestInput: { mode: "review" }, + capture, + }); + + expect(result).toEqual({ + url: legacyHostSnapshot.url, + title: legacyHostSnapshot.title, + loading: legacyHostSnapshot.loading, + interactiveElements: legacyHostSnapshot.interactiveElements, + screenshot: legacyHostSnapshot.screenshot, + }); + }); +}); diff --git a/apps/web/src/components/preview/previewAutomationReviewSnapshot.ts b/apps/web/src/components/preview/previewAutomationReviewSnapshot.ts new file mode 100644 index 00000000000..1a7f1f5f2fa --- /dev/null +++ b/apps/web/src/components/preview/previewAutomationReviewSnapshot.ts @@ -0,0 +1,29 @@ +import { + PreviewAutomationReviewSnapshot as PreviewAutomationReviewSnapshotSchema, + type DesktopPreviewAutomationSnapshotOptions, + type PreviewAutomationReviewSnapshot, + type PreviewAutomationSnapshot, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; + +const decodeReviewSnapshot = Schema.decodeUnknownSync(PreviewAutomationReviewSnapshotSchema); + +const isReviewSnapshotRequest = (input: unknown): boolean => + typeof input === "object" && + input !== null && + !Array.isArray(input) && + "mode" in input && + input.mode === "review"; + +export async function capturePreviewAutomationSnapshotResponse(input: { + readonly requestInput: unknown; + readonly capture: ( + options?: DesktopPreviewAutomationSnapshotOptions, + ) => Promise; +}): Promise { + if (!isReviewSnapshotRequest(input.requestInput)) { + return await input.capture(); + } + const snapshot = await input.capture({ mode: "review" }); + return decodeReviewSnapshot(snapshot); +} diff --git a/apps/web/src/lib/previewAnnotation.ts b/apps/web/src/lib/previewAnnotation.ts index f1723dd93aa..6348c9a6b24 100644 --- a/apps/web/src/lib/previewAnnotation.ts +++ b/apps/web/src/lib/previewAnnotation.ts @@ -1,103 +1,15 @@ import type { PreviewAnnotationPayload } from "@t3tools/contracts"; -import { buildElementContextBlock, normalizeElementContextSelection } from "./elementContext"; - -const TRAILING_PREVIEW_ANNOTATION_BLOCK_PATTERN = - /\n*\n((?:(?!)[\s\S])*)\n<\/preview_annotation>\s*$/; - -export interface ParsedPreviewAnnotation { - id: string; - title: string; - comment: string; - targetSummary: string; - styleChanges: string[]; - hasScreenshot: boolean; -} - -export interface ExtractedPreviewAnnotation { - promptText: string; - annotation: ParsedPreviewAnnotation | null; -} - -export function buildPreviewAnnotationPrompt(annotation: PreviewAnnotationPayload): string { - const lines = ["Preview annotation:"]; - lines.push(`Id: ${annotation.id}`); - const title = annotation.pageTitle?.trim() || annotation.pageUrl.trim() || "Preview"; - lines.push(`Page: ${title}`); - if (annotation.comment.trim()) lines.push(`Comment: ${annotation.comment.trim()}`); - const targets: string[] = []; - if (annotation.elements.length > 0) { - targets.push( - `${annotation.elements.length} selected element${annotation.elements.length === 1 ? "" : "s"}`, - ); - } - if (annotation.regions.length > 0) { - targets.push( - `${annotation.regions.length} marked region${annotation.regions.length === 1 ? "" : "s"}`, - ); - } - if (annotation.strokes.length > 0) { - targets.push( - `${annotation.strokes.length} drawing${annotation.strokes.length === 1 ? "" : "s"}`, - ); - } - if (targets.length > 0) lines.push(`Targets: ${targets.join(", ")}.`); - if (annotation.styleChanges.length > 0) { - lines.push("Requested visual changes:"); - for (const change of annotation.styleChanges) { - lines.push(`- ${change.property}: ${change.previousValue || "(unset)"} → ${change.value}`); - } - } - if (annotation.screenshot) { - lines.push("The attached screenshot is the annotated preview crop."); - } - const elementContexts = annotation.elements - .map((target) => normalizeElementContextSelection(target.element)) - .filter((context) => context !== null); - const elementBlock = buildElementContextBlock(elementContexts); - if (elementBlock) lines.push(elementBlock); - return ["", ...lines, ""].join("\n"); -} - -export function appendPreviewAnnotationPrompt( - prompt: string, - annotation: PreviewAnnotationPayload, -): string { - const annotationText = buildPreviewAnnotationPrompt(annotation); - const trimmed = prompt.trim(); - return trimmed ? `${trimmed}\n\n${annotationText}` : annotationText; -} - -export function extractTrailingPreviewAnnotation(prompt: string): ExtractedPreviewAnnotation { - const match = TRAILING_PREVIEW_ANNOTATION_BLOCK_PATTERN.exec(prompt); - if (!match) return { promptText: prompt, annotation: null }; - const body = match[1] ?? ""; - const lines = body.split("\n"); - const pageLine = lines.find((line) => line.startsWith("Page: ")); - const idLine = lines.find((line) => line.startsWith("Id: ")); - const commentLine = lines.find((line) => line.startsWith("Comment: ")); - const targetsLine = lines.find((line) => line.startsWith("Targets: ")); - const styleHeadingIndex = lines.indexOf("Requested visual changes:"); - const linesAfterStyleHeading = lines.slice(styleHeadingIndex + 1); - const elementContextIndex = linesAfterStyleHeading.indexOf(""); - const styleChanges = - styleHeadingIndex < 0 - ? [] - : linesAfterStyleHeading - .slice(0, elementContextIndex < 0 ? undefined : elementContextIndex) - .filter((line) => line.startsWith("- ")) - .map((line) => line.slice(2)); - return { - promptText: prompt.slice(0, match.index).replace(/\n+$/, ""), - annotation: { - id: idLine?.slice("Id: ".length).trim() || `${match.index}`, - title: pageLine?.slice("Page: ".length).trim() || "Preview annotation", - comment: commentLine?.slice("Comment: ".length).trim() || "", - targetSummary: targetsLine?.slice("Targets: ".length).trim() || "", - styleChanges, - hasScreenshot: body.includes("The attached screenshot is the annotated preview crop."), - }, - }; -} +export { + appendPreviewAnnotationPrompt, + buildPreviewAnnotationCopyText, + buildPreviewAnnotationPrompt, + extractTrailingPreviewAnnotation, + extractTrailingPreviewAnnotations, + type ExtractedPreviewAnnotation, + type ExtractedPreviewAnnotations, + type ParsedPreviewAnnotation, + type ParsedPreviewAnnotationCallout, +} from "@t3tools/client-runtime/annotations"; export async function previewAnnotationScreenshotFile( annotation: PreviewAnnotationPayload, diff --git a/docs/README.md b/docs/README.md index bc359826a04..3d65ab11ad5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,12 +6,13 @@ - [Permission modes](./user/permission-modes.md) - [Keyboard shortcuts](./user/keybindings.md) - [Remote access](./user/remote-access.md) +- [Mobile Browser and image markup](./user/mobile.md) - [Keeping app and server in sync](./user/updating.md) - [Source control integrations](./user/source-control.md) - [Background service (Linux)](./user/background-service.md) - Providers: [Codex](./user/providers-codex.md) · [Claude](./user/providers-claude.md) -Mobile app: [apps/mobile/README.md](../apps/mobile/README.md) +Mobile development: [apps/mobile/README.md](../apps/mobile/README.md) --- diff --git a/docs/internals/remote.md b/docs/internals/remote.md index afce95f725b..5d3f6c23c1f 100644 --- a/docs/internals/remote.md +++ b/docs/internals/remote.md @@ -155,6 +155,69 @@ A T3-managed `tailscale serve` mapping exposes the server on the tailnet over HT resulting private-network endpoints are advertised for pairing. Connection then follows the ordinary bearer path. +### Preview review versus preview browsing + +A preview browser session belongs to the desktop host that owns its Electron webview, cookies, and +page state. Remote clients must not infer that reaching a T3 environment also makes arbitrary +development ports reachable. + +Remote preview review therefore uses the environment WebSocket as a control path: + +```text +mobile review request + ↓ +T3 server preview broker + ↓ +connected desktop preview host captures one frame + ↓ +compact PNG + visible interactive bounds + ↓ +mobile freezes and annotates the frame locally +``` + +This path works through direct, relay, and T3 Connect environment connections and does not expose a +development server. The server authorizes the capture as an orchestration operation and routes it to +the desktop host already assigned to that environment. + +Live mobile browsing is separate and client-rendered: + +```text +public URL ───────────────────────────────→ mobile WebView +desktop loopback + LAN/Tailscale endpoint → rewritten private host + same port +desktop loopback + T3 Connect endpoint ───→ authenticated whole-origin gateway + ↓ + exact loopback origin +``` + +The mobile WebView, not the T3 server, owns page rendering, navigation, cookies, storage, DOM +inspection, and screenshot capture. LAN and Tailscale remain direct connections. T3 Connect exposes +only the T3 server origin, so an operate-scoped RPC validates the selected server-owned preview tab +and issues a one-use, 30-second bootstrap ticket. Consuming it creates a short-lived, HTTP-only +gateway lease for that exact loopback origin. + +Gateway routing is whole-origin within an isolated iOS WebView session. This is intentional: a +path-prefixed reverse proxy breaks root-relative assets, redirects, service-worker scope, and +development-server WebSockets. The gateway strips environment authorization, DPoP, forwarding, +Cloudflare, and hop-by-hop headers; filters T3 session cookies in both directions; does not follow +upstream redirects; and revokes leases when the authenticated client is removed, its parent auth +session expires, the preview tab closes, the lease expires, or the server restarts. Active HTTP +streams and WebSockets terminate with the lease, and each WebSocket direction has bounded frame and +byte buffering. + +Connect Live mode is iOS-only until the Android client can provide a genuinely isolated browser data +store or on-device proxy. Android WebView's incognito mode clears a process-global cookie jar without +creating an isolated one, so Android retains direct LAN/Tailscale browsing and the snapshot fallback. + +The gateway deliberately targets one loopback origin. Fully qualified localhost URLs are not +rewritten and therefore still refer to the mobile device, even when they name the selected preview +port; preview applications should use same-origin relative URLs or expose additional services through +an explicit endpoint. + +The gateway is available on the normal Node-based server runtime used by `npx t3` and desktop. The +Bun HTTP upgrade API cannot select the WebSocket subprotocol requested by a browser, so direct Bun +launches report the Connect gateway as unavailable and retain Snapshot review rather than exposing a +partially working live path. + ### Desktop-managed SSH access SSH is an access and launch helper, not a separate environment type. `DesktopSshEnvironment` diff --git a/docs/user/mobile.md b/docs/user/mobile.md new file mode 100644 index 00000000000..65c422d3f2b --- /dev/null +++ b/docs/user/mobile.md @@ -0,0 +1,103 @@ +# Mobile + +T3 Code Mobile connects to the same environments and threads as the web and +desktop clients. Image attachments use the normal composer and provider +pipeline, including when a message waits in the offline outbox. + +## Mark up an image attachment + +Image markup works on attachments in an existing thread, a new task, or a +review comment: + +1. Attach or paste an image. +2. Tap the pencil action on its thumbnail. +3. Draw with Pencil, a finger, or a pointer, or add a numbered point or region. +4. Add a comment to each numbered callout and tap **Done**. + +The composer replaces the attachment with a flattened PNG. Reopen the pencil +action to edit its retained callouts and strokes, or use **Remove markup** to +restore the original image. Removing the attachment removes its annotation as +well. + +T3 sends the flattened PNG and a compact numbered callout summary to the +selected provider. The editable vectors stay in the local mobile draft and are +not added to model context. Exported images are downscaled when needed. The +retained original and flattened PNG share the normal 10 MB attachment storage +budget so markup does not double draft or outbox payloads. + +The first release uses the same cross-platform canvas on iOS and Android. +Apple Pencil works as a drawing pointer, but pressure-sensitive ink, native +Pencil eraser/selection, and PencilKit palm handling are follow-up work. T3 +forwards the annotated image through the existing provider pipeline; the +selected model must support image input until model capability advertising can +gate that choice in the composer. + +This screenshot-first flow does not require the mobile device to reach a +development server or share the desktop preview's browser session. DOM and +element metadata is included when annotations come from a controlled desktop +snapshot or the mobile Browser. React component and source metadata is +included only when the desktop preview host can provide it. + +## Use Browser on iPad + +Open a thread and choose **Browser** from its header actions. T3 shows the +thread and Browser side by side at equal widths on iPad. Browser has its own +tabs along the top plus back, forward, address, reload, and annotation controls. +Use the expand action to give Browser the full workspace, then use it again to +return to the equal split. The divider remains draggable while split. Choosing +**Browser** again closes the pane; choosing **Files** switches the trailing pane +without stacking one over the other. + +Use the **+** action after the tabs to create another thread-scoped browser tab +from the iPad. The new tab opens with the address field focused; enter a URL and +choose **Go** on the keyboard. The tab and its current address are synchronized +through the environment, so other connected T3 clients can reopen it. + +Browser is an iPad-owned WKWebView, not a stream of the desktop window. Public +URLs load directly. On iPad, a desktop-local URL uses a one-use bootstrap URL +for an authenticated, short-lived gateway to the exact loopback browser tab. +The same path works when the iPad reaches its environment over LAN, Tailscale, +or T3 Connect, so the development server can remain bound to desktop loopback +and its port does not need to be exposed separately. + +Choose the pencil action to freeze the current WKWebView, sample bounded DOM +metadata, and open that exact frame in the markup canvas. Interactive elements +can be tapped to create semantic callouts alongside points, regions, and +freehand marks. Completing the markup adds the flattened PNG and numbered +callout context to the thread composer; sending remains an explicit action. + +The photo action is a review fallback rather than another browsing mode. It +asks a connected T3 desktop host to freeze the selected browser tab and sends +the PNG and compact interactive-element bounds through the existing +environment connection. This works over LAN, Tailscale, and T3 Connect without +exposing the development port to the iPad. Use the globe action to return to +the local Browser. + +The live browser owns its own cookies, storage, navigation history, and page +state; the preview gateway transports its requests but does not render or +continuously stream screenshots from the server. The bootstrap token is +one-use, the resulting cookie is HTTP-only and short-lived, and environment +authorization cookies are never forwarded to the development server. + +Gateway browsing is currently iOS/iPad-only. React Native WebView does not +provide an isolated Android cookie and storage container, so Android keeps +direct LAN/Tailscale browsing and the cross-platform desktop snapshot fallback. +For Android direct browsing, the development server must listen on the network +interface and its port must be allowed through the host firewall. + +Desktop snapshots are immutable review frames. If the desktop tab navigates, +resizes, or refreshes after capture, the iPad marks the frame as stale instead +of moving existing annotations. Refresh the snapshot to review the new frame. +The desktop app must be connected to the same environment and hosting the +selected browser tab for capture; Browser does not need to be visibly open on +the desktop. + +The preview gateway preserves same-origin and root-relative HTTP and WebSocket +traffic for the selected loopback origin. A page that hard-codes any fully +qualified `localhost` URL still points at the iPad's own loopback; use relative +URLs, or expose an additional service through a separately reachable endpoint. + +Gateway Browser requires the normal Node-based T3 server runtime used by +`npx t3` and the desktop app. A server launched directly under Bun cannot +complete WebSocket subprotocol negotiation for this gateway and reports +Browser as unavailable; desktop snapshot review remains available. diff --git a/packages/client-runtime/README.md b/packages/client-runtime/README.md index 722d6f6d389..12f9e097166 100644 --- a/packages/client-runtime/README.md +++ b/packages/client-runtime/README.md @@ -8,6 +8,7 @@ subpath. The package intentionally has no root export. | Subpath | Responsibility | | --------------------- | ---------------------------------------------------------------- | | `authorization` | Bearer and DPoP authorization plus token persistence contracts | +| `annotations` | Portable annotation prompt formatting and transcript parsing | | `connection` | Targets, catalog, supervision, retries, registry, and onboarding | | `environment` | Environment identity, descriptors, endpoints, and scoped keys | | `errors` | Shared client error inspection | diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 0b7b078a522..578037d655e 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -11,6 +11,10 @@ "types": "./src/authorization/index.ts", "default": "./src/authorization/index.ts" }, + "./annotations": { + "types": "./src/annotations/index.ts", + "default": "./src/annotations/index.ts" + }, "./environment": { "types": "./src/environment/index.ts", "default": "./src/environment/index.ts" diff --git a/packages/client-runtime/src/annotations/elementContext.ts b/packages/client-runtime/src/annotations/elementContext.ts new file mode 100644 index 00000000000..2718ce649b8 --- /dev/null +++ b/packages/client-runtime/src/annotations/elementContext.ts @@ -0,0 +1,140 @@ +import type { PickedElementPayload, PickedElementStackFrame } from "@t3tools/contracts"; + +const ELEMENT_CONTEXT_HTML_PREVIEW_LIMIT = 4000; +const ELEMENT_CONTEXT_STYLES_LIMIT = 4000; +const ELEMENT_CONTEXT_LABEL_TAG_MAX = 24; +const ELEMENT_CONTEXT_BLOCK_MAX_CHARS = 48_000; + +interface ElementContextSelection { + pageUrl: string; + pageTitle: string | null; + tagName: string; + selector: string | null; + htmlPreview: string; + componentName: string | null; + source: PickedElementStackFrame | null; + styles: string; +} + +function truncateString(value: string, limit: number): string { + if (value.length <= limit) return value; + return `${value.slice(0, Math.max(0, limit - 1))}…`; +} + +function normalizeText(value: string): string { + return value.replace(/\r\n/g, "\n").replace(/^\n+|\n+$/g, ""); +} + +export function normalizeElementContextSelection( + raw: PickedElementPayload, +): ElementContextSelection | null { + const pageUrl = raw.pageUrl.trim(); + const tagName = raw.tagName.trim().toLowerCase(); + if (pageUrl.length === 0 || tagName.length === 0) { + return null; + } + const stackFrame = raw.source ?? raw.stack[0] ?? null; + return { + pageUrl, + pageTitle: raw.pageTitle?.trim() ?? null, + tagName, + selector: raw.selector?.trim() || null, + htmlPreview: truncateString(normalizeText(raw.htmlPreview), ELEMENT_CONTEXT_HTML_PREVIEW_LIMIT), + componentName: raw.componentName?.trim() || null, + source: stackFrame + ? { + functionName: stackFrame.functionName?.trim() || null, + fileName: stackFrame.fileName?.trim() || null, + lineNumber: stackFrame.lineNumber ?? null, + columnNumber: stackFrame.columnNumber ?? null, + } + : null, + styles: truncateString(normalizeText(raw.styles), ELEMENT_CONTEXT_STYLES_LIMIT), + }; +} + +function shortenTagLabel(tagName: string): string { + if (tagName.length <= ELEMENT_CONTEXT_LABEL_TAG_MAX) return tagName; + return `${tagName.slice(0, ELEMENT_CONTEXT_LABEL_TAG_MAX - 1)}…`; +} + +function formatElementContextLabel(context: ElementContextSelection): string { + if (context.componentName) return `<${context.componentName}>`; + return `<${shortenTagLabel(context.tagName)}>`; +} + +function basenameFromPath(filePath: string): string { + const parts = filePath.split(/[\\/]/); + return parts[parts.length - 1] ?? filePath; +} + +function formatElementContextSourceLabel(context: ElementContextSelection): string | null { + const source = context.source; + if (!source?.fileName) return null; + const base = basenameFromPath(source.fileName); + if (source.lineNumber == null) return base; + return `${base}:${source.lineNumber}`; +} + +function buildContextHeader(context: ElementContextSelection): string { + const label = formatElementContextLabel(context); + const source = formatElementContextSourceLabel(context); + return source ? `${label} (${source})` : label; +} + +function indentLines(value: string): string[] { + return value.split("\n").map((line) => ` ${line}`); +} + +function buildSingleContextLines(context: ElementContextSelection): string[] { + const lines: string[] = []; + lines.push(`- ${buildContextHeader(context)}:`); + if (context.pageUrl.length > 0) { + lines.push(` url: ${context.pageUrl}`); + } + if (context.selector) { + lines.push(` selector: ${context.selector}`); + } + if (context.source?.fileName) { + const { fileName, lineNumber, columnNumber } = context.source; + const location = + lineNumber != null + ? `${fileName}:${lineNumber}${columnNumber != null ? `:${columnNumber}` : ""}` + : fileName; + lines.push(` source: ${location}`); + } + const html = context.htmlPreview.trim(); + if (html.length > 0) { + lines.push(" html:"); + lines.push(...indentLines(html)); + } + const styles = context.styles.trim(); + if (styles.length > 0) { + lines.push(" styles:"); + lines.push(...indentLines(styles)); + } + return lines; +} + +export function buildElementContextBlock(contexts: ReadonlyArray): string { + if (contexts.length === 0) return ""; + const lines = [""]; + for (let index = 0; index < contexts.length; index += 1) { + const context = contexts[index]!; + const contextLines = buildSingleContextLines(context); + const separator = lines.length > 1 ? [""] : []; + const closingLine = ""; + const candidate = [...lines, ...separator, ...contextLines, closingLine].join("\n"); + if (candidate.length > ELEMENT_CONTEXT_BLOCK_MAX_CHARS) { + const omitted = contexts.length - index; + lines.push( + "", + `- … ${omitted} additional element context${omitted === 1 ? "" : "s"} omitted to fit the annotation context budget.`, + ); + break; + } + lines.push(...separator, ...contextLines); + } + lines.push(""); + return lines.join("\n"); +} diff --git a/packages/client-runtime/src/annotations/index.ts b/packages/client-runtime/src/annotations/index.ts new file mode 100644 index 00000000000..b8c29d13e5a --- /dev/null +++ b/packages/client-runtime/src/annotations/index.ts @@ -0,0 +1,11 @@ +export { + appendPreviewAnnotationPrompt, + buildPreviewAnnotationCopyText, + buildPreviewAnnotationPrompt, + extractTrailingPreviewAnnotation, + extractTrailingPreviewAnnotations, + type ExtractedPreviewAnnotation, + type ExtractedPreviewAnnotations, + type ParsedPreviewAnnotation, + type ParsedPreviewAnnotationCallout, +} from "./previewAnnotation.ts"; diff --git a/packages/client-runtime/src/annotations/previewAnnotation.test.ts b/packages/client-runtime/src/annotations/previewAnnotation.test.ts new file mode 100644 index 00000000000..2b142b2a5ce --- /dev/null +++ b/packages/client-runtime/src/annotations/previewAnnotation.test.ts @@ -0,0 +1,353 @@ +import type { PreviewAnnotationPayload } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + appendPreviewAnnotationPrompt, + buildPreviewAnnotationCopyText, + buildPreviewAnnotationPrompt, + extractTrailingPreviewAnnotation, + extractTrailingPreviewAnnotations, +} from "./previewAnnotation.ts"; + +const legacyAnnotation: PreviewAnnotationPayload = { + id: "annotation_1", + pageUrl: "http://localhost:3000", + pageTitle: "Example", + comment: "Make these cards feel related.", + elements: [], + regions: [{ id: "region_1", rect: { x: 10, y: 20, width: 100, height: 80 } }], + strokes: [ + { + id: "stroke_1", + color: "#7c3aed", + width: 4, + points: [ + { x: 10, y: 10 }, + { x: 20, y: 20 }, + ], + bounds: { x: 6, y: 6, width: 18, height: 18 }, + }, + ], + styleChanges: [ + { + targetId: "element_1", + selector: ".card", + property: "border-radius", + previousValue: "4px", + value: "16px", + }, + ], + screenshot: { + dataUrl: "data:image/png;base64,AA==", + width: 100, + height: 80, + cropRect: { x: 10, y: 20, width: 100, height: 80 }, + }, + createdAt: "2026-06-11T00:00:00.000Z", +}; + +const element = { + id: "element_1", + rect: { x: 20, y: 10, width: 100, height: 40 }, + element: { + pageUrl: "http://localhost:3000", + pageTitle: "Example", + tagName: "button", + selector: "button.submit", + htmlPreview: '', + componentName: "SubmitButton", + source: { + functionName: "SubmitButton", + fileName: "/repo/src/SubmitButton.tsx", + lineNumber: 12, + columnNumber: 5, + }, + stack: [], + styles: ".submit { color: white; }", + pickedAt: "2026-06-11T00:00:00.000Z", + }, +} as const; + +const calloutAnnotation: PreviewAnnotationPayload = { + ...legacyAnnotation, + id: "annotation_callouts", + pageTitle: null, + comment: "", + source: { kind: "image", name: "checkout.png" }, + elements: [element], + regions: [], + strokes: [], + styleChanges: [], + callouts: [ + { + id: "callout_3", + number: 3, + comment: "Use the primary style.", + anchor: { + kind: "element", + targetId: "element_1", + rect: { x: 0.1, y: 0.7, width: 0.4, height: 0.2 }, + }, + }, + { + id: "callout_1", + number: 1, + comment: "Increase contrast.\nKeep the label short.", + anchor: { kind: "point", point: { x: 0.42, y: 0.18 } }, + }, + { + id: "callout_2", + number: 2, + comment: "Do not emit from a comment.", + anchor: { + kind: "region", + rect: { x: 0.4214, y: 0.1814, width: 0.3114, height: 0.1214 }, + }, + }, + ], + editable: { + version: 1, + coordinateSpace: "normalized", + strokes: [ + { + id: "private_stroke", + color: "#ff00ff", + width: 0.00987654, + points: [ + { x: 0.987654, y: 0.876543 }, + { x: 0.765432, y: 0.654321 }, + ], + bounds: { x: 0.6, y: 0.6, width: 0.39, height: 0.39 }, + }, + ], + }, +}; + +describe("preview annotation prompts", () => { + it("preserves the exact legacy prompt representation when callouts are absent", () => { + expect(buildPreviewAnnotationPrompt(legacyAnnotation)).toBe( + [ + "", + "Preview annotation:", + "Id: annotation_1", + "Page: Example", + "Comment: Make these cards feel related.", + "Targets: 1 marked region, 1 drawing.", + "Requested visual changes:", + "- border-radius: 4px → 16px", + "The attached screenshot is the annotated preview crop.", + "", + ].join("\n"), + ); + }); + + it("caps large multi-element semantic context at a compact shared budget", () => { + const result = buildPreviewAnnotationPrompt({ + ...legacyAnnotation, + elements: Array.from({ length: 20 }, (_, index) => ({ + ...element, + id: `element_${index}`, + element: { + ...element.element, + selector: `button.item-${index}`, + htmlPreview: ``, + styles: `.item-${index} { content: "${"y".repeat(4_000)}"; }`, + }, + })), + }); + + expect(result.length).toBeLessThan(60_000); + expect(result).toContain("additional element contexts omitted"); + expect(result).toContain(""); + expect(result).toMatch(/<\/element_context>\n<\/preview_annotation>$/); + }); + + it("describes portable image source and drawings without serializing their points", () => { + const result = buildPreviewAnnotationPrompt({ + ...legacyAnnotation, + source: { kind: "image", name: "drawing-only.png" }, + regions: [], + strokes: [], + editable: calloutAnnotation.editable!, + }); + expect(result).toContain("Page: drawing-only.png"); + expect(result).toContain("Targets: 1 drawing."); + expect(result).toContain("The attached screenshot is the annotated image."); + expect(result).not.toContain("private_stroke"); + expect(result).not.toContain("0.987654"); + }); + + it("formats sorted point, region, and semantic element callouts compactly", () => { + const result = buildPreviewAnnotationPrompt(calloutAnnotation); + expect(result).toContain("Page: checkout.png"); + expect(result).toContain("Targets: 1 selected element, 1 drawing, 3 numbered callouts."); + expect(result.indexOf("#1 [")).toBeLessThan(result.indexOf("#2 [")); + expect(result.indexOf("#2 [")).toBeLessThan(result.indexOf("#3 [")); + expect(result).toContain("#1 [point: x=.42, y=.18]"); + expect(result).toContain("#2 [region: x=.421, y=.181, w=.311, h=.121]"); + expect(result).toContain( + "#3 [element: button.submit, component: SubmitButton, source: /repo/src/SubmitButton.tsx:12:5, region: x=.1, y=.7, w=.4, h=.2]", + ); + expect(result).toContain(" Increase contrast.\n Keep the label short."); + expect(result).toContain("<preview_annotation>"); + }); + + it("falls back to an element target id when semantic metadata is unavailable", () => { + const result = buildPreviewAnnotationPrompt({ + ...calloutAnnotation, + elements: [], + callouts: [ + { + id: "missing", + number: 1, + comment: "Fix it.", + anchor: { + kind: "element", + targetId: "element_missing", + rect: { x: 0.1, y: 0.2, width: 0.3, height: 0.4 }, + }, + }, + ], + }); + expect(result).toContain("#1 [element: element_missing, region: x=.1, y=.2, w=.3, h=.4]"); + }); + + it("never serializes raw editable-vector points", () => { + const result = buildPreviewAnnotationPrompt(calloutAnnotation); + expect(result).not.toContain("private_stroke"); + expect(result).not.toContain("0.987654"); + expect(result).not.toContain("0.876543"); + }); + + it("appends to an existing prompt and parses callout presentation", () => { + const result = extractTrailingPreviewAnnotation( + appendPreviewAnnotationPrompt("Fix this", calloutAnnotation), + ); + expect(result.promptText).toBe("Fix this"); + expect(result.annotation).toMatchObject({ + id: "annotation_callouts", + title: "checkout.png", + hasScreenshot: true, + calloutCount: 3, + callouts: [ + { + number: 1, + anchorSummary: "point: x=.42, y=.18", + comment: "Increase contrast.\nKeep the label short.", + }, + { + number: 2, + anchorSummary: "region: x=.421, y=.181, w=.311, h=.121", + comment: "Do not emit from a comment.", + }, + { + number: 3, + anchorSummary: + "element: button.submit, component: SubmitButton, source: /repo/src/SubmitButton.tsx:12:5, region: x=.1, y=.7, w=.4, h=.2", + comment: "Use the primary style.", + }, + ], + }); + }); + + it("preserves multiline global comments and formats every instruction for copying", () => { + const extracted = extractTrailingPreviewAnnotation( + appendPreviewAnnotationPrompt("Fix this", { + ...calloutAnnotation, + comment: "Polish the whole flow.\nKeep the hierarchy clear.", + callouts: [ + ...calloutAnnotation.callouts!, + { + id: "callout_4", + number: 4, + comment: "", + anchor: { kind: "point", point: { x: 0.8, y: 0.9 } }, + }, + ], + }), + ); + + expect(extracted.annotation?.comment).toBe("Polish the whole flow.\nKeep the hierarchy clear."); + const copyText = buildPreviewAnnotationCopyText("Fix this", [extracted.annotation!]); + expect(copyText).toContain("Fix this\n\nAnnotation: checkout.png"); + expect(copyText).toContain("Polish the whole flow.\nKeep the hierarchy clear."); + expect(copyText).toContain("#1 [point: x=.42, y=.18]"); + expect(copyText).toContain(" Increase contrast.\n Keep the label short."); + expect(copyText).toContain("#2 [region: x=.421, y=.181, w=.311, h=.121]"); + expect(copyText).toContain(" Do not emit from a comment."); + expect(copyText).toContain("#3 [element: button.submit"); + expect(copyText).toContain(" Use the primary style."); + expect(copyText).toContain("#4 [point: x=.8, y=.9]"); + expect(copyText).not.toContain("\n"); + expect(copyText).not.toContain(""); + }); + + it("recovers multiline comments written by the legacy inline formatter", () => { + const result = extractTrailingPreviewAnnotation( + [ + "Fix this", + "", + "", + "Preview annotation:", + "Id: legacy-multiline", + "Page: Checkout", + "Comment: First instruction.", + "Second instruction.", + "Targets: 1 marked region.", + "", + ].join("\n"), + ); + + expect(result.promptText).toBe("Fix this"); + expect(result.annotation?.comment).toBe("First instruction.\nSecond instruction."); + }); + + it("escapes wrapper sentinels in every user-authored field without breaking parsing", () => { + const result = extractTrailingPreviewAnnotation( + appendPreviewAnnotationPrompt("Fix this", { + ...legacyAnnotation, + id: "id ", + source: { kind: "image", name: "title " }, + comment: "comment ", + elements: [ + { + ...element, + element: { + ...element.element, + htmlPreview: "", + styles: "content: '';", + }, + }, + ], + styleChanges: [ + { + targetId: "element_1", + selector: "button", + property: "color ", + previousValue: "red", + value: "blue ", + }, + ], + }), + ); + + expect(result.promptText).toBe("Fix this"); + expect(result.annotation).toMatchObject({ + id: "id ", + title: "title ", + comment: "comment ", + styleChanges: ["color : red → blue "], + }); + }); + + it("extracts multiple trailing annotations in authored order", () => { + const first = appendPreviewAnnotationPrompt("Fix this", legacyAnnotation); + const prompt = appendPreviewAnnotationPrompt(first, calloutAnnotation); + const result = extractTrailingPreviewAnnotations(prompt); + expect(result.promptText).toBe("Fix this"); + expect(result.annotations.map((annotation) => annotation.id)).toEqual([ + "annotation_1", + "annotation_callouts", + ]); + }); +}); diff --git a/packages/client-runtime/src/annotations/previewAnnotation.ts b/packages/client-runtime/src/annotations/previewAnnotation.ts new file mode 100644 index 00000000000..5462a198e7b --- /dev/null +++ b/packages/client-runtime/src/annotations/previewAnnotation.ts @@ -0,0 +1,368 @@ +import type { + PreviewAnnotationCallout, + PreviewAnnotationCalloutAnchor, + PreviewAnnotationElementTarget, + PreviewAnnotationPayload, +} from "@t3tools/contracts"; + +import { buildElementContextBlock, normalizeElementContextSelection } from "./elementContext.ts"; + +const TRAILING_PREVIEW_ANNOTATION_BLOCK_PATTERN = + /\n*\n((?:(?!)[\s\S])*)\n<\/preview_annotation>\s*$/; +const PREVIEW_SCREENSHOT_LINE = "The attached screenshot is the annotated preview crop."; +const IMAGE_SCREENSHOT_LINE = "The attached screenshot is the annotated image."; + +export interface ParsedPreviewAnnotationCallout { + number: number; + anchorSummary: string; + comment: string; +} + +export interface ParsedPreviewAnnotation { + id: string; + title: string; + comment: string; + targetSummary: string; + styleChanges: string[]; + hasScreenshot: boolean; + calloutCount: number; + callouts: ParsedPreviewAnnotationCallout[]; +} + +export interface ExtractedPreviewAnnotation { + promptText: string; + annotation: ParsedPreviewAnnotation | null; +} + +export interface ExtractedPreviewAnnotations { + promptText: string; + annotations: ParsedPreviewAnnotation[]; +} + +export function buildPreviewAnnotationCopyText( + visibleText: string, + annotations: ReadonlyArray, +): string { + const sections = annotations.map((annotation) => { + const lines = [`Annotation: ${annotation.title}`]; + const comment = annotation.comment.trim(); + if (comment) { + lines.push(comment); + } + if (annotation.targetSummary) { + lines.push(`Targets: ${annotation.targetSummary}`); + } + if (annotation.callouts.length > 0) { + lines.push("Callouts:"); + for (const callout of annotation.callouts) { + lines.push(`#${callout.number} [${callout.anchorSummary}]`); + const instruction = callout.comment.trim(); + if (instruction) { + lines.push(...instruction.split("\n").map((line) => ` ${line}`)); + } + } + } + if (annotation.styleChanges.length > 0) { + lines.push("Requested visual changes:"); + lines.push(...annotation.styleChanges.map((change) => `- ${change}`)); + } + return lines.join("\n"); + }); + const promptText = visibleText.trim(); + return [promptText, ...sections].filter((section) => section.length > 0).join("\n\n"); +} + +function compactCoordinate(value: number): string { + const fixed = value.toFixed(3).replace(/\.?0+$/, ""); + if (fixed.startsWith("0.")) return fixed.slice(1); + if (fixed.startsWith("-0.")) return `-${fixed.slice(2)}`; + return fixed; +} + +function formatRect(rect: { x: number; y: number; width: number; height: number }): string { + return [ + `x=${compactCoordinate(rect.x)}`, + `y=${compactCoordinate(rect.y)}`, + `w=${compactCoordinate(rect.width)}`, + `h=${compactCoordinate(rect.height)}`, + ].join(", "); +} + +function formatElementSource(target: PreviewAnnotationElementTarget): string | null { + const source = target.element.source ?? target.element.stack[0] ?? null; + if (!source?.fileName) return null; + if (source.lineNumber == null) return source.fileName; + return `${source.fileName}:${source.lineNumber}${ + source.columnNumber == null ? "" : `:${source.columnNumber}` + }`; +} + +function formatElementAnchor( + anchor: Extract, + elements: ReadonlyArray, +): string { + const target = elements.find((entry) => entry.id === anchor.targetId); + if (!target) { + return `element: ${anchor.targetId}, region: ${formatRect(anchor.rect)}`; + } + const elementLabel = + target.element.selector?.trim() || target.element.tagName.trim() || anchor.targetId; + const parts = [`element: ${elementLabel}`]; + const componentName = target.element.componentName?.trim(); + if (componentName) parts.push(`component: ${componentName}`); + const source = formatElementSource(target); + if (source) parts.push(`source: ${source}`); + parts.push(`region: ${formatRect(anchor.rect)}`); + return parts.join(", "); +} + +function formatCalloutAnchor( + anchor: PreviewAnnotationCalloutAnchor, + elements: ReadonlyArray, +): string { + switch (anchor.kind) { + case "point": + return `point: x=${compactCoordinate(anchor.point.x)}, y=${compactCoordinate(anchor.point.y)}`; + case "region": + return `region: ${formatRect(anchor.rect)}`; + case "element": + return formatElementAnchor(anchor, elements); + } +} + +function escapeAnnotationSentinels(value: string): string { + return value + .replace(/\r\n/g, "\n") + .replaceAll("", "<preview_annotation>") + .replaceAll("", "</preview_annotation>"); +} + +function unescapeAnnotationSentinels(value: string): string { + return value + .replaceAll("<preview_annotation>", "") + .replaceAll("</preview_annotation>", ""); +} + +function appendCalloutLines( + lines: string[], + callouts: ReadonlyArray, + elements: ReadonlyArray, +): void { + if (callouts.length === 0) return; + lines.push("Callouts:"); + const ordered = [...callouts].sort( + (left, right) => left.number - right.number || left.id.localeCompare(right.id), + ); + for (const callout of ordered) { + lines.push(`#${callout.number} [${formatCalloutAnchor(callout.anchor, elements)}]`); + const comment = escapeAnnotationSentinels(callout.comment).trim(); + if (!comment) continue; + for (const commentLine of comment.split("\n")) { + lines.push(` ${commentLine}`); + } + } +} + +function annotationTitle(annotation: PreviewAnnotationPayload): string { + if (annotation.source?.kind === "image") { + return annotation.source.name?.trim() || annotation.pageTitle?.trim() || "Image"; + } + if (annotation.source?.kind === "preview") { + return ( + annotation.source.title?.trim() || + annotation.source.url.trim() || + annotation.pageTitle?.trim() || + annotation.pageUrl.trim() || + "Preview" + ); + } + return annotation.pageTitle?.trim() || annotation.pageUrl.trim() || "Preview"; +} + +export function buildPreviewAnnotationPrompt(annotation: PreviewAnnotationPayload): string { + const lines = ["Preview annotation:"]; + lines.push(`Id: ${annotation.id}`); + lines.push(`Page: ${annotationTitle(annotation)}`); + const comment = annotation.comment.replace(/\r\n/g, "\n").trim(); + if (comment) { + if (comment.includes("\n")) { + lines.push("Comment:", ...comment.split("\n").map((line) => ` ${line}`)); + } else { + lines.push(`Comment: ${comment}`); + } + } + const targets: string[] = []; + if (annotation.elements.length > 0) { + targets.push( + `${annotation.elements.length} selected element${annotation.elements.length === 1 ? "" : "s"}`, + ); + } + if (annotation.regions.length > 0) { + targets.push( + `${annotation.regions.length} marked region${annotation.regions.length === 1 ? "" : "s"}`, + ); + } + const drawingCount = + annotation.strokes.length > 0 + ? annotation.strokes.length + : (annotation.editable?.strokes.length ?? 0); + if (drawingCount > 0) { + targets.push(`${drawingCount} drawing${drawingCount === 1 ? "" : "s"}`); + } + const calloutCount = annotation.callouts?.length ?? 0; + if (calloutCount > 0) { + targets.push(`${calloutCount} numbered callout${calloutCount === 1 ? "" : "s"}`); + } + if (targets.length > 0) lines.push(`Targets: ${targets.join(", ")}.`); + appendCalloutLines(lines, annotation.callouts ?? [], annotation.elements); + if (annotation.styleChanges.length > 0) { + lines.push("Requested visual changes:"); + for (const change of annotation.styleChanges) { + lines.push(`- ${change.property}: ${change.previousValue || "(unset)"} → ${change.value}`); + } + } + if (annotation.screenshot) { + lines.push( + annotation.source?.kind === "image" ? IMAGE_SCREENSHOT_LINE : PREVIEW_SCREENSHOT_LINE, + ); + } + const elementContexts = annotation.elements + .map((target) => normalizeElementContextSelection(target.element)) + .filter((context) => context !== null); + const elementBlock = buildElementContextBlock(elementContexts); + if (elementBlock) lines.push(elementBlock); + return [ + "", + ...lines.map(escapeAnnotationSentinels), + "", + ].join("\n"); +} + +export function appendPreviewAnnotationPrompt( + prompt: string, + annotation: PreviewAnnotationPayload, +): string { + const annotationText = buildPreviewAnnotationPrompt(annotation); + const trimmed = prompt.trim(); + return trimmed ? `${trimmed}\n\n${annotationText}` : annotationText; +} + +function parseCallouts(lines: ReadonlyArray): ParsedPreviewAnnotationCallout[] { + const headingIndex = lines.indexOf("Callouts:"); + if (headingIndex < 0) return []; + const callouts: ParsedPreviewAnnotationCallout[] = []; + let current: ParsedPreviewAnnotationCallout | null = null; + const commit = () => { + if (!current) return; + callouts.push({ + ...current, + comment: unescapeAnnotationSentinels(current.comment.trimEnd()), + }); + current = null; + }; + for (const line of lines.slice(headingIndex + 1)) { + const header = /^#(\d+) \[(.*)\]$/.exec(line); + if (header) { + commit(); + current = { + number: Number(header[1]), + anchorSummary: header[2] ?? "", + comment: "", + }; + continue; + } + if (current && line.startsWith(" ")) { + current.comment += `${current.comment ? "\n" : ""}${line.slice(2)}`; + continue; + } + if (current) break; + if ( + line === "Requested visual changes:" || + line === PREVIEW_SCREENSHOT_LINE || + line === IMAGE_SCREENSHOT_LINE || + line === "" + ) { + break; + } + } + commit(); + return callouts; +} + +function parseAnnotationComment(lines: ReadonlyArray): string { + const inlineCommentIndex = lines.findIndex((line) => line.startsWith("Comment: ")); + if (inlineCommentIndex >= 0) { + const commentLines = [lines[inlineCommentIndex]!.slice("Comment: ".length)]; + for (const line of lines.slice(inlineCommentIndex + 1)) { + if ( + line.startsWith("Targets: ") || + line === "Callouts:" || + line === "Requested visual changes:" || + line === PREVIEW_SCREENSHOT_LINE || + line === IMAGE_SCREENSHOT_LINE || + line === "" + ) { + break; + } + commentLines.push(line); + } + return unescapeAnnotationSentinels(commentLines.join("\n").trim()); + } + const headingIndex = lines.indexOf("Comment:"); + if (headingIndex < 0) return ""; + const commentLines: string[] = []; + for (const line of lines.slice(headingIndex + 1)) { + if (!line.startsWith(" ")) break; + commentLines.push(line.slice(2)); + } + return unescapeAnnotationSentinels(commentLines.join("\n").trim()); +} + +export function extractTrailingPreviewAnnotation(prompt: string): ExtractedPreviewAnnotation { + const match = TRAILING_PREVIEW_ANNOTATION_BLOCK_PATTERN.exec(prompt); + if (!match) return { promptText: prompt, annotation: null }; + const body = match[1] ?? ""; + const lines = body.split("\n"); + const pageLine = lines.find((line) => line.startsWith("Page: ")); + const idLine = lines.find((line) => line.startsWith("Id: ")); + const targetsLine = lines.find((line) => line.startsWith("Targets: ")); + const styleHeadingIndex = lines.indexOf("Requested visual changes:"); + const linesAfterStyleHeading = lines.slice(styleHeadingIndex + 1); + const elementContextIndex = linesAfterStyleHeading.indexOf(""); + const styleChanges = + styleHeadingIndex < 0 + ? [] + : linesAfterStyleHeading + .slice(0, elementContextIndex < 0 ? undefined : elementContextIndex) + .filter((line) => line.startsWith("- ")) + .map((line) => line.slice(2)); + const callouts = parseCallouts(lines); + return { + promptText: prompt.slice(0, match.index).replace(/\n+$/, ""), + annotation: { + id: + unescapeAnnotationSentinels(idLine?.slice("Id: ".length).trim() ?? "") || `${match.index}`, + title: + unescapeAnnotationSentinels(pageLine?.slice("Page: ".length).trim() ?? "") || + "Preview annotation", + comment: parseAnnotationComment(lines), + targetSummary: targetsLine?.slice("Targets: ".length).trim() || "", + styleChanges: styleChanges.map(unescapeAnnotationSentinels), + hasScreenshot: body.includes(PREVIEW_SCREENSHOT_LINE) || body.includes(IMAGE_SCREENSHOT_LINE), + calloutCount: callouts.length, + callouts, + }, + }; +} + +export function extractTrailingPreviewAnnotations(prompt: string): ExtractedPreviewAnnotations { + const annotations: ParsedPreviewAnnotation[] = []; + let promptText = prompt; + while (true) { + const extracted = extractTrailingPreviewAnnotation(promptText); + if (!extracted.annotation) break; + annotations.unshift(extracted.annotation); + promptText = extracted.promptText; + } + return { promptText, annotations }; +} diff --git a/packages/client-runtime/src/state/preview.test.ts b/packages/client-runtime/src/state/preview.test.ts index 7a83bccde98..c4701e01f6f 100644 --- a/packages/client-runtime/src/state/preview.test.ts +++ b/packages/client-runtime/src/state/preview.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vite-plus/test"; -import { previewAutomationHostFocusConcurrencyKey } from "./preview.ts"; +import { + previewAutomationHostFocusConcurrencyKey, + previewLiveGatewayConcurrencyKey, + previewReviewSnapshotConcurrencyKey, +} from "./preview.ts"; describe("preview state commands", () => { it("keeps focus updates from replacement host connections independent", () => { @@ -15,4 +19,45 @@ describe("preview state commands", () => { expect(first).not.toBe(replacement); }); + + it("deduplicates review captures only for the same environment, thread, and tab", () => { + const first = previewReviewSnapshotConcurrencyKey({ + environmentId: "environment-1", + input: { threadId: "thread-1", tabId: "tab-1" }, + }); + const same = previewReviewSnapshotConcurrencyKey({ + environmentId: "environment-1", + input: { threadId: "thread-1", tabId: "tab-1" }, + }); + const otherTab = previewReviewSnapshotConcurrencyKey({ + environmentId: "environment-1", + input: { threadId: "thread-1", tabId: "tab-2" }, + }); + const otherEnvironment = previewReviewSnapshotConcurrencyKey({ + environmentId: "environment-2", + input: { threadId: "thread-1", tabId: "tab-1" }, + }); + + expect(first).toBe(same); + expect(first).not.toBe(otherTab); + expect(first).not.toBe(otherEnvironment); + }); + + it("deduplicates live-gateway opens only for the same environment, thread, and tab", () => { + const first = previewLiveGatewayConcurrencyKey({ + environmentId: "environment-1", + input: { threadId: "thread-1", tabId: "tab-1" }, + }); + const same = previewLiveGatewayConcurrencyKey({ + environmentId: "environment-1", + input: { threadId: "thread-1", tabId: "tab-1" }, + }); + const otherTab = previewLiveGatewayConcurrencyKey({ + environmentId: "environment-1", + input: { threadId: "thread-1", tabId: "tab-2" }, + }); + + expect(first).toBe(same); + expect(first).not.toBe(otherTab); + }); }); diff --git a/packages/client-runtime/src/state/preview.ts b/packages/client-runtime/src/state/preview.ts index f9469ee96a5..95b543aa4bd 100644 --- a/packages/client-runtime/src/state/preview.ts +++ b/packages/client-runtime/src/state/preview.ts @@ -17,12 +17,30 @@ export const previewAutomationHostFocusConcurrencyKey = (value: { }; }): string => JSON.stringify([value.environmentId, value.input.clientId, value.input.connectionId]); +export const previewReviewSnapshotConcurrencyKey = (value: { + readonly environmentId: string; + readonly input: { + readonly threadId: string; + readonly tabId: string; + }; +}): string => JSON.stringify([value.environmentId, value.input.threadId, value.input.tabId]); + +export const previewLiveGatewayConcurrencyKey = (value: { + readonly environmentId: string; + readonly input: { + readonly threadId: string; + readonly tabId: string; + }; +}): string => JSON.stringify([value.environmentId, value.input.threadId, value.input.tabId]); + export function createPreviewEnvironmentAtoms( runtime: Atom.AtomRuntime, ) { const lifecycleScheduler = createAtomCommandScheduler(); const statusScheduler = createAtomCommandScheduler(); const automationScheduler = createAtomCommandScheduler(); + const reviewScheduler = createAtomCommandScheduler(); + const liveGatewayScheduler = createAtomCommandScheduler(); const lifecycleConcurrency = { mode: "serial" as const, key: ({ environmentId, input }: { environmentId: string; input: { threadId: string } }) => @@ -90,6 +108,24 @@ export function createPreviewEnvironmentAtoms( JSON.stringify([environmentId, input.threadId, input.tabId]), }, }), + reviewSnapshot: createEnvironmentRpcCommand(runtime, { + label: "environment-data:preview:review-snapshot", + tag: WS_METHODS.previewReviewSnapshot, + scheduler: reviewScheduler, + concurrency: { + mode: "singleFlight", + key: previewReviewSnapshotConcurrencyKey, + }, + }), + openLiveGateway: createEnvironmentRpcCommand(runtime, { + label: "environment-data:preview:open-live-gateway", + tag: WS_METHODS.previewOpenLiveGateway, + scheduler: liveGatewayScheduler, + concurrency: { + mode: "singleFlight", + key: previewLiveGatewayConcurrencyKey, + }, + }), respondToAutomation: createEnvironmentRpcCommand(runtime, { label: "environment-data:preview:automation-respond", tag: WS_METHODS.previewAutomationRespond, diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index f0ee1889177..3ccbf47637e 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -26,6 +26,7 @@ export * from "./filesystem.ts"; export * from "./assets.ts"; export * from "./review.ts"; export * from "./preview.ts"; +export * from "./previewAnnotation.ts"; export * from "./previewAutomation.ts"; export * from "./resourceTelemetry.ts"; export * from "./rpc.ts"; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index a92eb972d67..8466279aafe 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -95,6 +95,7 @@ import type { SourceControlRepositoryInfo, SourceControlRepositoryLookupInput, } from "./sourceControl.ts"; +import type { PreviewAnnotationPayload } from "./previewAnnotation.ts"; export interface ContextMenuItem { id: T; @@ -700,197 +701,6 @@ export const DesktopPreviewScreenshotArtifactSchema: Schema.Codec = Schema.Struct({ - functionName: Schema.NullOr(Schema.String), - fileName: Schema.NullOr(Schema.String), - lineNumber: Schema.NullOr(Schema.Number), - columnNumber: Schema.NullOr(Schema.Number), -}); - -/** - * A successful element pick from the preview webview. All fields are - * best-effort — pages that don't ship a React fiber tree (or aren't running - * in dev) will still produce a usable payload (selector + html preview), - * just without component / source attribution. - */ -export interface PickedElementPayload { - /** URL of the page the element was picked on. */ - pageUrl: string; - /** Optional `` of that page (best-effort). */ - pageTitle: string | null; - /** Lowercase tag name, e.g. `"button"`. */ - tagName: string; - /** CSS selector resolving back to the element on a re-render. */ - selector: string | null; - /** Truncated outer-HTML preview (matches react-grab's `htmlPreview`). */ - htmlPreview: string; - /** Nearest React component display name, or null when unavailable. */ - componentName: string | null; - /** First source-mapped stack frame (file + line of the JSX source). */ - source: PickedElementStackFrame | null; - /** Full owner-stack frames; can be empty. Useful for richer context. */ - stack: ReadonlyArray<PickedElementStackFrame>; - /** Author CSS only (UA defaults stripped) — react-grab's `styles`. */ - styles: string; - /** Wall-clock pick time as ISO-8601 string. */ - pickedAt: string; -} - -export const PickedElementPayloadSchema: Schema.Codec<PickedElementPayload> = Schema.Struct({ - pageUrl: Schema.String, - pageTitle: Schema.NullOr(Schema.String), - tagName: Schema.String, - selector: Schema.NullOr(Schema.String), - htmlPreview: Schema.String, - componentName: Schema.NullOr(Schema.String), - source: Schema.NullOr(PickedElementStackFrameSchema), - stack: Schema.Array(PickedElementStackFrameSchema), - styles: Schema.String, - pickedAt: Schema.String, -}); - -export interface PreviewAnnotationRect { - x: number; - y: number; - width: number; - height: number; -} - -export const PreviewAnnotationRectSchema: Schema.Codec<PreviewAnnotationRect> = Schema.Struct({ - x: Schema.Number, - y: Schema.Number, - width: Schema.Number, - height: Schema.Number, -}); - -export interface PreviewAnnotationPoint { - x: number; - y: number; -} - -export const PreviewAnnotationPointSchema: Schema.Codec<PreviewAnnotationPoint> = Schema.Struct({ - x: Schema.Number, - y: Schema.Number, -}); - -export interface PreviewAnnotationElementTarget { - id: string; - element: PickedElementPayload; - rect: PreviewAnnotationRect; -} - -export const PreviewAnnotationElementTargetSchema: Schema.Codec<PreviewAnnotationElementTarget> = - Schema.Struct({ - id: Schema.String, - element: PickedElementPayloadSchema, - rect: PreviewAnnotationRectSchema, - }); - -export interface PreviewAnnotationRegionTarget { - id: string; - rect: PreviewAnnotationRect; -} - -export const PreviewAnnotationRegionTargetSchema: Schema.Codec<PreviewAnnotationRegionTarget> = - Schema.Struct({ - id: Schema.String, - rect: PreviewAnnotationRectSchema, - }); - -export interface PreviewAnnotationStrokeTarget { - id: string; - color: string; - width: number; - points: ReadonlyArray<PreviewAnnotationPoint>; - bounds: PreviewAnnotationRect; -} - -export const PreviewAnnotationStrokeTargetSchema: Schema.Codec<PreviewAnnotationStrokeTarget> = - Schema.Struct({ - id: Schema.String, - color: Schema.String, - width: Schema.Number, - points: Schema.Array(PreviewAnnotationPointSchema), - bounds: PreviewAnnotationRectSchema, - }); - -export interface PreviewAnnotationStyleChange { - targetId: string; - selector: string | null; - property: string; - previousValue: string; - value: string; -} - -export const PreviewAnnotationStyleChangeSchema: Schema.Codec<PreviewAnnotationStyleChange> = - Schema.Struct({ - targetId: Schema.String, - selector: Schema.NullOr(Schema.String), - property: Schema.String, - previousValue: Schema.String, - value: Schema.String, - }); - -export interface PreviewAnnotationScreenshot { - dataUrl: string; - width: number; - height: number; - cropRect: PreviewAnnotationRect; -} - -export const PreviewAnnotationScreenshotSchema: Schema.Codec<PreviewAnnotationScreenshot> = - Schema.Struct({ - dataUrl: Schema.String, - width: Schema.Number, - height: Schema.Number, - cropRect: PreviewAnnotationRectSchema, - }); - -/** - * A submitted preview annotation. One annotation may reference multiple DOM - * elements, freeform regions, and ink strokes. The desktop main process adds - * the screenshot after the guest preload submits the structured draft. - */ -export interface PreviewAnnotationPayload { - id: string; - pageUrl: string; - pageTitle: string | null; - comment: string; - elements: ReadonlyArray<PreviewAnnotationElementTarget>; - regions: ReadonlyArray<PreviewAnnotationRegionTarget>; - strokes: ReadonlyArray<PreviewAnnotationStrokeTarget>; - styleChanges: ReadonlyArray<PreviewAnnotationStyleChange>; - screenshot: PreviewAnnotationScreenshot | null; - createdAt: string; -} - -export const PreviewAnnotationPayloadSchema: Schema.Codec<PreviewAnnotationPayload> = Schema.Struct( - { - id: Schema.String, - pageUrl: Schema.String, - pageTitle: Schema.NullOr(Schema.String), - comment: Schema.String, - elements: Schema.Array(PreviewAnnotationElementTargetSchema), - regions: Schema.Array(PreviewAnnotationRegionTargetSchema), - strokes: Schema.Array(PreviewAnnotationStrokeTargetSchema), - styleChanges: Schema.Array(PreviewAnnotationStyleChangeSchema), - screenshot: Schema.NullOr(PreviewAnnotationScreenshotSchema), - createdAt: Schema.String, - }, -); - export const DesktopPreviewTabInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, }); @@ -933,6 +743,17 @@ export const DesktopPreviewAutomationClickInputSchema = Schema.Struct({ input: PreviewAutomationClickInput, }); +export const DesktopPreviewAutomationSnapshotOptionsSchema = Schema.Struct({ + mode: Schema.Literal("review"), +}); +export type DesktopPreviewAutomationSnapshotOptions = + typeof DesktopPreviewAutomationSnapshotOptionsSchema.Type; + +export const DesktopPreviewAutomationSnapshotInputSchema = Schema.Struct({ + tabId: DesktopPreviewTabIdSchema, + mode: Schema.optional(DesktopPreviewAutomationSnapshotOptionsSchema.fields.mode), +}); + export const DesktopPreviewAutomationTypeInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, input: PreviewAutomationTypeInput, @@ -1082,7 +903,10 @@ export interface DesktopPreviewBridge { }; automation: { status: (tabId: string) => Promise<PreviewAutomationStatus>; - snapshot: (tabId: string) => Promise<PreviewAutomationSnapshot>; + snapshot: ( + tabId: string, + options?: DesktopPreviewAutomationSnapshotOptions, + ) => Promise<PreviewAutomationSnapshot>; click: (tabId: string, input: PreviewAutomationClickInput) => Promise<void>; type: (tabId: string, input: PreviewAutomationTypeInput) => Promise<void>; press: (tabId: string, input: PreviewAutomationPressInput) => Promise<void>; diff --git a/packages/contracts/src/preview.ts b/packages/contracts/src/preview.ts index dfc10e0b9b7..8856658fb68 100644 --- a/packages/contracts/src/preview.ts +++ b/packages/contracts/src/preview.ts @@ -1,10 +1,10 @@ /** * Preview - Schemas for the in-app browser preview surface. * - * The preview is desktop-only (Chromium <webview>); the server tracks per-thread - * tab metadata so it survives client reconnects and multi-window. The desktop - * renderer mediates: it owns the actual <webview> and reports navigation back to - * the server via these RPCs, the server fans events to all subscribers. + * The server tracks per-thread preview tab metadata so it survives client + * reconnects and multi-window. Desktop owns the canonical Chromium <webview> + * and reports navigation through these RPCs; mobile can render the recorded URL + * in its own WebView or request a short-lived Connect gateway for loopback tabs. * * @module Preview */ @@ -199,6 +199,49 @@ export const PreviewListResult = Schema.Struct({ }); export type PreviewListResult = typeof PreviewListResult.Type; +export const PreviewLiveGatewayOpenInput = Schema.Struct({ + version: Schema.Literal(1), + threadId: ThreadId, + tabId: PreviewTabId, +}); +export type PreviewLiveGatewayOpenInput = typeof PreviewLiveGatewayOpenInput.Type; + +/** + * A short-lived, same-environment URL that hands an isolated browser view to + * the live-preview gateway. The bootstrap token is deliberately returned as a + * relative URL so clients resolve it against the already-authenticated + * environment endpoint instead of trusting another origin. + */ +export const PreviewLiveGatewayOpenResult = Schema.Struct({ + version: Schema.Literal(1), + relativeUrl: Schema.String.check(Schema.isPattern(/^\/api\/preview-gateway\/bootstrap\//u)), + expiresAt: Schema.Number.check(Schema.isGreaterThan(0)), +}); +export type PreviewLiveGatewayOpenResult = typeof PreviewLiveGatewayOpenResult.Type; + +export const PreviewLiveGatewayUnavailableReason = Schema.Literals([ + "preview_idle", + "runtime_unsupported", + "session_expired", + "target_invalid", + "target_not_loopback", + "target_protocol_unsupported", +]); +export type PreviewLiveGatewayUnavailableReason = typeof PreviewLiveGatewayUnavailableReason.Type; + +export class PreviewLiveGatewayUnavailableError extends Schema.TaggedErrorClass<PreviewLiveGatewayUnavailableError>()( + "PreviewLiveGatewayUnavailableError", + { + threadId: ThreadId, + tabId: PreviewTabId, + reason: PreviewLiveGatewayUnavailableReason, + }, +) { + override get message(): string { + return `Live preview gateway is unavailable (${this.reason}): thread=${this.threadId}, tab=${this.tabId}`; + } +} + const PreviewEventBaseSchema = Schema.Struct({ threadId: TrimmedNonEmptyString, tabId: PreviewTabId, diff --git a/packages/contracts/src/previewAnnotation.test.ts b/packages/contracts/src/previewAnnotation.test.ts new file mode 100644 index 00000000000..c12bc76b8d2 --- /dev/null +++ b/packages/contracts/src/previewAnnotation.test.ts @@ -0,0 +1,211 @@ +import * as Schema from "effect/Schema"; +import { describe, expect, it } from "vite-plus/test"; + +import { PreviewAnnotationPayloadSchema } from "./previewAnnotation.ts"; + +const decode = Schema.decodeUnknownSync(PreviewAnnotationPayloadSchema); + +const legacyAnnotation = { + id: "annotation_1", + pageUrl: "http://localhost:3000", + pageTitle: "Example", + comment: "Tighten this area.", + elements: [], + regions: [{ id: "region_1", rect: { x: 10, y: 20, width: 100, height: 80 } }], + strokes: [], + styleChanges: [], + screenshot: null, + createdAt: "2026-06-11T00:00:00.000Z", +} as const; + +const pickedElement = { + pageUrl: "http://localhost:3000", + pageTitle: "Example", + tagName: "button", + selector: "button.submit", + htmlPreview: '<button class="submit">Save</button>', + componentName: "SubmitButton", + source: { + functionName: "SubmitButton", + fileName: "/repo/src/SubmitButton.tsx", + lineNumber: 12, + columnNumber: 5, + }, + stack: [], + styles: ".submit { color: white; }", + pickedAt: "2026-06-11T00:00:00.000Z", +} as const; + +describe("PreviewAnnotationPayloadSchema", () => { + it("continues to decode the legacy desktop payload unchanged", () => { + expect(decode(legacyAnnotation)).toEqual(legacyAnnotation); + }); + + it("decodes portable source, snapshot, callout, and editable-vector metadata", () => { + const annotation = { + ...legacyAnnotation, + schemaVersion: 1, + source: { + kind: "preview", + url: "http://localhost:3000", + title: "Example", + }, + elements: [ + { + id: "element_1", + element: pickedElement, + rect: { x: 40, y: 20, width: 120, height: 48 }, + }, + ], + callouts: [ + { + id: "callout_1", + number: 1, + comment: "Move this action.", + anchor: { kind: "point", point: { x: 0.2, y: 0.3 } }, + }, + { + id: "callout_2", + number: 2, + comment: "Reduce this gap.", + anchor: { + kind: "region", + rect: { x: 0.4, y: 0.1, width: 0.3, height: 0.2 }, + }, + }, + { + id: "callout_3", + number: 3, + comment: "Use the primary action style.", + anchor: { + kind: "element", + targetId: "element_1", + rect: { x: 0.1, y: 0.7, width: 0.4, height: 0.2 }, + }, + }, + ], + editable: { + version: 1, + coordinateSpace: "normalized", + strokes: [ + { + id: "stroke_1", + color: "#7c3aed", + width: 0.01, + points: [ + { x: 0.1, y: 0.2 }, + { x: 0.2, y: 0.3 }, + ], + bounds: { x: 0.08, y: 0.18, width: 0.14, height: 0.14 }, + }, + ], + }, + screenshot: { + dataUrl: "data:image/png;base64,AA==", + width: 800, + height: 600, + cropRect: { x: 20, y: 30, width: 400, height: 300 }, + scale: 2, + pageRevision: "revision_4", + }, + } as const; + + expect(decode(annotation)).toEqual(annotation); + }); + + it("accepts an arbitrary image source", () => { + expect( + decode({ + ...legacyAnnotation, + schemaVersion: 1, + source: { kind: "image", name: "checkout.png" }, + }).source, + ).toEqual({ kind: "image", name: "checkout.png" }); + }); + + it("rejects non-finite and out-of-bounds normalized anchors", () => { + expect(() => + decode({ + ...legacyAnnotation, + callouts: [ + { + id: "callout_nan", + number: 1, + comment: "", + anchor: { kind: "point", point: { x: Number.NaN, y: 0.5 } }, + }, + ], + }), + ).toThrow(); + + expect(() => + decode({ + ...legacyAnnotation, + callouts: [ + { + id: "callout_overflow", + number: 1, + comment: "", + anchor: { + kind: "region", + rect: { x: 0.8, y: 0.2, width: 0.3, height: 0.2 }, + }, + }, + ], + }), + ).toThrow(); + }); + + it("rejects zero-sized regions and non-positive callout numbers", () => { + expect(() => + decode({ + ...legacyAnnotation, + callouts: [ + { + id: "callout_zero", + number: 0, + comment: "", + anchor: { + kind: "region", + rect: { x: 0.2, y: 0.2, width: 0, height: 0.2 }, + }, + }, + ], + }), + ).toThrow(); + }); + + it("rejects non-normalized editable strokes and invalid screenshot scale", () => { + expect(() => + decode({ + ...legacyAnnotation, + editable: { + version: 1, + coordinateSpace: "normalized", + strokes: [ + { + id: "stroke_overflow", + color: "#000000", + width: 0.01, + points: [{ x: 1.1, y: 0.5 }], + bounds: { x: 0.1, y: 0.1, width: 0.2, height: 0.2 }, + }, + ], + }, + }), + ).toThrow(); + + expect(() => + decode({ + ...legacyAnnotation, + screenshot: { + dataUrl: "data:image/png;base64,AA==", + width: 100, + height: 100, + cropRect: { x: 0, y: 0, width: 100, height: 100 }, + scale: 0, + }, + }), + ).toThrow(); + }); +}); diff --git a/packages/contracts/src/previewAnnotation.ts b/packages/contracts/src/previewAnnotation.ts new file mode 100644 index 00000000000..bb35abe4e44 --- /dev/null +++ b/packages/contracts/src/previewAnnotation.ts @@ -0,0 +1,380 @@ +import * as Schema from "effect/Schema"; + +/** + * Single stack frame captured by react-grab's `getElementContext`. We surface + * the source file/line so coding agents can jump straight to the JSX that + * produced the picked DOM node. + */ +export interface PickedElementStackFrame { + functionName: string | null; + fileName: string | null; + lineNumber: number | null; + columnNumber: number | null; +} + +export const PickedElementStackFrameSchema: Schema.Codec<PickedElementStackFrame> = Schema.Struct({ + functionName: Schema.NullOr(Schema.String), + fileName: Schema.NullOr(Schema.String), + lineNumber: Schema.NullOr(Schema.Number), + columnNumber: Schema.NullOr(Schema.Number), +}); + +/** + * A successful element pick from the preview webview. All fields are + * best-effort — pages that don't ship a React fiber tree (or aren't running + * in dev) will still produce a usable payload (selector + html preview), + * just without component / source attribution. + */ +export interface PickedElementPayload { + /** URL of the page the element was picked on. */ + pageUrl: string; + /** Optional `<title>` of that page (best-effort). */ + pageTitle: string | null; + /** Lowercase tag name, e.g. `"button"`. */ + tagName: string; + /** CSS selector resolving back to the element on a re-render. */ + selector: string | null; + /** Truncated outer-HTML preview (matches react-grab's `htmlPreview`). */ + htmlPreview: string; + /** Nearest React component display name, or null when unavailable. */ + componentName: string | null; + /** First source-mapped stack frame (file + line of the JSX source). */ + source: PickedElementStackFrame | null; + /** Full owner-stack frames; can be empty. Useful for richer context. */ + stack: ReadonlyArray<PickedElementStackFrame>; + /** Author CSS only (UA defaults stripped) — react-grab's `styles`. */ + styles: string; + /** Wall-clock pick time as ISO-8601 string. */ + pickedAt: string; +} + +export const PickedElementPayloadSchema: Schema.Codec<PickedElementPayload> = Schema.Struct({ + pageUrl: Schema.String, + pageTitle: Schema.NullOr(Schema.String), + tagName: Schema.String, + selector: Schema.NullOr(Schema.String), + htmlPreview: Schema.String, + componentName: Schema.NullOr(Schema.String), + source: Schema.NullOr(PickedElementStackFrameSchema), + stack: Schema.Array(PickedElementStackFrameSchema), + styles: Schema.String, + pickedAt: Schema.String, +}); + +/** Legacy preview geometry measured in viewport-relative CSS pixels. */ +export interface PreviewAnnotationRect { + x: number; + y: number; + width: number; + height: number; +} + +export const PreviewAnnotationRectSchema: Schema.Codec<PreviewAnnotationRect> = Schema.Struct({ + x: Schema.Number, + y: Schema.Number, + width: Schema.Number, + height: Schema.Number, +}); + +/** Legacy preview point measured in viewport-relative CSS pixels. */ +export interface PreviewAnnotationPoint { + x: number; + y: number; +} + +export const PreviewAnnotationPointSchema: Schema.Codec<PreviewAnnotationPoint> = Schema.Struct({ + x: Schema.Number, + y: Schema.Number, +}); + +export interface PreviewAnnotationElementTarget { + id: string; + element: PickedElementPayload; + rect: PreviewAnnotationRect; +} + +export const PreviewAnnotationElementTargetSchema: Schema.Codec<PreviewAnnotationElementTarget> = + Schema.Struct({ + id: Schema.String, + element: PickedElementPayloadSchema, + rect: PreviewAnnotationRectSchema, + }); + +export interface PreviewAnnotationRegionTarget { + id: string; + rect: PreviewAnnotationRect; +} + +export const PreviewAnnotationRegionTargetSchema: Schema.Codec<PreviewAnnotationRegionTarget> = + Schema.Struct({ + id: Schema.String, + rect: PreviewAnnotationRectSchema, + }); + +export interface PreviewAnnotationStrokeTarget { + id: string; + color: string; + width: number; + points: ReadonlyArray<PreviewAnnotationPoint>; + bounds: PreviewAnnotationRect; +} + +export const PreviewAnnotationStrokeTargetSchema: Schema.Codec<PreviewAnnotationStrokeTarget> = + Schema.Struct({ + id: Schema.String, + color: Schema.String, + width: Schema.Number, + points: Schema.Array(PreviewAnnotationPointSchema), + bounds: PreviewAnnotationRectSchema, + }); + +export interface PreviewAnnotationStyleChange { + targetId: string; + selector: string | null; + property: string; + previousValue: string; + value: string; +} + +export const PreviewAnnotationStyleChangeSchema: Schema.Codec<PreviewAnnotationStyleChange> = + Schema.Struct({ + targetId: Schema.String, + selector: Schema.NullOr(Schema.String), + property: Schema.String, + previousValue: Schema.String, + value: Schema.String, + }); + +const NormalizedCoordinateSchema = Schema.Finite.check( + Schema.isBetween({ minimum: 0, maximum: 1 }), +); +const PositiveNormalizedDistanceSchema = NormalizedCoordinateSchema.check(Schema.isGreaterThan(0)); + +/** A point normalized against the final flattened annotation image. */ +export interface PreviewAnnotationNormalizedPoint { + x: number; + y: number; +} + +export const PreviewAnnotationNormalizedPointSchema: Schema.Codec<PreviewAnnotationNormalizedPoint> = + Schema.Struct({ + x: NormalizedCoordinateSchema, + y: NormalizedCoordinateSchema, + }); + +/** A rectangle normalized against the final flattened annotation image. */ +export interface PreviewAnnotationNormalizedRect { + x: number; + y: number; + width: number; + height: number; +} + +const normalizedRectBoundsFilter = Schema.makeFilter( + (rect: PreviewAnnotationNormalizedRect) => + (rect.x + rect.width <= 1 && rect.y + rect.height <= 1) || + "Normalized annotation rectangles must fit within the flattened image.", +); + +export const PreviewAnnotationNormalizedRectSchema: Schema.Codec<PreviewAnnotationNormalizedRect> = + Schema.Struct({ + x: NormalizedCoordinateSchema, + y: NormalizedCoordinateSchema, + width: PositiveNormalizedDistanceSchema, + height: PositiveNormalizedDistanceSchema, + }).check(normalizedRectBoundsFilter); + +export interface PreviewAnnotationPointAnchor { + kind: "point"; + point: PreviewAnnotationNormalizedPoint; +} + +export const PreviewAnnotationPointAnchorSchema: Schema.Codec<PreviewAnnotationPointAnchor> = + Schema.Struct({ + kind: Schema.Literal("point"), + point: PreviewAnnotationNormalizedPointSchema, + }); + +export interface PreviewAnnotationRegionAnchor { + kind: "region"; + rect: PreviewAnnotationNormalizedRect; +} + +export const PreviewAnnotationRegionAnchorSchema: Schema.Codec<PreviewAnnotationRegionAnchor> = + Schema.Struct({ + kind: Schema.Literal("region"), + rect: PreviewAnnotationNormalizedRectSchema, + }); + +export interface PreviewAnnotationElementAnchor { + kind: "element"; + /** References a target in the payload's legacy-compatible `elements` array. */ + targetId: string; + rect: PreviewAnnotationNormalizedRect; +} + +export const PreviewAnnotationElementAnchorSchema: Schema.Codec<PreviewAnnotationElementAnchor> = + Schema.Struct({ + kind: Schema.Literal("element"), + targetId: Schema.String, + rect: PreviewAnnotationNormalizedRectSchema, + }); + +export type PreviewAnnotationCalloutAnchor = + | PreviewAnnotationPointAnchor + | PreviewAnnotationRegionAnchor + | PreviewAnnotationElementAnchor; + +export const PreviewAnnotationCalloutAnchorSchema: Schema.Codec<PreviewAnnotationCalloutAnchor> = + Schema.Union([ + PreviewAnnotationPointAnchorSchema, + PreviewAnnotationRegionAnchorSchema, + PreviewAnnotationElementAnchorSchema, + ]); + +export interface PreviewAnnotationCallout { + id: string; + /** Visible number burned into the flattened image. */ + number: number; + comment: string; + anchor: PreviewAnnotationCalloutAnchor; +} + +export const PreviewAnnotationCalloutSchema: Schema.Codec<PreviewAnnotationCallout> = Schema.Struct( + { + id: Schema.String, + number: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), + comment: Schema.String, + anchor: PreviewAnnotationCalloutAnchorSchema, + }, +); + +export interface PreviewAnnotationEditableStrokeV1 { + id: string; + color: string; + /** + * Stroke width normalized against the shorter side of the flattened image. + */ + width: number; + points: ReadonlyArray<PreviewAnnotationNormalizedPoint>; + bounds: PreviewAnnotationNormalizedRect; +} + +export const PreviewAnnotationEditableStrokeV1Schema: Schema.Codec<PreviewAnnotationEditableStrokeV1> = + Schema.Struct({ + id: Schema.String, + color: Schema.String, + width: PositiveNormalizedDistanceSchema, + points: Schema.Array(PreviewAnnotationNormalizedPointSchema), + bounds: PreviewAnnotationNormalizedRectSchema, + }); + +export interface PreviewAnnotationEditableVectorV1 { + version: 1; + coordinateSpace: "normalized"; + strokes: ReadonlyArray<PreviewAnnotationEditableStrokeV1>; +} + +export const PreviewAnnotationEditableVectorV1Schema: Schema.Codec<PreviewAnnotationEditableVectorV1> = + Schema.Struct({ + version: Schema.Literal(1), + coordinateSpace: Schema.Literal("normalized"), + strokes: Schema.Array(PreviewAnnotationEditableStrokeV1Schema), + }); + +export interface PreviewAnnotationImageSource { + kind: "image"; + name: string | null; +} + +export const PreviewAnnotationImageSourceSchema: Schema.Codec<PreviewAnnotationImageSource> = + Schema.Struct({ + kind: Schema.Literal("image"), + name: Schema.NullOr(Schema.String), + }); + +export interface PreviewAnnotationPreviewSource { + kind: "preview"; + url: string; + title: string | null; +} + +export const PreviewAnnotationPreviewSourceSchema: Schema.Codec<PreviewAnnotationPreviewSource> = + Schema.Struct({ + kind: Schema.Literal("preview"), + url: Schema.String, + title: Schema.NullOr(Schema.String), + }); + +export type PreviewAnnotationSource = PreviewAnnotationImageSource | PreviewAnnotationPreviewSource; + +export const PreviewAnnotationSourceSchema: Schema.Codec<PreviewAnnotationSource> = Schema.Union([ + PreviewAnnotationImageSourceSchema, + PreviewAnnotationPreviewSourceSchema, +]); + +export interface PreviewAnnotationScreenshot { + dataUrl: string; + /** Encoded image width in pixels. */ + width: number; + /** Encoded image height in pixels. */ + height: number; + /** Crop in the source preview's viewport-relative CSS-pixel coordinate space. */ + cropRect: PreviewAnnotationRect; + /** Encoded-image pixels per source coordinate unit, when known. */ + scale?: number; + /** Host-provided page revision captured with this exact frame, when known. */ + pageRevision?: string | null; +} + +export const PreviewAnnotationScreenshotSchema: Schema.Codec<PreviewAnnotationScreenshot> = + Schema.Struct({ + dataUrl: Schema.String, + width: Schema.Number, + height: Schema.Number, + cropRect: PreviewAnnotationRectSchema, + scale: Schema.optionalKey(Schema.Finite.check(Schema.isGreaterThan(0))), + pageRevision: Schema.optionalKey(Schema.NullOr(Schema.String)), + }); + +/** + * A submitted preview or image annotation. Legacy arrays remain required so + * persisted drafts and mixed-version desktop hosts continue to round-trip. + * New producers may additionally provide normalized numbered callouts and an + * editable vector document. + */ +export interface PreviewAnnotationPayload { + id: string; + pageUrl: string; + pageTitle: string | null; + comment: string; + elements: ReadonlyArray<PreviewAnnotationElementTarget>; + regions: ReadonlyArray<PreviewAnnotationRegionTarget>; + strokes: ReadonlyArray<PreviewAnnotationStrokeTarget>; + styleChanges: ReadonlyArray<PreviewAnnotationStyleChange>; + screenshot: PreviewAnnotationScreenshot | null; + createdAt: string; + schemaVersion?: 1; + source?: PreviewAnnotationSource; + callouts?: ReadonlyArray<PreviewAnnotationCallout>; + editable?: PreviewAnnotationEditableVectorV1 | null; +} + +export const PreviewAnnotationPayloadSchema: Schema.Codec<PreviewAnnotationPayload> = Schema.Struct( + { + id: Schema.String, + pageUrl: Schema.String, + pageTitle: Schema.NullOr(Schema.String), + comment: Schema.String, + elements: Schema.Array(PreviewAnnotationElementTargetSchema), + regions: Schema.Array(PreviewAnnotationRegionTargetSchema), + strokes: Schema.Array(PreviewAnnotationStrokeTargetSchema), + styleChanges: Schema.Array(PreviewAnnotationStyleChangeSchema), + screenshot: Schema.NullOr(PreviewAnnotationScreenshotSchema), + createdAt: Schema.String, + schemaVersion: Schema.optionalKey(Schema.Literal(1)), + source: Schema.optionalKey(PreviewAnnotationSourceSchema), + callouts: Schema.optionalKey(Schema.Array(PreviewAnnotationCalloutSchema)), + editable: Schema.optionalKey(Schema.NullOr(PreviewAnnotationEditableVectorV1Schema)), + }, +); diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index d89c4d6f66e..130cc0b7938 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -1,6 +1,6 @@ import { Schema } from "effect"; -import { EnvironmentId, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; +import { EnvironmentId, NonNegativeInt, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; import { PREVIEW_VIEWPORT_MAX_AREA, PreviewRenderedViewportSize, @@ -504,6 +504,15 @@ export const PreviewAutomationElement = Schema.Struct({ }); export type PreviewAutomationElement = typeof PreviewAutomationElement.Type; +export const PreviewAutomationSnapshotViewport = Schema.Struct({ + width: Schema.Int.check(Schema.isGreaterThan(0)), + height: Schema.Int.check(Schema.isGreaterThan(0)), + scrollX: Schema.Finite, + scrollY: Schema.Finite, + devicePixelRatio: Schema.Finite.check(Schema.isGreaterThan(0)), +}); +export type PreviewAutomationSnapshotViewport = typeof PreviewAutomationSnapshotViewport.Type; + export const PreviewAutomationConsoleEntry = Schema.Struct({ level: Schema.String, text: Schema.String, @@ -536,6 +545,12 @@ export const PreviewAutomationSnapshot = Schema.Struct({ url: Schema.String, title: Schema.String, loading: Schema.Boolean, + /** Missing for desktop hosts predating human review snapshots. */ + viewport: Schema.optional(PreviewAutomationSnapshotViewport), + /** Host-provided identifier for the exact rendered page frame, when available. */ + pageRevision: Schema.optional(TrimmedNonEmptyString), + /** Host capture time, when available. */ + capturedAt: Schema.optional(Schema.String), visibleText: Schema.String, interactiveElements: Schema.Array(PreviewAutomationElement), accessibilityTree: Schema.Unknown, @@ -547,10 +562,128 @@ export const PreviewAutomationSnapshot = Schema.Struct({ data: Schema.String, width: Schema.Int, height: Schema.Int, + /** Encoded-image pixels per viewport CSS pixel. Missing on legacy hosts. */ + scale: Schema.optional(Schema.Finite.check(Schema.isGreaterThan(0))), }), }); export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type; +/** + * The human review capture path deliberately decodes only fields that can be + * shown or attached. Schema.Struct accepts the extra diagnostics returned by + * legacy hosts, so mixed-version desktop/server pairs remain compatible. + */ +export const PreviewAutomationReviewSnapshot = Schema.Struct({ + url: PreviewAutomationSnapshot.fields.url, + title: PreviewAutomationSnapshot.fields.title, + loading: PreviewAutomationSnapshot.fields.loading, + viewport: PreviewAutomationSnapshot.fields.viewport, + pageRevision: PreviewAutomationSnapshot.fields.pageRevision, + capturedAt: PreviewAutomationSnapshot.fields.capturedAt, + interactiveElements: PreviewAutomationSnapshot.fields.interactiveElements, + screenshot: PreviewAutomationSnapshot.fields.screenshot, +}); +export type PreviewAutomationReviewSnapshot = typeof PreviewAutomationReviewSnapshot.Type; + +export const PREVIEW_REVIEW_SNAPSHOT_MAX_ELEMENTS = 200; +export const PREVIEW_REVIEW_SNAPSHOT_MAX_IMAGE_BYTES = 10 * 1024 * 1024; +export const PREVIEW_REVIEW_SNAPSHOT_MAX_BASE64_CHARS = + Math.ceil(PREVIEW_REVIEW_SNAPSHOT_MAX_IMAGE_BYTES / 3) * 4; +export const PREVIEW_REVIEW_SNAPSHOT_MAX_PAGE_REVISION_LENGTH = 128; + +const PreviewReviewSnapshotBoundedString = (maximumLength: number) => + Schema.String.check(Schema.isMaxLength(maximumLength)); +const PreviewReviewSnapshotNonNegativeFinite = Schema.Finite.check( + Schema.isGreaterThanOrEqualTo(0), +); + +export const PreviewReviewSnapshotInput = Schema.Struct({ + version: Schema.Literal(1), + threadId: ThreadId, + tabId: PreviewTabId, +}); +export type PreviewReviewSnapshotInput = typeof PreviewReviewSnapshotInput.Type; + +export const PreviewReviewSnapshotRectV1 = Schema.Struct({ + x: PreviewReviewSnapshotNonNegativeFinite, + y: PreviewReviewSnapshotNonNegativeFinite, + width: PreviewReviewSnapshotNonNegativeFinite, + height: PreviewReviewSnapshotNonNegativeFinite, +}); +export type PreviewReviewSnapshotRectV1 = typeof PreviewReviewSnapshotRectV1.Type; + +export const PreviewReviewSnapshotElementV1 = Schema.Struct({ + id: TrimmedNonEmptyString.check(Schema.isMaxLength(128)), + tag: PreviewReviewSnapshotBoundedString(64), + role: Schema.NullOr(PreviewReviewSnapshotBoundedString(128)), + name: PreviewReviewSnapshotBoundedString(1_024), + selector: PreviewReviewSnapshotBoundedString(2_048), + rect: PreviewReviewSnapshotRectV1, +}); +export type PreviewReviewSnapshotElementV1 = typeof PreviewReviewSnapshotElementV1.Type; + +export const PreviewReviewSnapshotViewportV1 = PreviewAutomationSnapshotViewport; +export type PreviewReviewSnapshotViewportV1 = typeof PreviewReviewSnapshotViewportV1.Type; + +export const PreviewReviewSnapshotScreenshotV1 = Schema.Struct({ + mimeType: Schema.Literal("image/png"), + data: Schema.String.check(Schema.isMaxLength(PREVIEW_REVIEW_SNAPSHOT_MAX_BASE64_CHARS)), + width: Schema.Int.check(Schema.isGreaterThan(0)), + height: Schema.Int.check(Schema.isGreaterThan(0)), + scale: Schema.Finite.check(Schema.isGreaterThan(0)), +}); +export type PreviewReviewSnapshotScreenshotV1 = typeof PreviewReviewSnapshotScreenshotV1.Type; + +export const PreviewReviewSnapshotV1 = Schema.Struct({ + version: Schema.Literal(1), + snapshotId: TrimmedNonEmptyString.check(Schema.isMaxLength(128)), + pageRevision: TrimmedNonEmptyString.check( + Schema.isMaxLength(PREVIEW_REVIEW_SNAPSHOT_MAX_PAGE_REVISION_LENGTH), + ), + serverEpoch: TrimmedNonEmptyString, + previewRevision: NonNegativeInt, + threadId: ThreadId, + tabId: PreviewTabId, + capturedAt: Schema.String, + url: PreviewReviewSnapshotBoundedString(2_048), + title: PreviewReviewSnapshotBoundedString(512), + loading: Schema.Boolean, + viewport: PreviewReviewSnapshotViewportV1, + screenshot: PreviewReviewSnapshotScreenshotV1, + elements: Schema.Array(PreviewReviewSnapshotElementV1).check( + Schema.isMaxLength(PREVIEW_REVIEW_SNAPSHOT_MAX_ELEMENTS), + ), +}); +export type PreviewReviewSnapshotV1 = typeof PreviewReviewSnapshotV1.Type; + +export const PreviewReviewSnapshot = PreviewReviewSnapshotV1; +export type PreviewReviewSnapshot = typeof PreviewReviewSnapshot.Type; + +export class PreviewReviewSnapshotTooLargeError extends Schema.TaggedErrorClass<PreviewReviewSnapshotTooLargeError>()( + "PreviewReviewSnapshotTooLargeError", + { + threadId: ThreadId, + tabId: PreviewTabId, + maximumBytes: Schema.Int.check(Schema.isGreaterThan(0)), + }, +) { + override get message(): string { + return `Preview snapshot image exceeds the ${this.maximumBytes}-byte review limit.`; + } +} + +export class PreviewReviewSnapshotMalformedError extends Schema.TaggedErrorClass<PreviewReviewSnapshotMalformedError>()( + "PreviewReviewSnapshotMalformedError", + { + threadId: ThreadId, + tabId: PreviewTabId, + }, +) { + override get message(): string { + return "The desktop preview host returned an invalid review snapshot."; + } +} + export const PreviewAutomationRecordingStatus = Schema.Struct({ tabId: PreviewTabId, recording: Schema.Boolean, @@ -655,8 +788,9 @@ const PreviewAutomationScopeErrorFields = { operation: PreviewAutomationOperation, environmentId: EnvironmentId, threadId: ThreadId, - providerSessionId: TrimmedNonEmptyString, - providerInstanceId: ProviderInstanceId, + providerSessionId: Schema.optional(TrimmedNonEmptyString), + providerInstanceId: Schema.optional(ProviderInstanceId), + requesterId: Schema.optional(PreviewAutomationClientId), }; const PreviewAutomationRequestErrorFields = { diff --git a/packages/contracts/src/previewLiveGateway.test.ts b/packages/contracts/src/previewLiveGateway.test.ts new file mode 100644 index 00000000000..08bff4f898a --- /dev/null +++ b/packages/contracts/src/previewLiveGateway.test.ts @@ -0,0 +1,71 @@ +import * as Schema from "effect/Schema"; +import { describe, expect, it } from "vite-plus/test"; + +import { + PreviewLiveGatewayOpenInput, + PreviewLiveGatewayOpenResult, + PreviewLiveGatewayUnavailableError, +} from "./preview.ts"; + +const decodeInput = Schema.decodeUnknownSync(PreviewLiveGatewayOpenInput); +const decodeResult = Schema.decodeUnknownSync(PreviewLiveGatewayOpenResult); +const decodeUnavailable = Schema.decodeUnknownSync(PreviewLiveGatewayUnavailableError); + +describe("preview live gateway contracts", () => { + it("targets one exact versioned preview tab", () => { + expect( + decodeInput({ + version: 1, + threadId: "thread-1", + tabId: "tab-1", + }), + ).toEqual({ + version: 1, + threadId: "thread-1", + tabId: "tab-1", + }); + expect(() => decodeInput({ version: 1, threadId: "thread-1" })).toThrow(); + }); + + it("accepts only same-environment relative bootstrap URLs", () => { + expect( + decodeResult({ + version: 1, + relativeUrl: "/api/preview-gateway/bootstrap/opaque-token", + expiresAt: 1_785_369_600_000, + }), + ).toEqual({ + version: 1, + relativeUrl: "/api/preview-gateway/bootstrap/opaque-token", + expiresAt: 1_785_369_600_000, + }); + expect(() => + decodeResult({ + version: 1, + relativeUrl: "https://attacker.example/bootstrap", + expiresAt: 1_785_369_600_000, + }), + ).toThrow(); + }); + + it("serializes stable unavailability reasons", () => { + for (const reason of [ + "runtime_unsupported", + "session_expired", + "target_not_loopback", + ] as const) { + expect( + decodeUnavailable({ + _tag: "PreviewLiveGatewayUnavailableError", + threadId: "thread-1", + tabId: "tab-1", + reason, + }), + ).toMatchObject({ + threadId: "thread-1", + tabId: "tab-1", + reason, + }); + } + }); +}); diff --git a/packages/contracts/src/previewReview.test.ts b/packages/contracts/src/previewReview.test.ts new file mode 100644 index 00000000000..95dd65a9334 --- /dev/null +++ b/packages/contracts/src/previewReview.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vite-plus/test"; +import * as Schema from "effect/Schema"; + +import { + PREVIEW_REVIEW_SNAPSHOT_MAX_ELEMENTS, + PreviewAutomationReviewSnapshot, + PreviewAutomationSnapshot, + PreviewReviewSnapshot, + PreviewReviewSnapshotInput, +} from "./previewAutomation.ts"; + +const decodeInput = Schema.decodeUnknownSync(PreviewReviewSnapshotInput); +const decodeSnapshot = Schema.decodeUnknownSync(PreviewReviewSnapshot); +const decodeAutomationSnapshot = Schema.decodeUnknownSync(PreviewAutomationSnapshot); +const decodeAutomationReviewSnapshot = Schema.decodeUnknownSync(PreviewAutomationReviewSnapshot); + +const reviewSnapshot = { + version: 1, + snapshotId: "snapshot-1", + pageRevision: "page-1", + serverEpoch: "server-1", + previewRevision: 4, + threadId: "thread-1", + tabId: "tab-1", + capturedAt: "2026-07-30T12:00:00.000Z", + url: "http://localhost:5173", + title: "T3", + loading: false, + viewport: { + width: 1_280, + height: 800, + scrollX: 0, + scrollY: 120, + devicePixelRatio: 2, + }, + screenshot: { + mimeType: "image/png", + data: "iVBORw0KGgo=", + width: 1_280, + height: 800, + scale: 1, + }, + elements: [ + { + id: "element-1", + tag: "button", + role: "button", + name: "Send", + selector: "#send", + rect: { x: 12, y: 20, width: 100, height: 40 }, + }, + ], +} as const; + +describe("preview review snapshot contracts", () => { + it("requires a versioned exact thread and tab target", () => { + expect( + decodeInput({ + version: 1, + threadId: "thread-1", + tabId: "tab-1", + }), + ).toEqual({ + version: 1, + threadId: "thread-1", + tabId: "tab-1", + }); + expect(() => decodeInput({ version: 1, threadId: "thread-1" })).toThrow(); + }); + + it("decodes the compact versioned review result", () => { + expect(decodeSnapshot(reviewSnapshot)).toEqual(reviewSnapshot); + }); + + it("caps semantic elements at the desktop snapshot bound", () => { + expect(() => + decodeSnapshot({ + ...reviewSnapshot, + elements: Array.from({ length: PREVIEW_REVIEW_SNAPSHOT_MAX_ELEMENTS + 1 }, (_, index) => ({ + ...reviewSnapshot.elements[0], + id: `element-${index}`, + })), + }), + ).toThrow(); + }); + + it("keeps viewport and image scale optional on mixed-version automation hosts", () => { + const legacy = { + url: "http://localhost:5173", + title: "T3", + loading: false, + visibleText: "Send", + interactiveElements: [], + accessibilityTree: {}, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + screenshot: { + mimeType: "image/png", + data: "iVBORw0KGgo=", + width: 1_280, + height: 800, + }, + } as const; + expect(decodeAutomationSnapshot(legacy)).toEqual(legacy); + expect( + decodeAutomationSnapshot({ + ...legacy, + viewport: reviewSnapshot.viewport, + screenshot: reviewSnapshot.screenshot, + }), + ).toMatchObject({ + viewport: reviewSnapshot.viewport, + screenshot: { scale: 1 }, + }); + expect(decodeAutomationReviewSnapshot(legacy)).toEqual({ + url: legacy.url, + title: legacy.title, + loading: legacy.loading, + interactiveElements: legacy.interactiveElements, + screenshot: legacy.screenshot, + }); + }); +}); diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 400011f8843..56335c411f2 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -105,6 +105,9 @@ import { PreviewCloseInput, PreviewError, PreviewEvent, + PreviewLiveGatewayOpenInput, + PreviewLiveGatewayOpenResult, + PreviewLiveGatewayUnavailableError, PreviewListInput, PreviewListResult, PreviewNavigateInput, @@ -120,6 +123,10 @@ import { PreviewAutomationHostFocus, PreviewAutomationResponse, PreviewAutomationStreamEvent, + PreviewReviewSnapshot, + PreviewReviewSnapshotInput, + PreviewReviewSnapshotMalformedError, + PreviewReviewSnapshotTooLargeError, } from "./previewAutomation.ts"; import { ServerConfigStreamEvent, @@ -215,6 +222,8 @@ export const WS_METHODS = { previewClose: "preview.close", previewList: "preview.list", previewReportStatus: "preview.reportStatus", + previewReviewSnapshot: "preview.reviewSnapshot", + previewOpenLiveGateway: "preview.openLiveGateway", previewAutomationConnect: "previewAutomation.connect", previewAutomationRespond: "previewAutomation.respond", previewAutomationFocusHost: "previewAutomation.focusHost", @@ -644,6 +653,28 @@ export const WsPreviewReportStatusRpc = Rpc.make(WS_METHODS.previewReportStatus, error: Schema.Union([PreviewError, EnvironmentAuthorizationError]), }); +export const WsPreviewReviewSnapshotRpc = Rpc.make(WS_METHODS.previewReviewSnapshot, { + payload: PreviewReviewSnapshotInput, + success: PreviewReviewSnapshot, + error: Schema.Union([ + PreviewError, + PreviewAutomationError, + PreviewReviewSnapshotMalformedError, + PreviewReviewSnapshotTooLargeError, + EnvironmentAuthorizationError, + ]), +}); + +export const WsPreviewOpenLiveGatewayRpc = Rpc.make(WS_METHODS.previewOpenLiveGateway, { + payload: PreviewLiveGatewayOpenInput, + success: PreviewLiveGatewayOpenResult, + error: Schema.Union([ + PreviewError, + PreviewLiveGatewayUnavailableError, + EnvironmentAuthorizationError, + ]), +}); + export const WsPreviewAutomationConnectRpc = Rpc.make(WS_METHODS.previewAutomationConnect, { payload: PreviewAutomationHost, success: PreviewAutomationStreamEvent, @@ -846,6 +877,8 @@ export const WsRpcGroup = RpcGroup.make( WsPreviewCloseRpc, WsPreviewListRpc, WsPreviewReportStatusRpc, + WsPreviewReviewSnapshotRpc, + WsPreviewOpenLiveGatewayRpc, WsPreviewAutomationConnectRpc, WsPreviewAutomationRespondRpc, WsPreviewAutomationFocusHostRpc, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 85afdd75d7a..88b0aeac886 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -406,6 +406,9 @@ importers: react-native-svg: specifier: 15.15.4 version: 15.15.4(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) + react-native-view-shot: + specifier: 5.1.0 + version: 5.1.0(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) react-native-webview: specifier: ^13.16.1 version: 13.16.1(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) @@ -5544,6 +5547,10 @@ packages: base-64@1.0.0: resolution: {integrity: sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==} + base64-arraybuffer@1.0.2: + resolution: {integrity: sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==} + engines: {node: '>= 0.6.0'} + base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} @@ -5962,6 +5969,9 @@ packages: crypto-js@4.2.0: resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==} + css-line-break@2.1.0: + resolution: {integrity: sha512-FHcKFCZcAha3LwfVBhCQbW2nCNbkZXn7KVUJcsT5/P8YmfsVja0FMPJr0B903j/E69HUphKiV9iQArX8SDYA4w==} + css-select@5.2.2: resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==} @@ -7184,6 +7194,10 @@ packages: html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + html2canvas@1.4.1: + resolution: {integrity: sha512-fPU6BHNpsyIhr8yyMpTLLxAbkaK8ArIBcmZIRiBLiDhjeqvXolaEmDGmELFuX9I4xDcaKKcJl+TKZLqruBbmWA==} + engines: {node: '>=8.0.0'} + http-cache-semantics@4.2.0: resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} @@ -9021,6 +9035,13 @@ packages: peerDependencies: react-native: '*' + react-native-view-shot@5.1.0: + resolution: {integrity: sha512-JZgElCD82aO+hejIF/leUzI7JufL9mgJ6ChzGWIcdZ2ajpaEvvSnvIcw0qD32XWkrbId8wfSbyz/4u/ulTQzQA==} + engines: {node: '>=20', npm: '>=10'} + peerDependencies: + react: '>=18.0.0' + react-native: '>=0.76.0' + react-native-webview@13.16.1: resolution: {integrity: sha512-If0eHhoEdOYDcHsX+xBFwHMbWBGK1BvGDQDQdVkwtSIXiq1uiqjkpWVP2uQ1as94J0CzvFE9PUNDuhiX0Z6ubw==} peerDependencies: @@ -9704,6 +9725,9 @@ packages: engines: {node: '>=10'} hasBin: true + text-segmentation@1.0.3: + resolution: {integrity: sha512-iOiPUo/BGnZ6+54OsWxZidGCsdU8YbE4PSpdPinp7DeMtUJNJBoJ/ouUSTJjHkh1KntHaltHl/gDs2FC4i5+Nw==} + throat@5.0.0: resolution: {integrity: sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==} @@ -10073,6 +10097,9 @@ packages: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} engines: {node: '>= 0.4.0'} + utrie@1.0.2: + resolution: {integrity: sha512-1MLa5ouZiOmQzUbjbu9VmjLzn1QLXBhwpUa7kdLUQK+KQ5KA9I1vk5U4YHe/X2Ch7PYnJfWuWT+VbuxbGwljhw==} + uuid@14.0.1: resolution: {integrity: sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==} hasBin: true @@ -15846,6 +15873,8 @@ snapshots: base-64@1.0.0: {} + base64-arraybuffer@1.0.2: {} + base64-js@1.5.1: {} baseline-browser-mapping@2.10.33: {} @@ -16264,6 +16293,10 @@ snapshots: crypto-js@4.2.0: {} + css-line-break@2.1.0: + dependencies: + utrie: 1.0.2 + css-select@5.2.2: dependencies: boolbase: 1.0.0 @@ -17846,6 +17879,11 @@ snapshots: html-void-elements@3.0.0: {} + html2canvas@1.4.1: + dependencies: + css-line-break: 2.1.0 + text-segmentation: 1.0.3 + http-cache-semantics@4.2.0: {} http-errors@2.0.1: @@ -19942,6 +19980,12 @@ snapshots: dependencies: react-native: 0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6) + react-native-view-shot@5.1.0(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3): + dependencies: + html2canvas: 1.4.1 + react: 19.2.3 + react-native: 0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6) + react-native-webview@13.16.1(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3): dependencies: escape-string-regexp: 4.0.0 @@ -20971,6 +21015,10 @@ snapshots: commander: 2.20.3 source-map-support: 0.5.21 + text-segmentation@1.0.3: + dependencies: + utrie: 1.0.2 + throat@5.0.0: {} timestring@6.0.0: {} @@ -21289,6 +21337,10 @@ snapshots: utils-merge@1.0.1: {} + utrie@1.0.2: + dependencies: + base64-arraybuffer: 1.0.2 + uuid@14.0.1: {} uuid@7.0.3: {} From 628a6ae21eef280fddbafcde312eb375bf296618 Mon Sep 17 00:00:00 2001 From: Diego Carlino <carlid.dev@gmail.com> Date: Sat, 1 Aug 2026 04:39:00 +0200 Subject: [PATCH 2/6] fix(server): provide live gateway through Effect --- apps/server/src/preview/LiveGateway.ts | 2 - apps/server/src/preview/LiveGatewayHttp.ts | 176 ++++++++++----------- apps/server/src/server.test.ts | 23 ++- apps/server/src/server.ts | 63 +++----- apps/server/src/ws.ts | 120 +++++++------- 5 files changed, 186 insertions(+), 198 deletions(-) diff --git a/apps/server/src/preview/LiveGateway.ts b/apps/server/src/preview/LiveGateway.ts index 145362077f1..1349d87342c 100644 --- a/apps/server/src/preview/LiveGateway.ts +++ b/apps/server/src/preview/LiveGateway.ts @@ -359,8 +359,6 @@ export const make = Effect.gen(function* PreviewLiveGatewayMake() { }); }); -export const layer = Layer.effect(PreviewLiveGateway, make); - export const makeLive = Effect.gen(function* PreviewLiveGatewayMakeLive() { const gateway = yield* make; const sessions = yield* SessionStore.SessionStore; diff --git a/apps/server/src/preview/LiveGatewayHttp.ts b/apps/server/src/preview/LiveGatewayHttp.ts index 83734833864..38962d768bb 100644 --- a/apps/server/src/preview/LiveGatewayHttp.ts +++ b/apps/server/src/preview/LiveGatewayHttp.ts @@ -21,6 +21,7 @@ import { } from "effect/unstable/http"; import * as Socket from "effect/unstable/socket/Socket"; +import * as SessionStore from "../auth/SessionStore.ts"; import { LIVE_GATEWAY_BOOTSTRAP_PREFIX, LIVE_GATEWAY_COOKIE_NAME, @@ -54,11 +55,6 @@ const bootstrapResponseHeaders = { "x-content-type-options": "nosniff", } as const; -export interface LiveGatewayHttpOptions { - readonly gateway: PreviewLiveGateway["Service"]; - readonly environmentSessionCookieName: string; -} - export function isLiveGatewayControlCookie( name: string, environmentSessionCookieName: string, @@ -528,100 +524,100 @@ function expiredGatewayResponse(): HttpServerResponse.HttpServerResponse { }); } -export const makeLiveGatewayBootstrapRouteLayer = (gateway: PreviewLiveGateway["Service"]) => - HttpRouter.add( - "GET", - `${LIVE_GATEWAY_BOOTSTRAP_PREFIX}/*`, +export const liveGatewayBootstrapRouteLayer = HttpRouter.add( + "GET", + `${LIVE_GATEWAY_BOOTSTRAP_PREFIX}/*`, + Effect.gen(function* () { + const gateway = yield* PreviewLiveGateway; + const request = yield* HttpServerRequest.HttpServerRequest; + const requestUrl = HttpServerRequest.toURL(request); + if (Option.isNone(requestUrl)) { + return HttpServerResponse.text("Bad Request", { status: 400 }); + } + const token = requestUrl.value.pathname.slice(`${LIVE_GATEWAY_BOOTSTRAP_PREFIX}/`.length); + if (!BOOTSTRAP_TOKEN_PATTERN.test(token)) { + return expiredGatewayResponse(); + } + const consumed = yield* gateway.consumeBootstrap(token); + if (consumed === null) { + return expiredGatewayResponse(); + } + const cookieName = liveGatewayCookieName(requestUrl.value); + const cookie = yield* Effect.fromResult( + Cookies.set(Cookies.empty, cookieName, consumed.cookieValue, { + path: "/", + secure: cookieName === LIVE_GATEWAY_COOKIE_NAME, + httpOnly: true, + sameSite: "strict", + expires: DateTime.toDate(DateTime.makeUnsafe(consumed.lease.expiresAt)), + }), + ).pipe(Effect.orDie); + return HttpServerResponse.redirect(consumed.lease.target.redirectPath, { + status: 302, + headers: bootstrapResponseHeaders, + cookies: cookie, + }); + }), +); + +export const liveGatewayProxyLayer = HttpRouter.middleware( + (httpEffect) => Effect.gen(function* () { + const gateway = yield* PreviewLiveGateway; + const sessions = yield* SessionStore.SessionStore; const request = yield* HttpServerRequest.HttpServerRequest; const requestUrl = HttpServerRequest.toURL(request); - if (Option.isNone(requestUrl)) { - return HttpServerResponse.text("Bad Request", { status: 400 }); - } - const token = requestUrl.value.pathname.slice(`${LIVE_GATEWAY_BOOTSTRAP_PREFIX}/`.length); - if (!BOOTSTRAP_TOKEN_PATTERN.test(token)) { - return expiredGatewayResponse(); + if ( + Option.isNone(requestUrl) || + requestUrl.value.pathname.startsWith(`${LIVE_GATEWAY_BOOTSTRAP_PREFIX}/`) + ) { + return yield* httpEffect; } - const consumed = yield* gateway.consumeBootstrap(token); - if (consumed === null) { - return expiredGatewayResponse(); - } - const cookieName = liveGatewayCookieName(requestUrl.value); - const cookie = yield* Effect.fromResult( - Cookies.set(Cookies.empty, cookieName, consumed.cookieValue, { - path: "/", - secure: cookieName === LIVE_GATEWAY_COOKIE_NAME, - httpOnly: true, - sameSite: "strict", - expires: DateTime.toDate(DateTime.makeUnsafe(consumed.lease.expiresAt)), - }), - ).pipe(Effect.orDie); - return HttpServerResponse.redirect(consumed.lease.target.redirectPath, { - status: 302, - headers: bootstrapResponseHeaders, - cookies: cookie, - }); - }), - ); - -export const makeLiveGatewayProxyLayer = (options: LiveGatewayHttpOptions) => - HttpRouter.middleware( - (httpEffect) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const requestUrl = HttpServerRequest.toURL(request); - if ( - Option.isNone(requestUrl) || - requestUrl.value.pathname.startsWith(`${LIVE_GATEWAY_BOOTSTRAP_PREFIX}/`) - ) { - return yield* httpEffect; - } - const cookieValue = - request.cookies[LIVE_GATEWAY_COOKIE_NAME] ?? - request.cookies[LIVE_GATEWAY_HTTP_COOKIE_NAME]; - if (!cookieValue) return yield* httpEffect; - - const lease = yield* options.gateway.resolveLease(cookieValue); - if (lease === null) return expiredGatewayResponse(); - - if (request.headers.upgrade?.toLowerCase() === "websocket") { - return yield* proxyWebSocketRequest( - request, - requestUrl.value, - lease, - options.environmentSessionCookieName, - ).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Live preview gateway websocket failed", { - cause, - threadId: lease.threadId, - tabId: lease.tabId, - upstreamOrigin: lease.target.origin, - }).pipe(Effect.as(HttpServerResponse.empty())), - ), - ); - } - - return yield* Effect.raceFirst( - proxyHttpRequest(request, requestUrl.value, lease, options.environmentSessionCookieName), - lease.invalidated.pipe(Effect.as(expiredGatewayResponse())), + const cookieValue = + request.cookies[LIVE_GATEWAY_COOKIE_NAME] ?? request.cookies[LIVE_GATEWAY_HTTP_COOKIE_NAME]; + if (!cookieValue) return yield* httpEffect; + + const lease = yield* gateway.resolveLease(cookieValue); + if (lease === null) return expiredGatewayResponse(); + + if (request.headers.upgrade?.toLowerCase() === "websocket") { + return yield* proxyWebSocketRequest( + request, + requestUrl.value, + lease, + sessions.cookieName, ).pipe( Effect.catchCause((cause) => - Effect.logWarning("Live preview gateway upstream request failed", { + Effect.logWarning("Live preview gateway websocket failed", { cause, threadId: lease.threadId, tabId: lease.tabId, upstreamOrigin: lease.target.origin, - }).pipe( - Effect.as( - HttpServerResponse.text("Live preview upstream is unavailable.", { - status: 502, - headers: bootstrapResponseHeaders, - }), - ), - ), + }).pipe(Effect.as(HttpServerResponse.empty())), ), ); - }), - { global: true }, - ); + } + + return yield* Effect.raceFirst( + proxyHttpRequest(request, requestUrl.value, lease, sessions.cookieName), + lease.invalidated.pipe(Effect.as(expiredGatewayResponse())), + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Live preview gateway upstream request failed", { + cause, + threadId: lease.threadId, + tabId: lease.tabId, + upstreamOrigin: lease.target.origin, + }).pipe( + Effect.as( + HttpServerResponse.text("Live preview upstream is unavailable.", { + status: 502, + headers: bootstrapResponseHeaders, + }), + ), + ), + ), + ); + }), + { global: true }, +); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index d13c323a973..5722475f9f8 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -93,7 +93,11 @@ import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; import * as ServerSettings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import * as PreviewManager from "./preview/Manager.ts"; -import { LIVE_GATEWAY_COOKIE_NAME, LIVE_GATEWAY_HTTP_COOKIE_NAME } from "./preview/LiveGateway.ts"; +import { + LIVE_GATEWAY_COOKIE_NAME, + LIVE_GATEWAY_HTTP_COOKIE_NAME, + liveLayer as previewLiveGatewayLayer, +} from "./preview/LiveGateway.ts"; import { LIVE_GATEWAY_EXPIRED_STATUS, liveGatewayCookieName } from "./preview/LiveGatewayHttp.ts"; import * as PortScanner from "./preview/PortScanner.ts"; import * as BrowserTraceCollector from "./observability/BrowserTraceCollector.ts"; @@ -559,14 +563,17 @@ const buildAppUnderTest = (options?: { disableLogger: true, }).pipe( Layer.provide( - Layer.mock(Keybindings.Keybindings)({ - loadConfigState: Effect.succeed({ - keybindings: [], - issues: [], + Layer.mergeAll( + previewLiveGatewayLayer, + Layer.mock(Keybindings.Keybindings)({ + loadConfigState: Effect.succeed({ + keybindings: [], + issues: [], + }), + streamChanges: Stream.empty, + ...options?.layers?.keybindings, }), - streamChanges: Stream.empty, - ...options?.layers?.keybindings, - }), + ), ), Layer.provide( Layer.mock(ProviderRegistry.ProviderRegistry)({ diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index db34c046dbc..063b56a0de0 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -19,7 +19,7 @@ import { httpCompressionLayer, } from "./http.ts"; import { fixPath } from "./os-jank.ts"; -import { makeWebsocketRpcRouteLayer } from "./ws.ts"; +import { websocketRpcRouteLayer } from "./ws.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; import { layerConfig as SqlitePersistenceLayerLive } from "./persistence/Layers/Sqlite.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; @@ -46,8 +46,8 @@ import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; import * as PreviewManager from "./preview/Manager.ts"; import * as PreviewLiveGateway from "./preview/LiveGateway.ts"; import { - makeLiveGatewayBootstrapRouteLayer, - makeLiveGatewayProxyLayer, + liveGatewayBootstrapRouteLayer, + liveGatewayProxyLayer, } from "./preview/LiveGatewayHttp.ts"; import * as PortScanner from "./preview/PortScanner.ts"; import * as ProcessRunner from "./processRunner.ts"; @@ -86,7 +86,6 @@ import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; -import * as SessionStore from "./auth/SessionStore.ts"; import { connectHttpApiLayer, reconcileDesiredCloudLink, @@ -406,39 +405,28 @@ const RuntimeServicesLive = ServerRuntimeStartup.layer.pipe( Layer.provideMerge(RuntimeDependenciesLive), ); -export const makeRoutesLayer = Layer.unwrap( - Effect.gen(function* () { - const previewLiveGateway = yield* PreviewLiveGateway.makeLive; - const sessions = yield* SessionStore.SessionStore; - return Layer.mergeAll( - Layer.mergeAll( - HttpApiBuilder.layer(EnvironmentHttpApi).pipe( - Layer.provide(authHttpApiLayer), - Layer.provide(connectHttpApiLayer), - Layer.provide(orchestrationHttpApiLayer), - Layer.provide(serverEnvironmentHttpApiLayer), - Layer.provide(environmentAuthenticatedAuthLayer), - ), - otlpTracesProxyRouteLayer, - assetRouteLayer, - makeLiveGatewayBootstrapRouteLayer(previewLiveGateway), - staticAndDevRouteLayer, - makeWebsocketRpcRouteLayer(previewLiveGateway), - ), - McpHttpServer.layer.pipe(Layer.provide(McpSessionRegistry.layer)), - ).pipe( - Layer.provide(PreviewAutomationBroker.layer), - Layer.provide(ServerSelfUpdate.layer), - Layer.provide(browserApiCorsLayer), - Layer.provide(httpCompressionLayer), - Layer.provide( - makeLiveGatewayProxyLayer({ - gateway: previewLiveGateway, - environmentSessionCookieName: sessions.cookieName, - }), - ), - ); - }), +export const makeRoutesLayer = Layer.mergeAll( + Layer.mergeAll( + HttpApiBuilder.layer(EnvironmentHttpApi).pipe( + Layer.provide(authHttpApiLayer), + Layer.provide(connectHttpApiLayer), + Layer.provide(orchestrationHttpApiLayer), + Layer.provide(serverEnvironmentHttpApiLayer), + Layer.provide(environmentAuthenticatedAuthLayer), + ), + otlpTracesProxyRouteLayer, + assetRouteLayer, + liveGatewayBootstrapRouteLayer, + staticAndDevRouteLayer, + websocketRpcRouteLayer, + ), + McpHttpServer.layer.pipe(Layer.provide(McpSessionRegistry.layer)), +).pipe( + Layer.provide(PreviewAutomationBroker.layer), + Layer.provide(ServerSelfUpdate.layer), + Layer.provide(browserApiCorsLayer), + Layer.provide(httpCompressionLayer), + Layer.provide(liveGatewayProxyLayer), ); export const makeServerLayer = Layer.unwrap( @@ -597,6 +585,7 @@ export const makeServerLayer = Layer.unwrap( ); return serverApplicationLayer.pipe( + Layer.provide(PreviewLiveGateway.liveLayer), Layer.provideMerge(RuntimeServicesLive), Layer.provideMerge(serverRelayBrokerTracingLayer), Layer.provideMerge(HttpResponseCompressionLive), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 2334c808710..b8599a5bc6a 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2210,71 +2210,69 @@ const makeWsRpcLayer = ( }), ); -export const makeWebsocketRpcRouteLayer = ( - previewLiveGateway: PreviewLiveGateway.PreviewLiveGateway["Service"], -) => - Layer.unwrap( - Effect.gen(function* () { - const previewAutomationBroker = yield* PreviewAutomationBroker.PreviewAutomationBroker; - const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; - return HttpRouter.add( - "GET", - "/ws", - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; - const sessions = yield* SessionStore.SessionStore; - const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( - Effect.catchIf(EnvironmentAuth.isServerAuthCredentialError, (error) => - failEnvironmentAuthInvalid(EnvironmentAuth.serverAuthCredentialReason(error)), - ), - Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) => - failEnvironmentInternal("internal_error", error), - ), - ); - const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup, { - disableTracing: true, - }).pipe( - Effect.provide( - makeWsRpcLayer(session, previewAutomationBroker, previewLiveGateway).pipe( - Layer.provideMerge(RpcSerialization.layerJson), - Layer.provide(ProviderMaintenanceRunner.layer), - Layer.provide(Layer.succeed(ServerSelfUpdate.ServerSelfUpdate, serverSelfUpdate)), - Layer.provide( - SourceControlDiscovery.layer.pipe( - Layer.provide( - SourceControlProviderRegistry.layer.pipe( - Layer.provide( - Layer.mergeAll( - AzureDevOpsCli.layer, - BitbucketApi.layer, - GitHubCli.layer, - GitLabCli.layer, - ), - ), - Layer.provideMerge(GitVcsDriver.layer), - Layer.provide( - VcsDriverRegistry.layer.pipe(Layer.provide(VcsProjectConfig.layer)), +export const websocketRpcRouteLayer = Layer.unwrap( + Effect.gen(function* () { + const previewAutomationBroker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const previewLiveGateway = yield* PreviewLiveGateway.PreviewLiveGateway; + const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; + return HttpRouter.add( + "GET", + "/ws", + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const sessions = yield* SessionStore.SessionStore; + const session = yield* serverAuth.authenticateWebSocketUpgrade(request).pipe( + Effect.catchIf(EnvironmentAuth.isServerAuthCredentialError, (error) => + failEnvironmentAuthInvalid(EnvironmentAuth.serverAuthCredentialReason(error)), + ), + Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) => + failEnvironmentInternal("internal_error", error), + ), + ); + const rpcWebSocketHttpEffect = yield* RpcServer.toHttpEffectWebsocket(WsRpcGroup, { + disableTracing: true, + }).pipe( + Effect.provide( + makeWsRpcLayer(session, previewAutomationBroker, previewLiveGateway).pipe( + Layer.provideMerge(RpcSerialization.layerJson), + Layer.provide(ProviderMaintenanceRunner.layer), + Layer.provide(Layer.succeed(ServerSelfUpdate.ServerSelfUpdate, serverSelfUpdate)), + Layer.provide( + SourceControlDiscovery.layer.pipe( + Layer.provide( + SourceControlProviderRegistry.layer.pipe( + Layer.provide( + Layer.mergeAll( + AzureDevOpsCli.layer, + BitbucketApi.layer, + GitHubCli.layer, + GitLabCli.layer, ), ), + Layer.provideMerge(GitVcsDriver.layer), + Layer.provide( + VcsDriverRegistry.layer.pipe(Layer.provide(VcsProjectConfig.layer)), + ), ), - Layer.provide(VcsProcess.layer), ), + Layer.provide(VcsProcess.layer), ), ), ), - ); - return yield* Effect.acquireUseRelease( - sessions.markConnected(session.sessionId), - () => rpcWebSocketHttpEffect, - () => sessions.markDisconnected(session.sessionId), - ); - }).pipe( - Effect.catchTags({ - EnvironmentAuthInvalidError: HttpServerRespondable.toResponse, - EnvironmentInternalError: HttpServerRespondable.toResponse, - }), - ), - ); - }), - ); + ), + ); + return yield* Effect.acquireUseRelease( + sessions.markConnected(session.sessionId), + () => rpcWebSocketHttpEffect, + () => sessions.markDisconnected(session.sessionId), + ); + }).pipe( + Effect.catchTags({ + EnvironmentAuthInvalidError: HttpServerRespondable.toResponse, + EnvironmentInternalError: HttpServerRespondable.toResponse, + }), + ), + ); + }), +); From acf07412e5d949e2ef7e73705e413560339c6222 Mon Sep 17 00:00:00 2001 From: Diego Carlino <carlid.dev@gmail.com> Date: Sat, 1 Aug 2026 04:45:47 +0200 Subject: [PATCH 3/6] fix(preview): address correctness review --- .../src/features/annotations/model.test.ts | 18 +++++++- apps/mobile/src/features/annotations/model.ts | 13 ++++-- .../features/preview/ThreadPreviewPane.tsx | 20 ++++++-- .../preview/mobilePreviewDomBridge.test.ts | 3 ++ .../preview/mobilePreviewDomBridge.ts | 46 ++++++++++++++----- .../preview/mobilePreviewGatewayModel.test.ts | 1 + .../preview/mobilePreviewGatewayModel.ts | 6 +-- .../preview/previewLiveTarget.test.ts | 13 ++++++ .../src/features/preview/previewLiveTarget.ts | 6 ++- .../features/preview/previewPaneModel.test.ts | 26 +++++++++++ .../src/features/preview/previewPaneModel.ts | 6 +-- .../preview/previewReviewModel.test.ts | 8 ++++ .../features/preview/previewReviewModel.ts | 3 +- .../src/features/threads/ThreadFeed.tsx | 8 ++++ apps/server/src/preview/LiveGateway.test.ts | 20 ++++++++ apps/server/src/preview/LiveGateway.ts | 19 ++++++-- .../src/preview/LiveGatewayHttp.test.ts | 6 +++ apps/server/src/preview/LiveGatewayHttp.ts | 14 ++++++ 18 files changed, 204 insertions(+), 32 deletions(-) diff --git a/apps/mobile/src/features/annotations/model.test.ts b/apps/mobile/src/features/annotations/model.test.ts index 7d586d17c8b..6939d45d844 100644 --- a/apps/mobile/src/features/annotations/model.test.ts +++ b/apps/mobile/src/features/annotations/model.test.ts @@ -60,7 +60,7 @@ describe("annotation markup geometry", () => { { width: 4_000, height: 3_000 }, { maxEdge: 10_000, maxPixels: 1_000_000 }, ), - ).toEqual({ width: 1_155, height: 866 }); + ).toEqual({ width: 1_154, height: 866 }); expect(annotationExportSize({ width: 640, height: 480 })).toEqual({ width: 640, height: 480, @@ -185,6 +185,22 @@ describe("annotation markup document", () => { expect(eraseMarkupObjectAtPoint(document, { x: 0.3, y: 0.4 }).strokes).toEqual([]); }); + it("hit-tests a single-point stroke across its rendered width", () => { + const stroke = makeStroke({ + id: "dot-1", + color: "#f00", + width: 0.1, + points: [{ x: 0.5, y: 0.5 }], + }); + expect(stroke).not.toBeNull(); + const document = addStroke(EMPTY_MARKUP_DOCUMENT, stroke!); + + expect(hitTestMarkupObject(document, { x: 0.56, y: 0.5 }, 0.02)).toEqual({ + kind: "stroke", + id: "dot-1", + }); + }); + it("supports undo, redo, delete, and clear without mutating snapshots", () => { const initial = createMarkupHistory(EMPTY_MARKUP_DOCUMENT); const one = addPointCallout(initial.present, { diff --git a/apps/mobile/src/features/annotations/model.ts b/apps/mobile/src/features/annotations/model.ts index e90393b8e95..7931fc0f8b5 100644 --- a/apps/mobile/src/features/annotations/model.ts +++ b/apps/mobile/src/features/annotations/model.ts @@ -382,7 +382,10 @@ export function hitTestMarkupObject( continue; } const points = stroke.points; - if (points.length === 1 && squaredDistance(point, points[0]!) <= tolerance * tolerance) { + if ( + points.length === 1 && + squaredDistance(point, points[0]!) <= (tolerance + stroke.width / 2) ** 2 + ) { return { kind: "stroke", id: stroke.id }; } for (let index = 1; index < points.length; index += 1) { @@ -481,9 +484,13 @@ export function annotationExportSize( const edgeScale = maxEdge / Math.max(image.width, image.height); const pixelScale = Math.sqrt(maxPixels / (image.width * image.height)); const scale = Math.min(1, edgeScale, pixelScale); + const width = Math.max(1, Math.round(image.width * scale)); + const height = Math.max(1, Math.round(image.height * scale)); + if (width * height <= maxPixels) return { width, height }; + const adjustedScale = Math.sqrt(maxPixels / (image.width * image.height)); return { - width: Math.max(1, Math.round(image.width * scale)), - height: Math.max(1, Math.round(image.height * scale)), + width: Math.max(1, Math.floor(image.width * adjustedScale)), + height: Math.max(1, Math.floor(image.height * adjustedScale)), }; } diff --git a/apps/mobile/src/features/preview/ThreadPreviewPane.tsx b/apps/mobile/src/features/preview/ThreadPreviewPane.tsx index d9051eb6627..b53ac7e0a21 100644 --- a/apps/mobile/src/features/preview/ThreadPreviewPane.tsx +++ b/apps/mobile/src/features/preview/ThreadPreviewPane.tsx @@ -255,8 +255,15 @@ export function ThreadPreviewPane(props: { useEffect(() => { createTabRequestIdRef.current += 1; navigateTabRequestIdRef.current += 1; + captureRequestIdRef.current += 1; gatewayRequestIdRef.current += 1; requestedGatewayRef.current = null; + setSnapshot(null); + setShowingDesktopSnapshot(false); + setMarkupVisible(false); + setMarkupModalSeed(null); + setCapturing(false); + setCapturingLocalMarkup(false); setGatewayTarget(null); setGatewayError(null); setGatewayOpening(false); @@ -644,22 +651,27 @@ export function ThreadPreviewPane(props: { setCaptureError("Wait for Browser to finish loading."); return; } + const requestId = captureRequestIdRef.current + 1; + const tabId = selectedTabIdRef.current; + captureRequestIdRef.current = requestId; setCapturingLocalMarkup(true); try { const seed = await livePreview.captureForMarkup(); + if (captureRequestIdRef.current !== requestId || selectedTabIdRef.current !== tabId) return; setMarkupModalSeed(seed); setMarkupVisible(true); } catch (cause) { + if (captureRequestIdRef.current !== requestId || selectedTabIdRef.current !== tabId) return; setCaptureError(cause instanceof Error ? cause.message : "Browser could not be captured."); } finally { - setCapturingLocalMarkup(false); + if (captureRequestIdRef.current === requestId && selectedTabIdRef.current === tabId) { + setCapturingLocalMarkup(false); + } } }, [desktopMarkupSeed.error, desktopMarkupSeed.value, showingDesktopSnapshot]); const handleMarkupDone = useCallback( async (attachment: DraftComposerImageAttachment) => { - setMarkupVisible(false); - setMarkupModalSeed(null); try { const result = await mergeComposerDraftContent( scopedThreadKey(props.environmentId, props.threadId), @@ -669,6 +681,8 @@ export function ThreadPreviewPane(props: { setCaptureError("This message already has the maximum of 8 image attachments."); return; } + setMarkupVisible(false); + setMarkupModalSeed(null); setMarkupSeedGeneration((current) => current + 1); setCaptureError(null); setAttachmentNotice("Annotated snapshot added to your message."); diff --git a/apps/mobile/src/features/preview/mobilePreviewDomBridge.test.ts b/apps/mobile/src/features/preview/mobilePreviewDomBridge.test.ts index d5a19401ce9..4218d26fe3e 100644 --- a/apps/mobile/src/features/preview/mobilePreviewDomBridge.test.ts +++ b/apps/mobile/src/features/preview/mobilePreviewDomBridge.test.ts @@ -143,6 +143,9 @@ describe("mobile preview DOM bridge", () => { expect(script).toContain("window.ReactNativeWebView?.postMessage"); expect(script).toContain('kind: "t3.preview.capture-error"'); expect(script).toContain(".slice(0, 200)"); + expect(script).toContain("document.querySelectorAll(selector)"); + expect(script).toContain("Number.POSITIVE_INFINITY"); + expect(script).toContain("current = current.parentElement"); expect(script.trimEnd().endsWith("true;")).toBe(true); }); }); diff --git a/apps/mobile/src/features/preview/mobilePreviewDomBridge.ts b/apps/mobile/src/features/preview/mobilePreviewDomBridge.ts index 117dcd768fd..bdc1d8149b9 100644 --- a/apps/mobile/src/features/preview/mobilePreviewDomBridge.ts +++ b/apps/mobile/src/features/preview/mobilePreviewDomBridge.ts @@ -159,15 +159,27 @@ export function mobilePreviewDomCaptureScript(requestId: string): string { const requestId = ${encodedRequestId}; try { const selectorFor = (element) => { - if (element.id) return "#" + CSS.escape(element.id); + const identifiesElement = (selector) => { + try { + const matches = document.querySelectorAll(selector); + return matches.length === 1 && matches[0] === element; + } catch { + return false; + } + }; + if (element.id) { + const selector = "#" + CSS.escape(element.id); + if (identifiesElement(selector)) return selector; + } for (const attribute of ["data-testid", "name"]) { const value = element.getAttribute(attribute); if (value) { - return element.tagName.toLowerCase() + "[" + attribute + "=" + JSON.stringify(value) + "]"; + const selector = element.tagName.toLowerCase() + "[" + attribute + "=" + JSON.stringify(value) + "]"; + if (identifiesElement(selector)) return selector; } } - const buildParts = (current, parts = []) => { - if (!current || current.nodeType !== Node.ELEMENT_NODE || parts.length >= 8) { + const buildParts = (current, parts = [], maximumDepth = 8) => { + if (!current || current.nodeType !== Node.ELEMENT_NODE || parts.length >= maximumDepth) { return parts; } const parent = current.parentElement; @@ -178,22 +190,32 @@ export function mobilePreviewDomCaptureScript(requestId: string): string { const part = siblings.length > 1 ? base + ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")" : base; - return buildParts(parent, [part, ...parts]); + return buildParts(parent, [part, ...parts], maximumDepth); }; - return buildParts(element).join(" > "); + const compactSelector = buildParts(element).join(" > "); + if (identifiesElement(compactSelector)) return compactSelector; + const exactSelector = buildParts(element, [], Number.POSITIVE_INFINITY).join(" > "); + return identifiesElement(exactSelector) ? exactSelector : ""; }; const visible = (element) => { - const style = getComputedStyle(element); const rect = element.getBoundingClientRect(); - return style.visibility !== "hidden" - && style.display !== "none" - && style.opacity !== "0" - && rect.width > 0 + if (!(rect.width > 0 && rect.height > 0 && rect.right > 0 && rect.bottom > 0 && rect.left < window.innerWidth - && rect.top < window.innerHeight; + && rect.top < window.innerHeight)) return false; + for (let current = element; current; current = current.parentElement) { + const style = getComputedStyle(current); + if ( + style.visibility === "hidden" || + style.display === "none" || + Number(style.opacity) <= 0 + ) { + return false; + } + } + return true; }; const elements = Array.from(document.querySelectorAll( "a[href],button,input,textarea,select,[role],[tabindex]" diff --git a/apps/mobile/src/features/preview/mobilePreviewGatewayModel.test.ts b/apps/mobile/src/features/preview/mobilePreviewGatewayModel.test.ts index 78557535bcc..ce22e2bc1fd 100644 --- a/apps/mobile/src/features/preview/mobilePreviewGatewayModel.test.ts +++ b/apps/mobile/src/features/preview/mobilePreviewGatewayModel.test.ts @@ -54,6 +54,7 @@ describe("mobile preview gateway model", () => { "https://attacker.example/bootstrap", "api/preview-gateway/bootstrap/ticket", "/api/not-the-preview-gateway/ticket", + "/api/preview-gateway/bootstrap/../other", ]) { expect(() => resolveMobilePreviewGatewayUri({ diff --git a/apps/mobile/src/features/preview/mobilePreviewGatewayModel.ts b/apps/mobile/src/features/preview/mobilePreviewGatewayModel.ts index e5880fa7073..89d69dfd886 100644 --- a/apps/mobile/src/features/preview/mobilePreviewGatewayModel.ts +++ b/apps/mobile/src/features/preview/mobilePreviewGatewayModel.ts @@ -22,10 +22,7 @@ export function resolveMobilePreviewGatewayUri(input: { readonly environmentHttpBaseUrl: string; readonly relativeUrl: string; }): string { - if ( - !input.relativeUrl.startsWith(GATEWAY_BOOTSTRAP_PREFIX) || - input.relativeUrl.startsWith("//") - ) { + if (!input.relativeUrl.startsWith("/") || input.relativeUrl.startsWith("//")) { throw new Error(INVALID_GATEWAY_ADDRESS); } let environmentUrl: URL; @@ -39,6 +36,7 @@ export function resolveMobilePreviewGatewayUri(input: { if ( (environmentUrl.protocol !== "http:" && environmentUrl.protocol !== "https:") || gatewayUrl.origin !== environmentUrl.origin || + !gatewayUrl.pathname.startsWith(GATEWAY_BOOTSTRAP_PREFIX) || (gatewayUrl.protocol !== "http:" && gatewayUrl.protocol !== "https:") ) { throw new Error(INVALID_GATEWAY_ADDRESS); diff --git a/apps/mobile/src/features/preview/previewLiveTarget.test.ts b/apps/mobile/src/features/preview/previewLiveTarget.test.ts index 23fb5af4f9d..d5adfdbc76a 100644 --- a/apps/mobile/src/features/preview/previewLiveTarget.test.ts +++ b/apps/mobile/src/features/preview/previewLiveTarget.test.ts @@ -29,6 +29,19 @@ describe("resolveMobilePreviewLiveTarget", () => { }); }); + it("treats a DNS-root-qualified localhost as desktop loopback", () => { + expect( + resolveMobilePreviewLiveTarget({ + previewUrl: "http://localhost.:5173/dashboard", + environmentHttpBaseUrl: "http://192.168.1.25:3773", + }), + ).toEqual({ + kind: "available", + uri: "http://192.168.1.25:5173/dashboard", + resolution: "environment-private-network", + }); + }); + it("maps desktop localhost onto a Tailscale host", () => { expect( resolveMobilePreviewLiveTarget({ diff --git a/apps/mobile/src/features/preview/previewLiveTarget.ts b/apps/mobile/src/features/preview/previewLiveTarget.ts index 7036ab29ca9..d7a08fbbc1c 100644 --- a/apps/mobile/src/features/preview/previewLiveTarget.ts +++ b/apps/mobile/src/features/preview/previewLiveTarget.ts @@ -12,7 +12,11 @@ export type MobilePreviewLiveTarget = readonly detail: string; }; -const normalizeHostname = (host: string): string => host.toLowerCase().replace(/^\[|\]$/g, ""); +const normalizeHostname = (host: string): string => + host + .toLowerCase() + .replace(/^\[|\]$/g, "") + .replace(/\.$/u, ""); const parseIpv4Address = (host: string): readonly number[] | null => { const parts = normalizeHostname(host).split(".").map(Number); diff --git a/apps/mobile/src/features/preview/previewPaneModel.test.ts b/apps/mobile/src/features/preview/previewPaneModel.test.ts index a6095c5d014..63f7a2ffdea 100644 --- a/apps/mobile/src/features/preview/previewPaneModel.test.ts +++ b/apps/mobile/src/features/preview/previewPaneModel.test.ts @@ -190,6 +190,32 @@ describe("mobile preview pane state", () => { ).toBe("http://localhost:5173/old"); }); + it("reconciles epoch changes without letting an old event override the new list", () => { + const oldEpochEvent = navigatedEvent({ + serverEpoch: "epoch-1", + revision: 99, + }); + expect( + previewEventRequiresSessionRefresh({ + event: oldEpochEvent, + threadId, + serverEpoch: "epoch-2", + revision: 1, + }), + ).toBe(true); + expect( + previewLiveUrlForSelection({ + selectedTabId: "tab-1", + selectedSession: session, + snapshot: null, + latestEvent: oldEpochEvent, + threadId, + serverEpoch: "epoch-2", + revision: 1, + }), + ).toBe("http://localhost:5173/old"); + }); + it("lets an optimistic navigation supersede the new tab's newer Idle event", () => { const optimistic = { ...session, diff --git a/apps/mobile/src/features/preview/previewPaneModel.ts b/apps/mobile/src/features/preview/previewPaneModel.ts index ed615aafadb..976daec8cb8 100644 --- a/apps/mobile/src/features/preview/previewPaneModel.ts +++ b/apps/mobile/src/features/preview/previewPaneModel.ts @@ -30,8 +30,7 @@ function eventIsNewerThanList(input: { return ( input.serverEpoch === null || input.revision === null || - input.event.serverEpoch !== input.serverEpoch || - input.event.revision > input.revision + (input.event.serverEpoch === input.serverEpoch && input.event.revision > input.revision) ); } @@ -85,8 +84,9 @@ export function previewEventRequiresSessionRefresh(input: { readonly revision: number | null; }): boolean { if (input.event.threadId !== input.threadId) return false; + if (input.serverEpoch !== null && input.event.serverEpoch !== input.serverEpoch) return true; if (!eventIsNewerThanList(input)) return false; - return input.event.serverEpoch !== input.serverEpoch || input.event.type !== "resized"; + return input.event.type !== "resized"; } export function previewLiveUrlForSelection(input: { diff --git a/apps/mobile/src/features/preview/previewReviewModel.test.ts b/apps/mobile/src/features/preview/previewReviewModel.test.ts index 4b645f75ae6..8a2bf22b882 100644 --- a/apps/mobile/src/features/preview/previewReviewModel.test.ts +++ b/apps/mobile/src/features/preview/previewReviewModel.test.ts @@ -192,5 +192,13 @@ describe("preview review model", () => { revision: snapshot.previewRevision + 1, }), ).toBe(false); + expect( + previewSnapshotIsStale(snapshot, { + ...baseEvent, + threadId: ThreadId.make("thread-2"), + tabId: "tab-2", + serverEpoch: "server-epoch-2", + }), + ).toBe(true); }); }); diff --git a/apps/mobile/src/features/preview/previewReviewModel.ts b/apps/mobile/src/features/preview/previewReviewModel.ts index 26ba25b889d..96dda59d307 100644 --- a/apps/mobile/src/features/preview/previewReviewModel.ts +++ b/apps/mobile/src/features/preview/previewReviewModel.ts @@ -198,10 +198,11 @@ export function previewSnapshotIsStale( snapshot: PreviewReviewSnapshot, event: PreviewEvent, ): boolean { + if (event.serverEpoch !== snapshot.serverEpoch) return true; return ( event.threadId === snapshot.threadId && event.tabId === snapshot.tabId && - (event.serverEpoch !== snapshot.serverEpoch || event.revision > snapshot.previewRevision) + event.revision > snapshot.previewRevision ); } diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index 39cfd74ed32..1bee79c763b 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -1185,6 +1185,14 @@ const PreviewAnnotationCard = memo(function PreviewAnnotationCard(props: { +{hiddenCalloutCount} more callout{hiddenCalloutCount === 1 ? "" : "s"} </Text> ) : null} + {expanded + ? Array.from(new Set(props.annotation.styleChanges)).map((change) => ( + <View key={`${props.annotation.id}:style:${change}`} className="flex-row gap-1.5"> + <Text className="font-t3-bold text-xs text-white">Style</Text> + <Text className="min-w-0 flex-1 text-xs leading-snug text-white/90">{change}</Text> + </View> + )) + : null} <Text className="text-2xs text-white/65" numberOfLines={1}> {targetSummary} </Text> diff --git a/apps/server/src/preview/LiveGateway.test.ts b/apps/server/src/preview/LiveGateway.test.ts index b41b3a642b9..fd44a016818 100644 --- a/apps/server/src/preview/LiveGateway.test.ts +++ b/apps/server/src/preview/LiveGateway.test.ts @@ -122,6 +122,26 @@ it.layer(NodeServices.layer)("PreviewLiveGateway", (it) => { }), ); + it.effect("keeps pending bootstraps for different tabs in the same session", () => + Effect.gen(function* () { + const gateway = yield* make; + const first = yield* gateway.issue({ + sessionId, + snapshot: snapshot("http://localhost:5173/first"), + }); + const second = yield* gateway.issue({ + sessionId, + snapshot: { + ...snapshot("http://localhost:5173/second"), + tabId: PreviewTabId.make("tab-live-gateway-2"), + }, + }); + + expect(yield* gateway.consumeBootstrap(bootstrapToken(first.relativeUrl))).not.toBeNull(); + expect(yield* gateway.consumeBootstrap(bootstrapToken(second.relativeUrl))).not.toBeNull(); + }), + ); + it.effect("expires bootstraps and leases and supports explicit tab/session revocation", () => Effect.gen(function* () { const gateway = yield* make; diff --git a/apps/server/src/preview/LiveGateway.ts b/apps/server/src/preview/LiveGateway.ts index 1349d87342c..8058a174d4f 100644 --- a/apps/server/src/preview/LiveGateway.ts +++ b/apps/server/src/preview/LiveGateway.ts @@ -81,12 +81,19 @@ const withoutSession = (state: LiveGatewayState, sessionId: AuthSessionId): Live leases: new Map([...state.leases].filter(([, lease]) => lease.sessionId !== sessionId)), }); -const withoutSessionTickets = ( +const withoutTargetTickets = ( state: LiveGatewayState, - sessionId: AuthSessionId, + target: Pick<LiveGatewayGrant, "sessionId" | "threadId" | "tabId">, ): LiveGatewayState => ({ ...state, - tickets: new Map([...state.tickets].filter(([, ticket]) => ticket.sessionId !== sessionId)), + tickets: new Map( + [...state.tickets].filter( + ([, ticket]) => + ticket.sessionId !== target.sessionId || + ticket.threadId !== target.threadId || + ticket.tabId !== target.tabId, + ), + ), }); const withoutTarget = ( @@ -244,7 +251,11 @@ export const make = Effect.gen(function* PreviewLiveGatewayMake() { input.sessionExpiresAt ?? now + LIVE_GATEWAY_BOOTSTRAP_TTL_MS, ); yield* SynchronizedRef.modifyEffect(stateRef, (current) => { - const state = withoutSessionTickets(withoutExpired(current, now), input.sessionId); + const state = withoutTargetTickets(withoutExpired(current, now), { + sessionId: input.sessionId, + threadId: ThreadId.make(input.snapshot.threadId), + tabId: input.snapshot.tabId, + }); const tickets = new Map(state.tickets); tickets.set(digest, { sessionId: input.sessionId, diff --git a/apps/server/src/preview/LiveGatewayHttp.test.ts b/apps/server/src/preview/LiveGatewayHttp.test.ts index 04f5bd5521f..33913949e40 100644 --- a/apps/server/src/preview/LiveGatewayHttp.test.ts +++ b/apps/server/src/preview/LiveGatewayHttp.test.ts @@ -126,10 +126,12 @@ it("strips environment credentials while preserving application cookies and requ "cf-ray": "cloudflare-metadata", cookie: "raw-cookie-header", dpop: "environment-proof", + connection: "keep-alive, x-internal", host: "environment.example", origin: "https://environment.example", referer: "https://environment.example/app/page?q=1", "sec-websocket-key": "downstream-key", + "x-internal": "connection-specific", "x-forwarded-host": "environment.example", }, }); @@ -146,6 +148,7 @@ it("strips environment credentials while preserving application cookies and requ expect(headers["cf-ray"]).toBeUndefined(); expect(headers["x-forwarded-host"]).toBeUndefined(); expect(headers["sec-websocket-key"]).toBeUndefined(); + expect(headers["x-internal"]).toBeUndefined(); expect(headers.cookie).toBe("app_session=application-value"); expect(headers.host).toBe("127.0.0.1:5173"); expect(headers.origin).toBe("http://127.0.0.1:5173"); @@ -159,8 +162,10 @@ it("rewrites loopback redirects and strips transport headers after fetch decompr "access-control-allow-origin": "http://127.0.0.1:5173", "content-encoding": "gzip", "content-length": "123", + connection: "keep-alive, x-private", location: "http://localhost:5173/next?q=1#fragment", "set-cookie": "secret=value", + "x-private": "connection-specific", }), requestUrl: new URL("https://environment.example/current"), upstreamOrigin: "http://127.0.0.1:5173", @@ -171,6 +176,7 @@ it("rewrites loopback redirects and strips transport headers after fetch decompr expect(headers["content-encoding"]).toBeUndefined(); expect(headers["content-length"]).toBeUndefined(); expect(headers["set-cookie"]).toBeUndefined(); + expect(headers["x-private"]).toBeUndefined(); }); it("prevents upstream cookies from replacing gateway or environment credentials", () => { diff --git a/apps/server/src/preview/LiveGatewayHttp.ts b/apps/server/src/preview/LiveGatewayHttp.ts index 38962d768bb..841a72b5fe4 100644 --- a/apps/server/src/preview/LiveGatewayHttp.ts +++ b/apps/server/src/preview/LiveGatewayHttp.ts @@ -47,6 +47,7 @@ const HOP_BY_HOP_HEADERS = [ "upgrade", ] as const; const PRIVATE_REQUEST_HEADERS = new Set([...HOP_BY_HOP_HEADERS, "authorization", "dpop", "host"]); +const HTTP_HEADER_NAME_PATTERN = /^[!#$%&'*+\-.^_`|~\dA-Za-z]+$/u; const bootstrapResponseHeaders = { "cache-control": "no-store", @@ -55,6 +56,15 @@ const bootstrapResponseHeaders = { "x-content-type-options": "nosniff", } as const; +function connectionHeaderNames(value: string | undefined): ReadonlySet<string> { + return new Set( + (value ?? "") + .split(",") + .map((name) => name.trim().toLowerCase()) + .filter((name) => HTTP_HEADER_NAME_PATTERN.test(name)), + ); +} + export function isLiveGatewayControlCookie( name: string, environmentSessionCookieName: string, @@ -102,9 +112,11 @@ export function liveGatewayUpstreamHeaders(input: { readonly environmentSessionCookieName: string; }): Headers.Headers { const forwarded: Record<string, string> = {}; + const connectionSpecificHeaders = connectionHeaderNames(input.request.headers.connection); for (const [name, value] of Object.entries(input.request.headers)) { if ( PRIVATE_REQUEST_HEADERS.has(name) || + connectionSpecificHeaders.has(name) || name === "cookie" || name === "forwarded" || name.startsWith("cf-") || @@ -167,8 +179,10 @@ export function liveGatewayResponseHeaders(input: { readonly requestUrl: URL; readonly upstreamOrigin: string; }): Headers.Headers { + const connectionSpecificHeaders = connectionHeaderNames(input.headers.connection); let headers = Headers.removeMany(input.headers, [ ...HOP_BY_HOP_HEADERS, + ...connectionSpecificHeaders, "set-cookie", "content-encoding", "content-length", From 09d985b6b49cd58eaf08721a4052daefb8f8170c Mon Sep 17 00:00:00 2001 From: Diego Carlino <carlid.dev@gmail.com> Date: Sat, 1 Aug 2026 04:47:29 +0200 Subject: [PATCH 4/6] fix(preview): retain snapshot decode causes --- apps/server/src/ws.ts | 3 ++- packages/contracts/src/previewAutomation.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index b8599a5bc6a..a10ce1b15c2 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -469,10 +469,11 @@ const makeWsRpcLayer = ( }); const snapshot = yield* decodePreviewAutomationReviewSnapshot(rawSnapshot).pipe( Effect.mapError( - () => + (cause) => new PreviewReviewSnapshotMalformedError({ threadId: input.threadId, tabId: input.tabId, + cause, }), ), ); diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 130cc0b7938..b7f0034a0b3 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -677,6 +677,7 @@ export class PreviewReviewSnapshotMalformedError extends Schema.TaggedErrorClass { threadId: ThreadId, tabId: PreviewTabId, + cause: Schema.optional(Schema.Defect()), }, ) { override get message(): string { From 5fd6b39acf03904c105ec508a7a4c1e64522c36f Mon Sep 17 00:00:00 2001 From: Diego Carlino <carlid.dev@gmail.com> Date: Sat, 1 Aug 2026 04:51:22 +0200 Subject: [PATCH 5/6] fix(mobile): hit-test markup in pixel space --- .../features/annotations/ImageMarkupModal.tsx | 4 +- .../src/features/annotations/model.test.ts | 30 ++++++-- apps/mobile/src/features/annotations/model.ts | 70 +++++++++++++------ 3 files changed, 76 insertions(+), 28 deletions(-) diff --git a/apps/mobile/src/features/annotations/ImageMarkupModal.tsx b/apps/mobile/src/features/annotations/ImageMarkupModal.tsx index 9579377b146..9daf978e851 100644 --- a/apps/mobile/src/features/annotations/ImageMarkupModal.tsx +++ b/apps/mobile/src/features/annotations/ImageMarkupModal.tsx @@ -607,7 +607,7 @@ export function ImageMarkupModal(props: ImageMarkupModalProps) { setSelection({ kind: "callout", id }); return; } - setSelection(hitTestMarkupObject(history.present, point)); + setSelection(hitTestMarkupObject(history.present, point, fit)); }, [commitCurrentComment, commitDocument, fit, history.present, semanticElements, tool], ); @@ -645,7 +645,7 @@ export function ImageMarkupModal(props: ImageMarkupModalProps) { if (fit.width <= 0 || fit.height <= 0) return; const point = normalizedPoint({ x, y }, fit); commitCurrentComment(); - commitDocument((document) => eraseMarkupObjectAtPoint(document, point)); + commitDocument((document) => eraseMarkupObjectAtPoint(document, point, fit)); setSelection(null); }, [commitCurrentComment, commitDocument, fit], diff --git a/apps/mobile/src/features/annotations/model.test.ts b/apps/mobile/src/features/annotations/model.test.ts index 6939d45d844..0025187255b 100644 --- a/apps/mobile/src/features/annotations/model.test.ts +++ b/apps/mobile/src/features/annotations/model.test.ts @@ -174,15 +174,17 @@ describe("annotation markup document", () => { id: "point-1", point: { x: 0.8, y: 0.2 }, }); - expect(hitTestMarkupObject(document, { x: 0.3, y: 0.4 })).toEqual({ + expect(hitTestMarkupObject(document, { x: 0.3, y: 0.4 }, { width: 1, height: 1 })).toEqual({ kind: "stroke", id: "stroke-1", }); - expect(hitTestMarkupObject(document, { x: 0.8, y: 0.2 })).toEqual({ + expect(hitTestMarkupObject(document, { x: 0.8, y: 0.2 }, { width: 1, height: 1 })).toEqual({ kind: "callout", id: "point-1", }); - expect(eraseMarkupObjectAtPoint(document, { x: 0.3, y: 0.4 }).strokes).toEqual([]); + expect( + eraseMarkupObjectAtPoint(document, { x: 0.3, y: 0.4 }, { width: 1, height: 1 }).strokes, + ).toEqual([]); }); it("hit-tests a single-point stroke across its rendered width", () => { @@ -195,12 +197,32 @@ describe("annotation markup document", () => { expect(stroke).not.toBeNull(); const document = addStroke(EMPTY_MARKUP_DOCUMENT, stroke!); - expect(hitTestMarkupObject(document, { x: 0.56, y: 0.5 }, 0.02)).toEqual({ + expect( + hitTestMarkupObject(document, { x: 0.56, y: 0.5 }, { width: 1, height: 1 }, 0.02), + ).toEqual({ kind: "stroke", id: "dot-1", }); }); + it("uses a circular pixel-space hit area on non-square images", () => { + const document = addPointCallout(EMPTY_MARKUP_DOCUMENT, { + id: "point-wide", + point: { x: 0.5, y: 0.5 }, + }); + const wideImage = { width: 4_000, height: 500 }; + + expect(hitTestMarkupObject(document, { x: 0.5025, y: 0.5 }, wideImage)).toEqual({ + kind: "callout", + id: "point-wide", + }); + expect(hitTestMarkupObject(document, { x: 0.5, y: 0.52 }, wideImage)).toEqual({ + kind: "callout", + id: "point-wide", + }); + expect(hitTestMarkupObject(document, { x: 0.51, y: 0.5 }, wideImage)).toBeNull(); + }); + it("supports undo, redo, delete, and clear without mutating snapshots", () => { const initial = createMarkupHistory(EMPTY_MARKUP_DOCUMENT); const one = addPointCallout(initial.present, { diff --git a/apps/mobile/src/features/annotations/model.ts b/apps/mobile/src/features/annotations/model.ts index 7931fc0f8b5..e5d11065464 100644 --- a/apps/mobile/src/features/annotations/model.ts +++ b/apps/mobile/src/features/annotations/model.ts @@ -323,9 +323,10 @@ export function clearMarkupDocument(document: MarkupDocument): MarkupDocument { function squaredDistance( left: PreviewAnnotationNormalizedPoint, right: PreviewAnnotationNormalizedPoint, + size: MarkupSize, ): number { - const dx = left.x - right.x; - const dy = left.y - right.y; + const dx = (left.x - right.x) * size.width; + const dy = (left.y - right.y) * size.height; return dx * dx + dy * dy; } @@ -333,65 +334,89 @@ function squaredDistanceToSegment( point: PreviewAnnotationNormalizedPoint, start: PreviewAnnotationNormalizedPoint, end: PreviewAnnotationNormalizedPoint, + size: MarkupSize, ): number { - const dx = end.x - start.x; - const dy = end.y - start.y; - if (dx === 0 && dy === 0) return squaredDistance(point, start); + const dx = (end.x - start.x) * size.width; + const dy = (end.y - start.y) * size.height; + if (dx === 0 && dy === 0) return squaredDistance(point, start, size); const progress = Math.max( 0, - Math.min(1, ((point.x - start.x) * dx + (point.y - start.y) * dy) / (dx * dx + dy * dy)), + Math.min( + 1, + ((point.x - start.x) * size.width * dx + (point.y - start.y) * size.height * dy) / + (dx * dx + dy * dy), + ), + ); + return squaredDistance( + point, + { + x: start.x + progress * (end.x - start.x), + y: start.y + progress * (end.y - start.y), + }, + size, ); - return squaredDistance(point, { - x: start.x + progress * dx, - y: start.y + progress * dy, - }); } function pointTouchesRect( point: PreviewAnnotationNormalizedPoint, rect: PreviewAnnotationNormalizedRect, - tolerance: number, + horizontalTolerance: number, + verticalTolerance: number, ): boolean { return ( - point.x >= rect.x - tolerance && - point.x <= rect.x + rect.width + tolerance && - point.y >= rect.y - tolerance && - point.y <= rect.y + rect.height + tolerance + point.x >= rect.x - horizontalTolerance && + point.x <= rect.x + rect.width + horizontalTolerance && + point.y >= rect.y - verticalTolerance && + point.y <= rect.y + rect.height + verticalTolerance ); } export function hitTestMarkupObject( document: MarkupDocument, point: PreviewAnnotationNormalizedPoint, + size: MarkupSize, tolerance = 0.025, ): MarkupSelection | null { + if (size.width <= 0 || size.height <= 0) return null; + const shortSide = Math.min(size.width, size.height); + const tolerancePixels = tolerance * shortSide; + const horizontalTolerance = tolerancePixels / size.width; + const verticalTolerance = tolerancePixels / size.height; for (const callout of document.callouts.toReversed()) { if (callout.anchor.kind === "point") { - if (squaredDistance(point, callout.anchor.point) <= tolerance * tolerance) { + if (squaredDistance(point, callout.anchor.point, size) <= tolerancePixels * tolerancePixels) { return { kind: "callout", id: callout.id }; } continue; } - if (pointTouchesRect(point, callout.anchor.rect, tolerance)) { + if (pointTouchesRect(point, callout.anchor.rect, horizontalTolerance, verticalTolerance)) { return { kind: "callout", id: callout.id }; } } for (const stroke of document.strokes.toReversed()) { - if (!pointTouchesRect(point, stroke.bounds, tolerance + stroke.width / 2)) { + const strokeTolerancePixels = (tolerance + stroke.width / 2) * shortSide; + if ( + !pointTouchesRect( + point, + stroke.bounds, + strokeTolerancePixels / size.width, + strokeTolerancePixels / size.height, + ) + ) { continue; } const points = stroke.points; if ( points.length === 1 && - squaredDistance(point, points[0]!) <= (tolerance + stroke.width / 2) ** 2 + squaredDistance(point, points[0]!, size) <= strokeTolerancePixels * strokeTolerancePixels ) { return { kind: "stroke", id: stroke.id }; } for (let index = 1; index < points.length; index += 1) { if ( - squaredDistanceToSegment(point, points[index - 1]!, points[index]!) <= - (tolerance + stroke.width / 2) ** 2 + squaredDistanceToSegment(point, points[index - 1]!, points[index]!, size) <= + strokeTolerancePixels * strokeTolerancePixels ) { return { kind: "stroke", id: stroke.id }; } @@ -404,9 +429,10 @@ export function hitTestMarkupObject( export function eraseMarkupObjectAtPoint( document: MarkupDocument, point: PreviewAnnotationNormalizedPoint, + size: MarkupSize, tolerance = 0.035, ): MarkupDocument { - return deleteMarkupObject(document, hitTestMarkupObject(document, point, tolerance)); + return deleteMarkupObject(document, hitTestMarkupObject(document, point, size, tolerance)); } export function createMarkupHistory(document: MarkupDocument): MarkupHistory { From ba8208908d6f139648a7f60199d847117006631f Mon Sep 17 00:00:00 2001 From: Diego Carlino <carlid.dev@gmail.com> Date: Sat, 1 Aug 2026 04:56:08 +0200 Subject: [PATCH 6/6] refactor(server): align gateway layer naming --- apps/server/src/preview/LiveGateway.ts | 2 +- apps/server/src/server.test.ts | 16 ++++++---------- apps/server/src/server.ts | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/apps/server/src/preview/LiveGateway.ts b/apps/server/src/preview/LiveGateway.ts index 8058a174d4f..7e459c24544 100644 --- a/apps/server/src/preview/LiveGateway.ts +++ b/apps/server/src/preview/LiveGateway.ts @@ -382,4 +382,4 @@ export const makeLive = Effect.gen(function* PreviewLiveGatewayMakeLive() { return gateway; }); -export const liveLayer = Layer.effect(PreviewLiveGateway, makeLive); +export const layer = Layer.effect(PreviewLiveGateway, makeLive); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 5722475f9f8..3231bafa31d 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -93,11 +93,7 @@ import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; import * as ServerSettings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import * as PreviewManager from "./preview/Manager.ts"; -import { - LIVE_GATEWAY_COOKIE_NAME, - LIVE_GATEWAY_HTTP_COOKIE_NAME, - liveLayer as previewLiveGatewayLayer, -} from "./preview/LiveGateway.ts"; +import * as PreviewLiveGateway from "./preview/LiveGateway.ts"; import { LIVE_GATEWAY_EXPIRED_STATUS, liveGatewayCookieName } from "./preview/LiveGatewayHttp.ts"; import * as PortScanner from "./preview/PortScanner.ts"; import * as BrowserTraceCollector from "./observability/BrowserTraceCollector.ts"; @@ -564,7 +560,7 @@ const buildAppUnderTest = (options?: { }).pipe( Layer.provide( Layer.mergeAll( - previewLiveGatewayLayer, + PreviewLiveGateway.layer, Layer.mock(Keybindings.Keybindings)({ loadConfigState: Effect.succeed({ keybindings: [], @@ -4364,8 +4360,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => { response.setHeader("set-cookie", [ "app_session=server; Domain=127.0.0.1; Path=/", "t3_session=upstream-attack; Path=/", - `${LIVE_GATEWAY_COOKIE_NAME}=upstream-attack; Secure; Path=/`, - `${LIVE_GATEWAY_HTTP_COOKIE_NAME}=upstream-http-attack; Path=/`, + `${PreviewLiveGateway.LIVE_GATEWAY_COOKIE_NAME}=upstream-attack; Secure; Path=/`, + `${PreviewLiveGateway.LIVE_GATEWAY_HTTP_COOKIE_NAME}=upstream-http-attack; Path=/`, ]); if (request.url?.startsWith("/upstream-auth")) { response.statusCode = 401; @@ -4500,8 +4496,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => { assert.include(upstreamCookies, "app_session=server"); assert.notInclude(upstreamCookies.toLowerCase(), "domain="); assert.notInclude(upstreamCookies, "t3_session"); - assert.notInclude(upstreamCookies, LIVE_GATEWAY_COOKIE_NAME); - assert.notInclude(upstreamCookies, LIVE_GATEWAY_HTTP_COOKIE_NAME); + assert.notInclude(upstreamCookies, PreviewLiveGateway.LIVE_GATEWAY_COOKIE_NAME); + assert.notInclude(upstreamCookies, PreviewLiveGateway.LIVE_GATEWAY_HTTP_COOKIE_NAME); const appAuth = yield* fetchEffect("/upstream-auth", { headers: { cookie: gatewayCookie }, diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 063b56a0de0..882d049f453 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -585,7 +585,7 @@ export const makeServerLayer = Layer.unwrap( ); return serverApplicationLayer.pipe( - Layer.provide(PreviewLiveGateway.liveLayer), + Layer.provide(PreviewLiveGateway.layer), Layer.provideMerge(RuntimeServicesLive), Layer.provideMerge(serverRelayBrokerTracingLayer), Layer.provideMerge(HttpResponseCompressionLive),