fix(preview): add an accessible view for streamed browser tabs

apps/mobile/metro.config.js

@@ -93,7 +93,7 @@ async function prepareStreamScripts() {
       ["packages/client-runtime/src/device", ["stream.ts", "hubAccess.ts"], generateAll],
       [
         "packages/client-runtime/src/preview",
-        ["serverBrowserStream.ts"],
+        ["serverBrowserStream.ts", "accessibleDocument.ts"],
         generatePreviewStreamScript,
       ],
     ]) {

apps/mobile/src/features/browser/PreviewStreamWebView.tsx

@@ -164,6 +164,8 @@ function AuthorizedPreviewStream({
   ...props
 }: PreviewStreamConfiguration & NativeStreamBridge) {
   const [attempt, setAttempt] = useState(0);
+  // Outlives reconnects, which remount the document view.
+  const [accessible, setAccessible] = useState(false);
   const [previousAccess, setPreviousAccess] = useState(props.access);
   // Wait for refreshed access, including cookie credentials with unchanged JSON.
   if (props.access !== previousAccess) {
@@ -185,6 +187,8 @@ function AuthorizedPreviewStream({
       {...props}
       ref={ref}
       configuration={configuration}
+      accessible={accessible}
+      onAccessibleChange={setAccessible}
       onUnauthorized={() => {
         const refusals = ++unauthorized.current;
         if (refusals >= MAX_REFUSALS) return false;
@@ -225,8 +229,12 @@ function PreviewStreamDocumentView({
   onRetry,
   onStreaming,
   onRecoverProcess,
+  accessible,
+  onAccessibleChange,
 }: Omit<NativeStreamBridge, "onUnauthorized" | "onRefused"> & {
   readonly configuration: string;
+  readonly accessible: boolean;
+  readonly onAccessibleChange: (accessible: boolean) => void;
   readonly background: string;
   readonly onUnauthorized: () => boolean;
   readonly onRetry: () => void;
@@ -260,6 +268,8 @@ function PreviewStreamDocumentView({
     webView.current?.injectJavaScript(
       `window.T3PreviewStream?.command(${JSON.stringify(input)}); true;`,
     );
+  const showAccessible = (next: boolean) =>
+    webView.current?.injectJavaScript(`window.T3PreviewStream?.accessibility(${next}); true;`);
   // Reads here rather than in the screen: a tab switch during the iOS paste prompt unmounts
   // this view and nulls its WebView ref, so the text can never type into another tab.
   const paste = () =>
@@ -328,16 +338,27 @@ function PreviewStreamDocumentView({
           <AppText className="text-xs text-foreground-muted">
             {previewStreamControlLabel(control)}
           </AppText>
-          {control?.canOperate ? (
+          <View className="flex-row items-center gap-2">
             <Button
-              label={control.controller === "you" ? "Release control" : "Take control"}
+              label={accessible ? "Hide accessible view" : "Accessible view"}
               tone="secondary"
-              disabled={control.controller === "another-viewer"}
-              onPress={() =>
-                command({ type: control.controller === "you" ? "releaseControl" : "takeControl" })
-              }
+              disabled={pdfUri != null}
+              onPress={() => {
+                onAccessibleChange(!accessible);
+                showAccessible(!accessible);
+              }}
             />
-          ) : null}
+            {control?.canOperate ? (
+              <Button
+                label={control.controller === "you" ? "Release control" : "Take control"}
+                tone="secondary"
+                disabled={control.controller === "another-viewer"}
+                onPress={() =>
+                  command({ type: control.controller === "you" ? "releaseControl" : "takeControl" })
+                }
+              />
+            ) : null}
+          </View>
         </View>
       ) : null}
       <View className="relative flex-1">
@@ -405,6 +426,7 @@ function PreviewStreamDocumentView({
                 onPictureInPicture?.(message, message.detail);
                 return;
               case "status":
+                if (message.status === "connecting" && accessible) showAccessible(true);
                 clearBootstrapTimer();
                 if (message.status === "error") {
                   fail(message.detail ?? "Browser stream failed.");

apps/mobile/src/features/browser/preview-accessible-document.test.ts

@@ -0,0 +1,42 @@
+import { createAccessibleDocumentView } from "@t3tools/client-runtime/preview/accessible-document";
+import type { PreviewAccessibilityDocument } from "@t3tools/contracts";
+import { expect, it, vi } from "vite-plus/test";
+
+it("keeps an unsent field edit through unrelated page updates and cancels it on Escape", () => {
+  const container = document.createElement("div");
+  document.body.append(container);
+  const onAction = vi.fn(() => true);
+  const view = createAccessibleDocumentView(container, { onAction });
+  const page: PreviewAccessibilityDocument = {
+    document: "page",
+    version: 1,
+    truncated: false,
+    root: {
+      id: "root",
+      role: "document",
+      children: [{ id: "field", role: "textbox", name: "Name", value: "Server" }],
+    },
+  };
+  try {
+    view.update(page, true);
+    const input = container.querySelector("input")!;
+    input.focus();
+    input.value = "Unsent draft";
+    input.setSelectionRange(3, 6);
+    input.dispatchEvent(new Event("input", { bubbles: true }));
+
+    view.update({ ...page, version: 2, root: { ...page.root, name: "Updated title" } }, true);
+
+    expect(document.activeElement).toBe(input);
+    expect(input.value).toBe("Unsent draft");
+    expect([input.selectionStart, input.selectionEnd]).toEqual([3, 6]);
+    expect(onAction).not.toHaveBeenCalled();
+
+    input.dispatchEvent(new KeyboardEvent("keydown", { key: "Escape", bubbles: true }));
+    expect(input.value).toBe("Server");
+    expect(onAction).not.toHaveBeenCalled();
+  } finally {
+    view.destroy();
+    container.remove();
+  }
+});

apps/mobile/src/features/browser/preview-stream-document.ts

@@ -47,6 +47,12 @@ export function previewStreamDocument(configuration: string, script: string) {
   canvas, video { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; }
   canvas { touch-action: none; user-select: none; -webkit-user-select: none; -webkit-touch-callout: none; }
   video { pointer-events: none; }
+  .accessible-view {
+    box-sizing: border-box;
+    position: fixed; inset: 0; z-index: 2; overflow: auto; -webkit-overflow-scrolling: touch;
+    background: inherit; font: -apple-system-body; font-family: system-ui, sans-serif;
+  }
+  .accessible-view[hidden] { display: none; }
   /* The agent's pointer, positioned over the letterboxed frame by the viewer script. */
   .agent-cursor {
     position: absolute; left: 0; top: 0; width: 20px; height: 20px; pointer-events: none;
@@ -63,7 +69,7 @@ export function previewStreamDocument(configuration: string, script: string) {
     .agent-cursor .ping { animation: none; opacity: 0; }
   }
   /* Pinned so focus never scrolls; 16px keeps iOS from zooming on focus. */
-  textarea {
+  body > textarea {
     position: fixed; left: 0; top: 0; width: 1px; height: 1px;
     padding: 0; margin: -1px; border: 0; overflow: hidden;
     clip: rect(0, 0, 0, 0); white-space: nowrap; font-size: 16px;

apps/mobile/src/features/browser/preview-stream.browser.ts

@@ -8,6 +8,11 @@ import {
   type PreviewStreamPointer,
   type PreviewStreamViewport,
 } from "@t3tools/client-runtime/preview/server-browser-stream";
+import {
+  type AccessibleDocumentView,
+  createAccessibleDocumentView,
+} from "@t3tools/client-runtime/preview/accessible-document";
+import type { PreviewAccessibilityDocument } from "@t3tools/contracts";
 
 import type { PreviewStreamConfiguration, PreviewStreamMessage } from "./preview-stream-document";
 
@@ -46,6 +51,7 @@ interface Viewer {
   readonly stop: () => void;
   readonly command: (input: PreviewStreamInput) => void;
   readonly togglePictureInPicture: () => void;
+  readonly accessibility: (enabled: boolean) => void;
 }
 
 let activeViewer: Viewer | null = null;
@@ -63,6 +69,16 @@ export function pictureInPicture() {
   activeViewer?.togglePictureInPicture();
 }
 
+export function accessibility(enabled: boolean) {
+  activeViewer?.accessibility(enabled);
+}
+
+/** The app's theme only sends a background, so pick the text color that contrasts with it. */
+const readableTextColor = (background: string) => {
+  const [red = 255, green = 255, blue = 255] = (background.match(/\d+(\.\d+)?/g) ?? []).map(Number);
+  return 0.2126 * red + 0.7152 * green + 0.0722 * blue > 140 ? "#111" : "#f5f5f5";
+};
+
 export function start(configuration: PreviewStreamConfiguration) {
   stop();
   const post = (message: PreviewStreamMessage) => {
@@ -87,7 +103,11 @@ export function start(configuration: PreviewStreamConfiguration) {
   input.spellcheck = false;
   input.setAttribute("autocorrect", "off");
   input.disabled = true;
-  document.body.replaceChildren(container, ...(interactive ? [input] : []));
+  // The page as a document the platform screen reader can walk; empty until turned on.
+  const accessibleContainer = document.createElement("div");
+  accessibleContainer.className = "accessible-view";
+  accessibleContainer.hidden = true;
+  document.body.replaceChildren(container, ...(interactive ? [input, accessibleContainer] : []));
 
   let stopped = false;
   let streaming = false;
@@ -96,6 +116,11 @@ export function start(configuration: PreviewStreamConfiguration) {
   let cap: { width: number; height: number } | null = null;
   let client: PreviewStreamClient | null = null;
   let control: PreviewStreamControl | null = null;
+  let accessible = false;
+  let accessibleView: AccessibleDocumentView | null = null;
+  let accessibleDocument: PreviewAccessibilityDocument | null = null;
+  const showAccessibleDocument = () =>
+    accessibleView?.update(accessibleDocument, control?.controller === "you");
   let resizeTimer: ReturnType<typeof setTimeout> | null = null;
   let wheelFrame: number | null = null;
   let pendingWheel: WheelInput | null = null;
@@ -205,10 +230,11 @@ export function start(configuration: PreviewStreamConfiguration) {
           const previous = control;
           control = nextControl;
           post({ type: "control", ...nextControl });
+          showAccessibleDocument();
           if (nextControl.controller === "you") agentCursor.style.opacity = "0";
           if (nextControl.controller !== "you") clearInput();
           else {
-            input.disabled = !interactive;
+            input.disabled = !interactive || accessible;
             if (interactive && size && previous?.controller !== "you")
               next.send({ type: "resize", ...size });
           }
@@ -224,9 +250,15 @@ export function start(configuration: PreviewStreamConfiguration) {
         onUnauthorized: () => post({ type: "unauthorized" }),
         onGone: () => post({ type: "gone" }),
         onHostSetup: (setup) => post({ type: "hostSetup", ...setup }),
+        onAccessibility: (document) => {
+          accessibleDocument = document;
+          showAccessibleDocument();
+        },
+        onAccessibilityNotice: (message) => accessibleView?.notice(message),
       },
     );
     client = next;
+    next.accessibility(accessible);
   };
 
   const measure = () => {
@@ -671,9 +703,33 @@ export function start(configuration: PreviewStreamConfiguration) {
     }
   };
 
+  const setAccessible = (enabled: boolean) => {
+    // Passive previews never subscribe, even if asked.
+    if (!interactive || enabled === accessible) return;
+    accessible = enabled;
+    input.disabled = enabled || control?.controller !== "you";
+    client?.accessibility(enabled);
+    accessibleContainer.hidden = !enabled;
+    container.setAttribute("aria-hidden", String(enabled));
+    if (enabled) {
+      accessibleContainer.style.color = readableTextColor(
+        getComputedStyle(document.body).backgroundColor,
+      );
+      accessibleView = createAccessibleDocumentView(accessibleContainer, {
+        onAction: (action) => send(action),
+      });
+      showAccessibleDocument();
+    } else {
+      accessibleView?.destroy();
+      accessibleView = null;
+      accessibleDocument = null;
+    }
+  };
+
   activeViewer = {
     stop: () => {
       stopped = true;
+      setAccessible(false);
       observer.disconnect();
       visualViewport?.removeEventListener("resize", followVisualViewport);
       visualViewport?.removeEventListener("scroll", followVisualViewport);
@@ -691,6 +747,7 @@ export function start(configuration: PreviewStreamConfiguration) {
       send(message);
     },
     togglePictureInPicture,
+    accessibility: setAccessible,
   };
   window.addEventListener("pagehide", stop, { once: true });
   reportStatus("connecting");

apps/mobile/vite.config.ts

@@ -4,6 +4,7 @@ import { defineConfig, normalizePath } from "vite-plus";
 import "vite-plus/test/config";
 
 const reactTests = [
+  "src/features/browser/preview-accessible-document.test.ts",
   "src/features/threads/new-task-flow-provider.test.tsx",
   "src/features/workspace/workspace-tabs-state.test.ts",
   "src/state/use-thread-pr.demand.test.tsx",