diff --git a/apps/mobile/metro.config.js b/apps/mobile/metro.config.js index 6371da34fd190..0d369db37484a 100644 --- a/apps/mobile/metro.config.js +++ b/apps/mobile/metro.config.js @@ -93,7 +93,7 @@ async function prepareStreamScripts() { ["packages/client-runtime/src/device", ["stream.ts", "hubAccess.ts"], generateAll], [ "packages/client-runtime/src/preview", - ["serverBrowserStream.ts"], + ["serverBrowserStream.ts", "accessibleDocument.ts"], generatePreviewStreamScript, ], ]) { diff --git a/apps/mobile/src/features/browser/PreviewStreamWebView.tsx b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx index 3fae93f483bcc..3aa8f0123cb8c 100644 --- a/apps/mobile/src/features/browser/PreviewStreamWebView.tsx +++ b/apps/mobile/src/features/browser/PreviewStreamWebView.tsx @@ -164,6 +164,8 @@ function AuthorizedPreviewStream({ ...props }: PreviewStreamConfiguration & NativeStreamBridge) { const [attempt, setAttempt] = useState(0); + // Outlives reconnects, which remount the document view. + const [accessible, setAccessible] = useState(false); const [previousAccess, setPreviousAccess] = useState(props.access); // Wait for refreshed access, including cookie credentials with unchanged JSON. if (props.access !== previousAccess) { @@ -185,6 +187,8 @@ function AuthorizedPreviewStream({ {...props} ref={ref} configuration={configuration} + accessible={accessible} + onAccessibleChange={setAccessible} onUnauthorized={() => { const refusals = ++unauthorized.current; if (refusals >= MAX_REFUSALS) return false; @@ -225,8 +229,12 @@ function PreviewStreamDocumentView({ onRetry, onStreaming, onRecoverProcess, + accessible, + onAccessibleChange, }: Omit & { readonly configuration: string; + readonly accessible: boolean; + readonly onAccessibleChange: (accessible: boolean) => void; readonly background: string; readonly onUnauthorized: () => boolean; readonly onRetry: () => void; @@ -260,6 +268,8 @@ function PreviewStreamDocumentView({ webView.current?.injectJavaScript( `window.T3PreviewStream?.command(${JSON.stringify(input)}); true;`, ); + const showAccessible = (next: boolean) => + webView.current?.injectJavaScript(`window.T3PreviewStream?.accessibility(${next}); true;`); // Reads here rather than in the screen: a tab switch during the iOS paste prompt unmounts // this view and nulls its WebView ref, so the text can never type into another tab. const paste = () => @@ -328,16 +338,27 @@ function PreviewStreamDocumentView({ {previewStreamControlLabel(control)} - {control?.canOperate ? ( + - ) : null} + {control?.canOperate ? ( + + ) : null} +
{pdfUrl ? ( @@ -777,6 +804,14 @@ export function ServerBrowserSurface(props: { className="absolute inset-0 z-10 size-full border-0" /> ) : null} + {accessibleView && !pdfUrl ? ( + clientRef.current?.send(action) ?? false} + /> + ) : null} resetInput(event.currentTarget)} className="sr-only top-0 left-0 text-base" diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index d1b48e1d76f4f..fb3461f72c165 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -43,6 +43,10 @@ "types": "./src/preview/serverBrowserStream.ts", "default": "./src/preview/serverBrowserStream.ts" }, + "./preview/accessible-document": { + "types": "./src/preview/accessibleDocument.ts", + "default": "./src/preview/accessibleDocument.ts" + }, "./worktree-setup": { "types": "./src/worktreeSetup.ts", "default": "./src/worktreeSetup.ts" diff --git a/packages/client-runtime/src/preview/accessibleDocument.test.ts b/packages/client-runtime/src/preview/accessibleDocument.test.ts new file mode 100644 index 0000000000000..c8f7094b08c32 --- /dev/null +++ b/packages/client-runtime/src/preview/accessibleDocument.test.ts @@ -0,0 +1,210 @@ +import type { PreviewAccessibilityNode } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { planAccessibleNode, syncChildren } from "./accessibleDocument.ts"; + +const plan = (node: PreviewAccessibilityNode, canAct = true) => planAccessibleNode(node, canAct); + +describe("planAccessibleNode", () => { + it("renders headings, links and landmarks as their native elements", () => { + expect(plan({ id: "1", role: "heading", name: "Cart", level: 9 })).toMatchObject({ + tag: "h6", + text: "Cart", + }); + expect( + plan({ id: "2", role: "link", name: "Docs", url: "https://t3.codes/docs" }), + ).toMatchObject({ + tag: "a", + attributes: { href: "https://t3.codes/docs" }, + control: "activate", + }); + // Script URLs never become live links in the viewer's own document. + expect(plan({ id: "3", role: "link", name: "Run", url: "javascript:alert(1)" })).toMatchObject({ + tag: "a", + attributes: { role: "link", tabindex: "0" }, + }); + expect(plan({ id: "4", role: "navigation", name: "Primary" })).toMatchObject({ + tag: "nav", + attributes: { "aria-label": "Primary" }, + }); + }); + + it("keeps controls reachable but inert for viewers without control", () => { + expect(plan({ id: "1", role: "button", name: "Pay", pressed: "true" }, false)).toMatchObject({ + tag: "button", + attributes: { type: "button", "aria-pressed": "true", "aria-disabled": "true" }, + control: "activate", + }); + expect(plan({ id: "2", role: "textbox", name: "Name", value: "Ada" }, false)).toMatchObject({ + tag: "input", + attributes: { type: "text", readonly: "" }, + value: "Ada", + }); + }); + + it("shows secret fields as empty password inputs and multiline fields as text areas", () => { + expect( + plan({ id: "1", role: "textbox", name: "Password", protected: true, editable: true }), + ).toMatchObject({ tag: "input", attributes: { type: "password" }, value: "" }); + expect(plan({ id: "2", role: "textbox", name: "Bio", multiline: true })).toMatchObject({ + tag: "textarea", + control: "edit", + }); + }); + + it("turns comboboxes with options into a native select and checkboxes into inputs", () => { + expect( + plan({ + id: "1", + role: "combobox", + name: "Size", + children: [ + { + id: "2", + role: "listbox", + children: [ + { id: "3", role: "option", name: "Small" }, + { id: "4", role: "option", name: "Large", selected: true }, + ], + }, + ], + }), + ).toMatchObject({ + tag: "select", + control: "select", + options: [ + { id: "3", label: "Small", selected: false }, + { id: "4", label: "Large", selected: true }, + ], + }); + expect(plan({ id: "5", role: "checkbox", name: "All", checked: "mixed" })).toMatchObject({ + tag: "input", + attributes: { type: "checkbox" }, + checked: false, + indeterminate: true, + }); + }); + + it("keeps an expanded tree item's nested items as their own controls", () => { + const file = { id: "4", role: "treeitem", name: "index.ts", selected: false }; + const folder = { + id: "1", + role: "treeitem", + name: "src", + expanded: true, + selected: true, + children: [ + { id: "2", role: "text", name: "src" }, + { id: "3", role: "group", children: [file] }, + ], + }; + const shown = plan(folder); + expect(shown).toMatchObject({ + tag: "div", + attributes: { + role: "treeitem", + tabindex: "0", + "aria-label": "src", + "aria-expanded": "true", + "aria-selected": "true", + }, + control: "activate", + children: folder.children, + }); + expect(shown).not.toHaveProperty("text"); + expect(plan(file)).toMatchObject({ + tag: "div", + attributes: { role: "treeitem", tabindex: "0", "aria-selected": "false" }, + text: "index.ts", + control: "activate", + }); + }); + + it("skips empty presentational nodes and keeps text", () => { + expect(plan({ id: "1", role: "image" })).toBeNull(); + expect(plan({ id: "2", role: "unknownwidget" })).toBeNull(); + expect(plan({ id: "3", role: "text", name: "Hello" })).toEqual({ kind: "text", text: "Hello" }); + }); +}); + +interface FakeNode { + readonly name: string; + parentNode: FakeParent | null; + readonly contains: (other: FakeNode | null) => boolean; + readonly nextSibling: FakeNode | null; +} +interface FakeParent { + readonly childNodes: Array; + readonly insertBefore: (child: FakeNode, before: FakeNode | null) => FakeNode; + readonly removeChild: (child: FakeNode) => FakeNode; +} + +/** Records every placement, since the DOM blurs a focused element whenever it moves. */ +const fakeParent = (names: ReadonlyArray) => { + const moved: Array = []; + const parent: FakeParent = { + childNodes: [], + insertBefore: (child, before) => { + if (child.parentNode) parent.removeChild(child); + moved.push(child.name); + const index = before ? parent.childNodes.indexOf(before) : parent.childNodes.length; + parent.childNodes.splice(index, 0, child); + child.parentNode = parent; + return child; + }, + removeChild: (child) => { + parent.childNodes.splice(parent.childNodes.indexOf(child), 1); + child.parentNode = null; + return child; + }, + }; + const make = (name: string): FakeNode => { + const node: FakeNode = { + name, + parentNode: null, + contains: (other) => other === node, + get nextSibling() { + const siblings = node.parentNode?.childNodes ?? []; + return siblings[siblings.indexOf(node) + 1] ?? null; + }, + }; + return node; + }; + const nodes = new Map(names.map((name) => [name, make(name)])); + for (const node of nodes.values()) parent.insertBefore(node, null); + moved.length = 0; + const get = (name: string) => nodes.get(name) ?? nodes.set(name, make(name)).get(name)!; + const sync = (next: ReadonlyArray, focused?: string) => + syncChildren( + parent as unknown as Element, + next.map(get) as unknown as ReadonlyArray, + (focused ? get(focused) : null) as unknown as Node | null, + ); + const order = () => parent.childNodes.map((node) => node.name); + return { moved, sync, order }; +}; + +describe("syncChildren", () => { + it("never moves a focused child that keeps its place among survivors", () => { + const removal = fakeParent(["A", "B"]); + removal.sync(["B"], "B"); + expect(removal.order()).toEqual(["B"]); + expect(removal.moved).toEqual([]); + + const swap = fakeParent(["A", "B", "C"]); + swap.sync(["B", "C", "A", "D"], "B"); + expect(swap.order()).toEqual(["B", "C", "A", "D"]); + expect(swap.moved).not.toContain("B"); + + const forward = fakeParent(["C", "D", "B"]); + forward.sync(["B", "C", "D"], "B"); + expect(forward.order()).toEqual(["B", "C", "D"]); + expect(forward.moved).not.toContain("B"); + }); + + it("reorders, inserts and removes children to match", () => { + const list = fakeParent(["A", "B", "C", "D"]); + list.sync(["D", "E", "B"]); + expect(list.order()).toEqual(["D", "E", "B"]); + }); +}); diff --git a/packages/client-runtime/src/preview/accessibleDocument.ts b/packages/client-runtime/src/preview/accessibleDocument.ts new file mode 100644 index 0000000000000..a2b97381663cd --- /dev/null +++ b/packages/client-runtime/src/preview/accessibleDocument.ts @@ -0,0 +1,548 @@ +import type { + PreviewAccessibilityAction, + PreviewAccessibilityDocument, + PreviewAccessibilityNode, +} from "@t3tools/contracts"; + +export type AccessibleControl = "activate" | "check" | "edit" | "select"; + +export interface AccessibleOption { + readonly id: string; + readonly label: string; + readonly selected: boolean; + readonly disabled: boolean; +} + +export type AccessiblePlan = + | { readonly kind: "text"; readonly text: string } + | { + readonly kind: "element"; + readonly tag: string; + readonly attributes: Readonly>; + /** Rendered as the element's own text; children are then not rendered. */ + readonly text?: string; + /** A visible label wrapped around form fields. */ + readonly label?: string; + readonly children: ReadonlyArray; + readonly control: AccessibleControl | null; + readonly value?: string; + readonly checked?: boolean; + readonly indeterminate?: boolean; + readonly options?: ReadonlyArray; + }; + +const LANDMARK_TAGS: Record = { + main: "main", + navigation: "nav", + banner: "header", + contentinfo: "footer", + complementary: "aside", + region: "section", + article: "article", + paragraph: "p", + list: "ul", + listitem: "li", + blockquote: "blockquote", + figure: "figure", + code: "code", + separator: "hr", + term: "dt", + definition: "dd", +}; +const ARIA_CONTAINER_ROLES = new Set([ + "table", + "grid", + "treegrid", + "row", + "rowgroup", + "cell", + "gridcell", + "columnheader", + "rowheader", + "form", + "search", + "dialog", + "alertdialog", + "alert", + "status", + "log", + "timer", + "tablist", + "tabpanel", + "menu", + "menubar", + "toolbar", + "tree", + "radiogroup", + "group", + "note", + "feed", +]); +const BUTTON_ROLES: Record = { + button: null, + PopUpButton: null, + ToggleButton: null, + DisclosureTriangle: null, + tab: "tab", + menuitem: "menuitem", + menuitemcheckbox: "menuitemcheckbox", + menuitemradio: "menuitemradio", + switch: "switch", + option: null, +}; +const FIELD_ROLES = new Set(["textbox", "searchbox", "spinbutton", "combobox"]); +const SAFE_LINK = /^(https?:|mailto:)/i; + +const textOf = (node: PreviewAccessibilityNode): string => + node.role === "text" + ? (node.name ?? "") + : (node.name ?? (node.children ?? []).map(textOf).join(" ").trim()); + +const descendants = (node: PreviewAccessibilityNode, role: string): PreviewAccessibilityNode[] => + (node.children ?? []).flatMap((child) => + child.role === role ? [child] : descendants(child, role), + ); + +const stateAttributes = (node: PreviewAccessibilityNode, inert: boolean) => { + const attributes: Record = {}; + if (node.description) attributes["aria-description"] = node.description; + if (node.expanded !== undefined) attributes["aria-expanded"] = String(node.expanded); + if (node.required) attributes["aria-required"] = "true"; + if (node.invalid) attributes["aria-invalid"] = "true"; + // aria-disabled keeps controls reachable so a reader can still find them. + if (inert) attributes["aria-disabled"] = "true"; + return attributes; +}; + +/** How one streamed node renders locally. Pure, so web and the mobile WebView agree. */ +export function planAccessibleNode( + node: PreviewAccessibilityNode, + canAct: boolean, +): AccessiblePlan | null { + if (node.role === "text") return node.name ? { kind: "text", text: node.name } : null; + const children = node.children ?? []; + const inert = !canAct || node.disabled === true; + const states = stateAttributes(node, inert); + const named = (attributes: Record) => + node.name ? { ...attributes, "aria-label": node.name } : attributes; + const element = ( + tag: string, + attributes: Record, + extra: Partial> = {}, + ): AccessiblePlan => ({ + kind: "element", + tag, + attributes: { ...states, ...attributes }, + children, + control: null, + ...extra, + }); + + if (node.role === "heading") { + const level = Math.min(Math.max(node.level ?? 2, 1), 6); + return element(`h${level}`, {}, children.length > 0 ? {} : { text: node.name ?? "" }); + } + if (node.role === "link") { + const safe = node.url !== undefined && SAFE_LINK.test(node.url); + return element("a", safe ? { href: node.url! } : { role: "link", tabindex: "0" }, { + text: textOf(node) || node.url || "Link", + children: [], + control: "activate", + }); + } + if (node.role === "treeitem") { + // Not a button: an expanded item holds nested items that stay their own controls. + const attributes = named({ role: "treeitem", tabindex: "0" }); + if (node.selected !== undefined) attributes["aria-selected"] = String(node.selected); + if (node.checked !== undefined) attributes["aria-checked"] = node.checked; + return element( + "div", + attributes, + children.length > 0 + ? { control: "activate" } + : { text: node.name || "Tree item", control: "activate" }, + ); + } + if (node.role in BUTTON_ROLES) { + const role = BUTTON_ROLES[node.role]; + const attributes: Record = { type: "button", ...(role ? { role } : {}) }; + if (node.pressed !== undefined) attributes["aria-pressed"] = node.pressed; + if (node.checked !== undefined && role) attributes["aria-checked"] = node.checked; + if (node.selected !== undefined && role === "tab") { + attributes["aria-selected"] = String(node.selected); + } else if (node.selected) { + attributes["aria-current"] = "true"; + } + return element("button", attributes, { + text: textOf(node) || node.role, + children: [], + control: "activate", + }); + } + if (node.role === "checkbox" || node.role === "radio") { + return element( + "input", + { type: node.role }, + { + label: node.name ?? "", + children: [], + control: "check", + checked: node.checked === "true", + indeterminate: node.checked === "mixed", + }, + ); + } + const options = + node.role === "combobox" || node.role === "listbox" ? descendants(node, "option") : []; + if (options.length > 0) { + return element( + "select", + node.role === "listbox" ? { size: String(Math.min(Math.max(options.length, 2), 8)) } : {}, + { + label: node.name ?? "", + children: [], + control: "select", + options: options.map((option) => ({ + id: option.id, + label: textOf(option) || option.value || "Option", + selected: option.selected === true, + disabled: option.disabled === true, + })), + }, + ); + } + if (FIELD_ROLES.has(node.role) || (node.editable && children.length === 0)) { + const readOnly = inert || node.readonly === true; + const attributes: Record = readOnly ? { readonly: "" } : {}; + if (!node.multiline) { + attributes.type = node.protected ? "password" : node.role === "searchbox" ? "search" : "text"; + } + if (node.protected) attributes["aria-description"] = "Its value is hidden."; + return element(node.multiline ? "textarea" : "input", attributes, { + label: node.name ?? "", + children: [], + control: "edit", + value: node.protected ? "" : (node.value ?? ""), + }); + } + if (node.role === "image" || node.role === "img" || node.role === "graphics-symbol") { + return node.name + ? element("span", { role: "img", "aria-label": node.name }, { children: [] }) + : null; + } + if (node.role === "document") { + return element("section", { "aria-label": node.name ?? "Frame" }); + } + const tag = LANDMARK_TAGS[node.role]; + if (tag) { + const attributes = node.role === "region" || node.role === "navigation" ? named({}) : {}; + if (tag === "hr") return element("hr", {}, { children: [] }); + return element(tag, attributes); + } + if (ARIA_CONTAINER_ROLES.has(node.role)) { + return element("div", named({ role: node.role })); + } + const fallback = children.length === 0 ? [node.name, node.value].filter(Boolean).join(": ") : ""; + if (children.length === 0 && !fallback) return null; + return element("div", {}, fallback ? { text: fallback } : {}); +} + +/** + * Moving an element blurs it, so obsolete children go first and the child + * holding focus stays put while its siblings move around it. + */ +export function syncChildren(parent: Element, next: ReadonlyArray, focused: Node | null) { + const keep = new Set(next); + for (const child of Array.from(parent.childNodes)) { + if (!keep.has(child)) parent.removeChild(child); + } + next.forEach((child, index) => { + if (focused && child.parentNode === parent && child.contains(focused)) { + while (parent.childNodes[index] !== child) { + parent.insertBefore(parent.childNodes[index]!, child.nextSibling); + } + } + if (parent.childNodes[index] !== child) { + parent.insertBefore(child, parent.childNodes[index] ?? null); + } + }); +} + +export interface AccessibleDocumentView { + /** `null` shows a loading message. Updates keep elements, focus and edits in place. */ + readonly update: (document: PreviewAccessibilityDocument | null, canAct: boolean) => void; + readonly notice: (message: string) => void; + readonly destroy: () => void; +} + +const ACCESSIBLE_VIEW_NOT_CONTROLLER = "Take control of the browser to use the page."; + +const STYLE = ` +.t3-accessible-document { font: inherit; line-height: 1.5; padding: 12px; overflow-wrap: anywhere; } +.t3-accessible-document :is(h1,h2,h3,h4,h5,h6) { font-weight: 600; margin: 0.75em 0 0.25em; } +.t3-accessible-document h1 { font-size: 1.5em; } .t3-accessible-document h2 { font-size: 1.3em; } +.t3-accessible-document h3 { font-size: 1.15em; } +.t3-accessible-document :is(p, ul, blockquote, figure, section, nav, main, article, aside, header, footer) { margin: 0.4em 0; } +.t3-accessible-document ul { list-style: disc; padding-left: 1.5em; } +.t3-accessible-document a { color: inherit; text-decoration: underline; } +.t3-accessible-document :is(button, input, textarea, select) { font: inherit; color: inherit; background: transparent; border: 1px solid currentColor; border-radius: 4px; padding: 2px 6px; margin: 2px 0; } +.t3-accessible-document input[type=checkbox], .t3-accessible-document input[type=radio] { margin-right: 6px; } +.t3-accessible-document label { display: block; margin: 4px 0; } +.t3-accessible-document label > span { margin-right: 6px; } +.t3-accessible-document [aria-disabled=true] { opacity: 0.7; } +.t3-accessible-document [role=row] { display: flex; gap: 12px; } +.t3-accessible-document > [role=status] { font-size: 0.85em; opacity: 0.8; margin: 0 0 8px; } +`; + +interface Rendered { + readonly node: Node; + readonly signature: string; + readonly control: Element | null; +} + +/** Keyed DOM view of streamed documents. It never moves remote focus by itself. */ +export function createAccessibleDocumentView( + container: HTMLElement, + options: { readonly onAction: (action: PreviewAccessibilityAction) => boolean }, +): AccessibleDocumentView { + const doc = container.ownerDocument; + const style = doc.createElement("style"); + style.textContent = STYLE; + const status = doc.createElement("p"); + status.setAttribute("role", "status"); + const body = doc.createElement("div"); + container.classList.add("t3-accessible-document"); + container.replaceChildren(style, status, body); + const rendered = new Map(); + const nodes = new Map(); + let current: PreviewAccessibilityDocument | null = null; + let canAct = false; + let summary = ""; + + const say = (message: string) => { + status.textContent = message || summary; + }; + const act = (action: Omit) => { + if (!current) return; + const sent = + canAct && + options.onAction({ type: "accessibilityAction", document: current.document, ...action }); + if (sent) return; + // Put back local changes the page never saw, then say why. + view.update(current, canAct); + say(ACCESSIBLE_VIEW_NOT_CONTROLLER); + }; + + const render = (node: PreviewAccessibilityNode, seen: Set): Node | null => { + const plan = planAccessibleNode(node, canAct); + if (!plan) return null; + seen.add(node.id); + nodes.set(node.id, node); + const signature = + plan.kind === "text" ? "#text" : `${plan.tag}:${plan.control}:${plan.attributes.type ?? ""}`; + let entry = rendered.get(node.id); + if (!entry || entry.signature !== signature) { + entry = create(plan, node.id, signature); + rendered.set(node.id, entry); + } + if (plan.kind === "text") { + if (entry.node.textContent !== plan.text) entry.node.textContent = plan.text; + return entry.node; + } + const element = entry.control as HTMLElement; + // Copied: removing from the live attribute map while iterating it skips entries. + for (const attribute of Array.from(element.attributes)) { + if ( + attribute.name !== "data-node" && + attribute.name !== "data-dirty" && + attribute.name !== "data-server-value" && + !(attribute.name in plan.attributes) + ) { + element.removeAttribute(attribute.name); + } + } + for (const [name, value] of Object.entries(plan.attributes)) { + if (element.getAttribute(name) !== value) element.setAttribute(name, value); + } + if (entry.node !== element) { + const label = (entry.node as HTMLElement).firstChild as HTMLElement; + if (label.textContent !== (plan.label ?? "")) label.textContent = plan.label ?? ""; + } + if (element instanceof HTMLInputElement && plan.control === "check") { + element.checked = plan.checked === true; + element.indeterminate = plan.indeterminate === true; + } else if ( + (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) && + plan.control === "edit" + ) { + element.dataset.serverValue = plan.value ?? ""; + // An edit in progress wins over the page until it is committed or cancelled. + if (!(element.dataset.dirty && doc.activeElement === element)) { + delete element.dataset.dirty; + if (element.value !== plan.value) element.value = plan.value ?? ""; + } + } else if (element instanceof HTMLSelectElement && plan.options) { + syncChildren( + element, + plan.options.map((option) => { + const existing = [...element.options].find((each) => each.dataset.node === option.id); + const item = existing ?? doc.createElement("option"); + item.dataset.node = option.id; + item.value = option.id; + if (item.textContent !== option.label) item.textContent = option.label; + item.disabled = option.disabled; + item.selected = option.selected; + return item; + }), + null, + ); + } + if (plan.text !== undefined) { + if (element.textContent !== plan.text) element.textContent = plan.text; + } else if (plan.control === null || plan.control === "activate") { + syncChildren( + element, + plan.children.flatMap((child) => { + const childNode = render(child, seen); + return childNode ? [childNode] : []; + }), + doc.activeElement, + ); + } + return entry.node; + }; + + const create = (plan: AccessiblePlan, id: string, signature: string): Rendered => { + if (plan.kind === "text") + return { node: doc.createTextNode(plan.text), signature, control: null }; + const element = doc.createElement(plan.tag); + element.dataset.node = id; + if (plan.label === undefined) return { node: element, signature, control: element }; + const label = doc.createElement("label"); + label.append(doc.createElement("span"), element); + return { node: label, signature, control: element }; + }; + + const nodeOf = (target: EventTarget | null) => { + const element = target instanceof Element ? target.closest("[data-node]") : null; + const id = element?.dataset.node; + const node = id ? nodes.get(id) : undefined; + return node && element ? { node, element } : null; + }; + + const onClick = (event: Event) => { + const hit = nodeOf(event.target); + if (!hit) return; + const plan = planAccessibleNode(hit.node, canAct); + if (plan?.kind !== "element" || (plan.control !== "activate" && plan.control !== "check")) + return; + // Local links and checkboxes must not change on their own; the page answers. + event.preventDefault(); + if (hit.node.disabled) { + say("That control is unavailable."); + return; + } + act({ node: hit.node.id, action: "activate" }); + }; + const commit = (element: HTMLInputElement | HTMLTextAreaElement, submit: boolean) => { + const hit = nodeOf(element); + if (!hit || (!submit && element.value === element.dataset.serverValue)) return; + delete element.dataset.dirty; + act({ + node: hit.node.id, + action: "setValue", + value: element.value, + ...(submit ? { submit } : {}), + }); + }; + const onChange = (event: Event) => { + const target = event.target; + if (target instanceof HTMLSelectElement) { + const chosen = target.selectedOptions[0]?.dataset.node; + if (chosen) act({ node: chosen, action: "activate" }); + return; + } + if ( + (target instanceof HTMLInputElement && + target.type !== "checkbox" && + target.type !== "radio") || + target instanceof HTMLTextAreaElement + ) { + if (target.dataset.dirty) commit(target, false); + } + }; + const onInput = (event: Event) => { + const target = event.target; + if (target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement) { + target.dataset.dirty = "1"; + } + }; + const onKeyDown = (event: Event) => { + const keyboard = event as KeyboardEvent; + const target = keyboard.target; + if (keyboard.isComposing) return; + if (target instanceof HTMLInputElement && keyboard.key === "Enter" && target.dataset.node) { + if (target.type === "checkbox" || target.type === "radio") return; + keyboard.preventDefault(); + commit(target, true); + } else if ( + (target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement) && + keyboard.key === "Escape" && + target.dataset.dirty + ) { + keyboard.preventDefault(); + keyboard.stopPropagation(); + delete target.dataset.dirty; + target.value = target.dataset.serverValue ?? ""; + } else if (keyboard.key === "Enter" || keyboard.key === " ") { + const hit = nodeOf(target); + const role = hit?.element === target ? hit.element.getAttribute("role") : null; + if (hit && (role === "treeitem" || (role === "link" && keyboard.key === "Enter"))) { + keyboard.preventDefault(); + act({ node: hit.node.id, action: "activate" }); + } + } + }; + container.addEventListener("click", onClick); + container.addEventListener("change", onChange); + container.addEventListener("input", onInput); + container.addEventListener("keydown", onKeyDown); + + const view: AccessibleDocumentView = { + update: (document, nextCanAct) => { + canAct = nextCanAct; + if (document?.document !== current?.document) { + rendered.clear(); + nodes.clear(); + } + current = document; + summary = !document + ? "Loading the page's accessible view..." + : document.truncated + ? "Some page content could not be shown. The page may be too long or contain an unavailable frame." + : ""; + say(""); + if (!document) { + body.replaceChildren(); + return; + } + const seen = new Set(); + nodes.clear(); + const root = render(document.root, seen); + syncChildren(body, root ? [root] : [], doc.activeElement); + for (const id of rendered.keys()) if (!seen.has(id)) rendered.delete(id); + }, + notice: say, + destroy: () => { + container.removeEventListener("click", onClick); + container.removeEventListener("change", onChange); + container.removeEventListener("input", onInput); + container.removeEventListener("keydown", onKeyDown); + container.classList.remove("t3-accessible-document"); + container.replaceChildren(); + rendered.clear(); + nodes.clear(); + }, + }; + return view; +} diff --git a/packages/client-runtime/src/preview/serverBrowserStream.test.ts b/packages/client-runtime/src/preview/serverBrowserStream.test.ts index cfde00363a005..b71fe54ddf1e1 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.test.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.test.ts @@ -166,3 +166,97 @@ describe("preview stream agent pointer", () => { }); }); }); + +describe("preview stream accessible view", () => { + beforeEach(() => vi.stubGlobal("WebSocket", FakeSocket)); + afterEach(() => vi.unstubAllGlobals()); + const document = { + document: "1:loader", + version: 3, + truncated: false, + root: { id: "0.1", role: "document", children: [{ id: "0.2", role: "button", name: "Go" }] }, + }; + const connect = () => { + const onAccessibility = vi.fn(); + const onAccessibilityNotice = vi.fn(); + const onConnectedChange = vi.fn(); + const client = createPreviewStreamClient(target, { + onFrame: vi.fn(), + onAccessibility, + onAccessibilityNotice, + onViewport: vi.fn(), + onConnectedChange, + onUnauthorized: vi.fn(), + }); + const socket = FakeSocket.current; + socket.dispatchEvent(new Event("open")); + return { client, socket, onAccessibility, onAccessibilityNotice }; + }; + + it("lets a read-only viewer subscribe without granting it page actions", () => { + const { client, socket, onAccessibility } = connect(); + socket.control({ canOperate: false, controller: "agent", generation: 1, dialog: null }); + socket.message(JSON.stringify({ type: "accessibility", document })); + expect(onAccessibility).not.toHaveBeenCalled(); + client.accessibility(true); + expect(socket.sent).toEqual(['{"type":"accessibility","enabled":true}']); + socket.message(JSON.stringify({ type: "accessibility", document })); + expect(onAccessibility).toHaveBeenLastCalledWith(document); + expect( + client.send({ + type: "accessibilityAction", + document: document.document, + node: "0.2", + action: "activate", + }), + ).toBe(false); + expect(socket.sent).toHaveLength(1); + client.stop(); + }); + + it("drops documents that break the contract and clears on off, reset and disconnect", () => { + const { client, socket, onAccessibility, onAccessibilityNotice } = connect(); + client.accessibility(true); + socket.message( + JSON.stringify({ type: "accessibility", document: { ...document, root: { role: "x" } } }), + ); + socket.message( + JSON.stringify({ type: "accessibility", document: { ...document, version: -1 } }), + ); + expect(onAccessibility).not.toHaveBeenCalled(); + socket.message(JSON.stringify({ type: "accessibility", document })); + socket.message(JSON.stringify({ type: "accessibilityNotice", message: "The page changed." })); + expect(onAccessibilityNotice).toHaveBeenCalledWith("The page changed."); + socket.message(JSON.stringify({ type: "accessibility", document: null })); + expect(onAccessibility).toHaveBeenLastCalledWith(null); + + socket.message(JSON.stringify({ type: "accessibility", document })); + client.accessibility(false); + expect(onAccessibility).toHaveBeenLastCalledWith(null); + expect(socket.sent.at(-1)).toBe('{"type":"accessibility","enabled":false}'); + + client.accessibility(true); + socket.message(JSON.stringify({ type: "accessibility", document })); + onAccessibility.mockClear(); + socket.dispatchEvent(new CloseEvent("close", { code: 1012 })); + expect(onAccessibility).toHaveBeenCalledExactlyOnceWith(null); + client.stop(); + }); + + it("resubscribes after a reconnect", () => { + vi.useFakeTimers(); + try { + const { client, socket } = connect(); + client.accessibility(true); + socket.dispatchEvent(new CloseEvent("close", { code: 1012 })); + vi.advanceTimersByTime(500); + const next = FakeSocket.current; + expect(next).not.toBe(socket); + next.dispatchEvent(new Event("open")); + expect(next.sent).toEqual(['{"type":"accessibility","enabled":true}']); + client.stop(); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/packages/client-runtime/src/preview/serverBrowserStream.ts b/packages/client-runtime/src/preview/serverBrowserStream.ts index b71fff335f881..bd93904d81816 100644 --- a/packages/client-runtime/src/preview/serverBrowserStream.ts +++ b/packages/client-runtime/src/preview/serverBrowserStream.ts @@ -2,9 +2,13 @@ import { type DeviceHubAccess, withDeviceHubQuery } from "../device/hubAccess.ts"; import { PREVIEW_STREAM_HOST_SETUP_CLOSE_CODE, + PreviewAccessibilityDocument, + type PreviewAccessibilityAction, type PreviewStreamHostSetup, type PreviewViewportSetting, } from "@t3tools/contracts"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; export const PREVIEW_STREAM_BASE_PATH = "/api/preview-stream"; @@ -90,7 +94,8 @@ export type PreviewStreamInput = | { readonly type: "navigate"; readonly url: string } | { readonly type: "history"; readonly delta: -1 | 1 } | { readonly type: "reload"; readonly ignoreCache?: boolean } - | { readonly type: "probe"; readonly x: number; readonly y: number }; + | { readonly type: "probe"; readonly x: number; readonly y: number } + | PreviewAccessibilityAction; export interface PreviewStreamDownload { readonly fileName: string; @@ -197,13 +202,20 @@ export interface PreviewStreamEvents { readonly onUnauthorized: () => void; readonly onGone?: () => void; readonly onHostSetup?: (setup: PreviewStreamHostSetup) => void; + /** `null` clears the view: turned off, disconnected, or the server reset it. */ + readonly onAccessibility?: (document: PreviewAccessibilityDocument | null) => void; + readonly onAccessibilityNotice?: (message: string) => void; } export interface PreviewStreamClient { readonly send: (input: PreviewStreamInput) => boolean; + /** Not a page mutation, so read-only viewers may subscribe. Survives reconnects. */ + readonly accessibility: (enabled: boolean) => void; readonly stop: () => void; } +const decodeAccessibilityDocument = Schema.decodeUnknownOption(PreviewAccessibilityDocument); + const ACK_MESSAGE = JSON.stringify({ type: "ack" }); export function createPreviewStreamClient( target: PreviewStreamTarget, @@ -223,6 +235,13 @@ export function createPreviewStreamClient( let failures = 0; let control: PreviewStreamControl | null = null; let fileChooser: string | null = null; + let accessible = false; + let accessibilityShown = false; + const clearAccessibility = () => { + if (!accessibilityShown) return; + accessibilityShown = false; + events.onAccessibility?.(null); + }; const connect = () => { if (stopped) return; @@ -234,6 +253,7 @@ export function createPreviewStreamClient( if (socket !== ws) return; opened = true; events.onConnectedChange(true); + if (accessible) ws.send(JSON.stringify({ type: "accessibility", enabled: true })); }); ws.addEventListener("message", (event) => { if (socket !== ws) return; @@ -271,8 +291,22 @@ export function createPreviewStreamClient( phase, sequence, tabId, + document, + message: noticeText, } = message as Record; - if ( + if (type === "accessibility") { + if (!accessible) return; + if (document === null) { + clearAccessibility(); + return; + } + const decoded = decodeAccessibilityDocument(document); + if (Option.isNone(decoded)) return; + accessibilityShown = true; + events.onAccessibility?.(decoded.value); + } else if (type === "accessibilityNotice" && typeof noticeText === "string") { + if (accessible) events.onAccessibilityNotice?.(noticeText); + } else if ( type === "fileChooser" && typeof id === "string" && typeof multiple === "boolean" && @@ -346,6 +380,7 @@ export function createPreviewStreamClient( if (socket !== ws) return; socket = null; control = null; + clearAccessibility(); if (opened) events.onConnectedChange(false); if (stopped) return; if (event.code === 4404) { @@ -379,8 +414,18 @@ export function createPreviewStreamClient( socket.send(JSON.stringify(input)); return true; }, + accessibility: (enabled) => { + if (enabled === accessible) return; + accessible = enabled; + if (!enabled) clearAccessibility(); + if (socket?.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify({ type: "accessibility", enabled })); + } + }, stop: () => { stopped = true; + accessible = false; + clearAccessibility(); if (retryTimer !== null) clearTimeout(retryTimer); const ws = socket; socket = null; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index f00976d243b37..72d49adaf57ad 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -58,6 +58,7 @@ export * from "./browserImport.ts"; export * from "./browserProfile.ts"; export * from "./device.ts"; export * from "./preview.ts"; +export * from "./previewAccessibility.ts"; export * from "./previewAutomation.ts"; export * from "./previewCapture.ts"; export * from "./usage.ts"; diff --git a/packages/contracts/src/previewAccessibility.ts b/packages/contracts/src/previewAccessibility.ts new file mode 100644 index 0000000000000..2ec7d4ff100ed --- /dev/null +++ b/packages/contracts/src/previewAccessibility.ts @@ -0,0 +1,113 @@ +import { Schema } from "effect"; + +import { NonNegativeInt } from "./baseSchemas.ts"; + +export const PREVIEW_ACCESSIBILITY_MAX_NODES = 4_000; +export const PREVIEW_ACCESSIBILITY_VALUE_MAX_CHARS = 10_000; + +const Text = (max: number) => Schema.String.check(Schema.isMaxLength(max)); +const Tristate = Schema.Literals(["true", "false", "mixed"]); + +/** + * One node of a streamed page's accessibility tree. Ids are namespaced per + * capture session and never reused, so an id from a replaced document is stale. + */ +export interface PreviewAccessibilityNode { + readonly id: string; + readonly role: string; + readonly name?: string; + readonly description?: string; + /** Omitted for password and other secret fields; see `protected`. */ + readonly value?: string; + readonly url?: string; + readonly level?: number; + readonly checked?: "true" | "false" | "mixed"; + readonly pressed?: "true" | "false" | "mixed"; + readonly expanded?: boolean; + readonly selected?: boolean; + readonly disabled?: boolean; + readonly readonly?: boolean; + readonly required?: boolean; + readonly invalid?: boolean; + readonly focused?: boolean; + readonly editable?: boolean; + readonly multiline?: boolean; + readonly protected?: boolean; + readonly children?: ReadonlyArray; +} + +export const PreviewAccessibilityNode: Schema.Codec = Schema.Struct({ + id: Text(100), + role: Text(100), + name: Schema.optionalKey(Text(2_000)), + description: Schema.optionalKey(Text(2_000)), + value: Schema.optionalKey(Text(PREVIEW_ACCESSIBILITY_VALUE_MAX_CHARS)), + url: Schema.optionalKey(Text(2_048)), + level: Schema.optionalKey(NonNegativeInt), + checked: Schema.optionalKey(Tristate), + pressed: Schema.optionalKey(Tristate), + expanded: Schema.optionalKey(Schema.Boolean), + selected: Schema.optionalKey(Schema.Boolean), + disabled: Schema.optionalKey(Schema.Boolean), + readonly: Schema.optionalKey(Schema.Boolean), + required: Schema.optionalKey(Schema.Boolean), + invalid: Schema.optionalKey(Schema.Boolean), + focused: Schema.optionalKey(Schema.Boolean), + editable: Schema.optionalKey(Schema.Boolean), + multiline: Schema.optionalKey(Schema.Boolean), + protected: Schema.optionalKey(Schema.Boolean), + children: Schema.optionalKey( + Schema.Array( + Schema.suspend((): Schema.Codec => PreviewAccessibilityNode), + ), + ), +}); + +const countNodes = (root: PreviewAccessibilityNode) => { + let count = 0; + const stack = [root]; + while (stack.length > 0) { + const node = stack.pop()!; + if (++count > PREVIEW_ACCESSIBILITY_MAX_NODES) return count; + if (node.children) stack.push(...node.children); + } + return count; +}; + +/** A full replacement of the semantic page; viewers never patch an older one. */ +export const PreviewAccessibilityDocument = Schema.Struct({ + /** Changes whenever the top-level document or the capture session is replaced. */ + document: Text(200), + version: NonNegativeInt, + truncated: Schema.Boolean, + root: PreviewAccessibilityNode, +}).check( + Schema.makeFilter((value: { readonly root: PreviewAccessibilityNode }) => + countNodes(value.root) > PREVIEW_ACCESSIBILITY_MAX_NODES + ? `Accessibility documents must not exceed ${PREVIEW_ACCESSIBILITY_MAX_NODES} nodes.` + : undefined, + ), +); +export type PreviewAccessibilityDocument = typeof PreviewAccessibilityDocument.Type; + +/** Read-only viewers may subscribe; it never changes page state. */ +export const PreviewAccessibilitySubscription = Schema.Struct({ + type: Schema.Literal("accessibility"), + enabled: Schema.Boolean, +}); +export type PreviewAccessibilitySubscription = typeof PreviewAccessibilitySubscription.Type; + +export const PreviewAccessibilityActionKind = Schema.Literals(["activate", "setValue"]); +export type PreviewAccessibilityActionKind = typeof PreviewAccessibilityActionKind.Type; + +/** Page mutation: only the controlling viewer's actions run, against the current document. */ +export const PreviewAccessibilityAction = Schema.Struct({ + type: Schema.Literal("accessibilityAction"), + document: Text(200), + node: Text(100), + action: PreviewAccessibilityActionKind, + value: Schema.optionalKey(Text(PREVIEW_ACCESSIBILITY_VALUE_MAX_CHARS)), + /** After `setValue`, press Enter in the field. */ + submit: Schema.optionalKey(Schema.Boolean), +}); +export type PreviewAccessibilityAction = typeof PreviewAccessibilityAction.Type;