From dc6bdfb71bba894d62b6faa53f75cc5d43281683 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 6 Oct 2026 10:27:40 -0700 Subject: [PATCH 1/3] feat(cua): turn on computer use from settings on any host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings → Integrations → Computer turns Cua computer use on for the selected environment, and shows whether its driver is running or why it could not start. Setup fits the host: on the Mac that runs the environment it walks through Accessibility and Screen Recording for T3 Code and restarts the driver after the grants; for a remote Mac it says who must grant them; Windows and Linux hosts need only a signed-in desktop. The mobile app's agent settings get the same switch. A new server.checkComputerUse RPC reports the host status, so every client reads the same answer. User docs cover each platform. Ported from #10899 (Julius's permission onboarding). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../SettingsServerControlsRouteScreen.tsx | 10 ++ apps/server/src/auth/RpcAuthorization.ts | 1 + apps/server/src/cua/CuaDriver.ts | 19 ++- .../src/observability/RpcInstrumentation.ts | 1 + apps/server/src/ws.ts | 9 + .../components/settings/CuaSetupDialog.tsx | 156 ++++++++++++++++++ .../settings/IntegrationsSettings.tsx | 103 +++++++++++- .../components/settings/SettingsPanels.tsx | 2 + .../src/components/settings/settingsSearch.ts | 7 + docs/README.md | 1 + docs/user/computer-use.md | 42 +++++ packages/client-runtime/src/state/server.ts | 11 ++ packages/contracts/src/cua.ts | 14 ++ packages/contracts/src/rpc.ts | 9 + 14 files changed, 383 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/components/settings/CuaSetupDialog.tsx create mode 100644 docs/user/computer-use.md diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index 91b0b1b044e5..3f16ebe4f0ea 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -377,6 +377,16 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { onValueChange={(value) => write({ enableAgentBrowserAccess: value })} /> + + write({ enableCua: value })} + /> + 0} /> ) : null} diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index bf2b8d49daf8..956f978bb974 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -61,6 +61,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.projectsMutate]: AuthOrchestrationOperateScope, [WS_METHODS.serverProbe]: AuthOrchestrationReadScope, [WS_METHODS.serverGetConfig]: AuthOrchestrationReadScope, + [WS_METHODS.serverCheckComputerUse]: AuthOrchestrationReadScope, [WS_METHODS.serverRefreshProviders]: AuthOrchestrationReadScope, [WS_METHODS.serverUpdateProvider]: AuthProvidersManageScope, [WS_METHODS.providerAuthStart]: AuthProvidersManageScope, diff --git a/apps/server/src/cua/CuaDriver.ts b/apps/server/src/cua/CuaDriver.ts index 928b09c450b6..6769569b7435 100644 --- a/apps/server/src/cua/CuaDriver.ts +++ b/apps/server/src/cua/CuaDriver.ts @@ -1,6 +1,11 @@ import * as NodeModule from "node:module"; -import type { CuaDriverMcpConfiguration, DesktopCuaDriverReport } from "@t3tools/contracts"; +import type { + CuaDriverMcpConfiguration, + CuaHostStatus, + DesktopCuaDriverReport, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import type { EmbeddedCuaDriverHost, EmbeddedDriverConnection } from "@trycua/cua-driver/embedded"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; @@ -144,6 +149,8 @@ export class CuaDriver extends Context.Service< readonly acquire: Effect.Effect>; /** Why the last start failed, until a later start succeeds. */ readonly lastFailure: Effect.Effect>; + /** What setup flows show: the host OS, and whether the driver runs or why not. */ + readonly status: Effect.Effect; } >()("t3/cua/CuaDriver") {} @@ -175,6 +182,8 @@ export const make = Effect.fn("CuaDriver.make")(function* ( stopped?: boolean; /** Message of the typed error that ended this attempt, for the readiness signal. */ failure?: string; + /** Set once the host reported its MCP descriptor. */ + running?: boolean; }; let current: Attempt | undefined; let closed = false; @@ -241,6 +250,7 @@ export const make = Effect.fn("CuaDriver.make")(function* ( return; } lastFailure = undefined; + attempt.running = true; yield* Deferred.succeed(attempt.ready, Option.some(mcp)); }), ); @@ -290,10 +300,17 @@ export const make = Effect.fn("CuaDriver.make")(function* ( return attempt ? yield* Deferred.await(attempt.ready) : Option.none(); }); + const platform = yield* HostProcess.Platform; return CuaDriver.of({ enabled: enabled, acquire, lastFailure: Effect.sync(() => Option.fromNullishOr(lastFailure)), + status: Effect.sync(() => ({ + platform: + platform === "darwin" || platform === "linux" || platform === "win32" ? platform : "other", + running: current?.running === true, + ...(lastFailure === undefined ? {} : { failure: lastFailure }), + })), }); }); diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index cb8b0b6a86cf..e5a1ae6e3982 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -32,6 +32,7 @@ const RPC_AGGREGATES = { [ORCHESTRATION_V2_WS_METHODS.subscribeCommandOutput]: "orchestrationV2", [WS_METHODS.projectsMutate]: "orchestration", [WS_METHODS.serverProbe]: "server", + [WS_METHODS.serverCheckComputerUse]: "server", [WS_METHODS.serverGetConfig]: "server", [WS_METHODS.serverRefreshProviders]: "server", [WS_METHODS.serverUpdateProvider]: "server", diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index cfe49ad39d52..ea601276ad83 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -187,6 +187,7 @@ import * as TerminalManager from "./terminal/Manager.ts"; import { terminalOutputStream, withTerminalOutputWindow } from "./terminal/OutputProtocol.ts"; import * as ServerBrowser from "./preview/ServerBrowser.ts"; import * as DeviceService from "./device/DeviceService.ts"; +import * as CuaDriver from "./cua/CuaDriver.ts"; import { remoteSshDeviceHosts } from "./device/localSshDeviceHost.ts"; import * as PreviewManager from "./preview/Manager.ts"; import { issueAssetUrl } from "./assets/AssetAccess.ts"; @@ -2176,6 +2177,14 @@ export const layerWsRpc = ( Effect.andThen(scheduledTasks.getWebhookDelivery(input)), ), [WS_METHODS.serverProbe]: (_input) => Effect.succeed({}), + [WS_METHODS.serverCheckComputerUse]: (_input) => + Effect.serviceOption(CuaDriver.CuaDriver).pipe( + Effect.flatMap((driver) => + Option.isSome(driver) + ? driver.value.status + : Effect.succeed({ platform: "other", running: false } as const), + ), + ), [WS_METHODS.serverGetConfig]: (_input) => loadServerConfig({ usageLimitsCommand: false, workspaceSnapshotsOnDemand: false }), [WS_METHODS.serverSearchAcpRegistry]: (input) => diff --git a/apps/web/src/components/settings/CuaSetupDialog.tsx b/apps/web/src/components/settings/CuaSetupDialog.tsx new file mode 100644 index 000000000000..44e4f2eec302 --- /dev/null +++ b/apps/web/src/components/settings/CuaSetupDialog.tsx @@ -0,0 +1,156 @@ +import type { CuaHostStatus } from "@t3tools/contracts"; +import { useState } from "react"; + +import { MacAccessibilityIcon, MacScreenRecordingIcon } from "../Icons"; +import { PermissionChecklist, PermissionContinueButton } from "../permissions/PermissionChecklist"; +import { usePermissionStatus } from "../permissions/usePermissionStatus"; +import { Button } from "../ui/button"; +import { + Dialog, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPanel, + DialogPopup, + DialogTitle, +} from "../ui/dialog"; + +export type CuaPermission = "accessibility" | "screen-recording"; + +const CUA_PERMISSIONS: readonly CuaPermission[] = ["accessibility", "screen-recording"]; + +/** What the host itself needs before computer use can work there. */ +function hostRequirements(platform: CuaHostStatus["platform"], hostLabel: string): string { + switch (platform) { + case "darwin": + return `T3 Code on ${hostLabel} needs Accessibility and Screen Recording. Open T3 Code on that Mac to grant them; the setting turns on from here either way.`; + case "linux": + return `${hostLabel} needs a signed-in graphical session. On Wayland, someone at ${hostLabel} approves the screen capture prompt the first time an agent looks at the screen.`; + case "win32": + return `${hostLabel} needs a signed-in desktop session. No permissions are needed; Windows Defender may ask once before Cua Driver first runs.`; + default: + return `Computer use needs a desktop on ${hostLabel}.`; + } +} + +/** + * Setup for Cua computer use. On the Mac that hosts the environment it walks + * through Accessibility and Screen Recording, which macOS grants to T3 Code; + * elsewhere it explains what the host needs, since only someone at it can act. + */ +export function CuaSetupDialog({ + enabled, + hostLabel, + platform, + localMac, + onCheck, + onAllow, + onEnable, + onClose, +}: { + enabled: boolean; + hostLabel: string; + platform: CuaHostStatus["platform"]; + /** This desktop is the Mac that runs the environment, so it can grant access here. */ + localMac: boolean; + onCheck: (permission: CuaPermission) => Promise; + onAllow: (permission: CuaPermission) => Promise; + onEnable: () => Promise; + onClose: () => void; +}) { + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const permission = usePermissionStatus( + async () => ({ + accessibility: await onCheck("accessibility"), + "screen-recording": await onCheck("screen-recording"), + }), + { accessibility: false, "screen-recording": false }, + localMac && !busy, + ); + const ready = !localMac || permission.isReady(CUA_PERMISSIONS); + const allow = (target: CuaPermission) => { + if (busy) return; + setBusy(true); + setError(null); + void onAllow(target) + .catch(() => setError("Could not open System Settings. Try Allow again.")) + .finally(() => setBusy(false)); + }; + const finish = async () => { + if (busy) return; + setBusy(true); + setError(null); + try { + await onEnable(); + onClose(); + } catch { + setError("Could not save the setting. Try again."); + } finally { + setBusy(false); + } + }; + return ( + { + if (!open && !busy) onClose(); + }} + > + + + Let agents use {hostLabel} + + Cua Driver lets agents see and use apps on {hostLabel} in the background, without moving + your cursor. Agent sessions started afterwards get the computer use tools. + + + + {localMac ? ( + , + title: "Accessibility", + description: "Click, type, and read controls in other apps.", + granted: permission.status.accessibility, + onAllow: () => allow("accessibility"), + }, + { + id: "screen-recording", + icon: , + title: "Screen Recording", + description: "See the screen to decide what to do next.", + granted: permission.status["screen-recording"], + onAllow: () => allow("screen-recording"), + }, + ]} + /> + ) : ( +

{hostRequirements(platform, hostLabel)}

+ )} + {error || (localMac && permission.error) ? ( +

+ {error ?? permission.error} +

+ ) : null} + {localMac && !ready ? ( +

+ If a permission stays off after you allow it, quit and reopen T3 Code. +

+ ) : null} + + + + void finish()}> + {enabled ? "Done" : "Turn on"} + + + + + ); +} diff --git a/apps/web/src/components/settings/IntegrationsSettings.tsx b/apps/web/src/components/settings/IntegrationsSettings.tsx index 5239f57bfdc3..0dfe90a45489 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.tsx @@ -3,6 +3,7 @@ import { DeviceToolVersions } from "../device/DeviceToolVersions"; import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"; import { ScopedSwitch } from "./ScopedSwitch"; import { DeviceSourcesSettings } from "./DeviceSourcesSettings"; +import { CuaSetupDialog } from "./CuaSetupDialog"; import { AuthSettingsWriteScope, BrowserImportFailureReason, @@ -48,7 +49,8 @@ import { useEnvironments, usePrimaryEnvironment } from "~/state/environments"; import { deviceEnvironment, useDeviceState } from "~/state/device"; import { readEnvironmentScope, useEnvironmentScope } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; -import { formatEnvironmentQueryError } from "~/state/query"; +import { formatEnvironmentQueryError, useEnvironmentQuery } from "~/state/query"; +import { serverEnvironment } from "~/state/server"; import { previewEnvironment } from "~/state/preview"; import { useServerConfigs } from "~/state/entities"; import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; @@ -1050,6 +1052,104 @@ function AgentBrowserAccessSetting() { ); } +/** + * Cua computer use for the selected environment. The switch records intent; + * the host status shows whether the driver can actually start there. + */ +function ComputerUseSettings() { + const { environment: selected } = useSettingsScope(); + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + const [setupOpen, setSetupOpen] = useState(false); + const environmentId = + selected?.connection.phase === "connected" && selected.serverConfig !== null + ? selected.environmentId + : null; + const hostLabel = selected?.label ?? "this computer"; + const { data: status, refresh } = useEnvironmentQuery( + environmentId === null ? null : serverEnvironment.computerUse({ environmentId, input: {} }), + ); + const platform = status?.platform ?? "other"; + // Only the Mac that runs the environment can grant it macOS permissions. + const localMac = + isElectron && + platform === "darwin" && + window.desktopBridge?.getClientPlatform() === "darwin" && + selected?.entry.target._tag === "PrimaryConnectionTarget"; + const enabled = settings.enableCua; + const statusText = !enabled + ? null + : status?.running + ? "Running" + : status?.failure + ? status.failure + : "Starts with the next agent session"; + + return ( + + void updateSettings({ enableCua: DEFAULT_SERVER_SETTINGS.enableCua })} + /> + ) : null + } + control={ + <> + {enabled && platform !== "other" ? ( + + ) : null} + { + if (checked) setSetupOpen(true); + else void updateSettings({ enableCua: false }); + }} + /> + + } + /> + {setupOpen ? ( + + window.desktopBridge?.checkSystemPermission(permission) ?? Promise.resolve(false) + } + onAllow={async (permission) => { + const api = readLocalApi(); + if (!api) throw new Error("Unable to open System Settings."); + await api.shell.openSystemSettings(permission); + }} + onEnable={async () => { + if (!enabled) await updateSettings({ enableCua: true }); + // A driver started before the grants keeps macOS's cached denial. + if (localMac) await window.desktopBridge?.restartCuaDriver(); + }} + onClose={() => { + setSetupOpen(false); + refresh(); + }} + /> + ) : null} + + ); +} + function BrowserAutoShowFloatingPreviewSetting({ disabled }: { readonly disabled: boolean }) { const autoShow = useClientSettings((settings) => settings.browserAutoShowFloatingPreview); const updateSettings = useUpdatePrimarySettings(); @@ -1591,6 +1691,7 @@ export function DevicesSettingsPanel() { return ( + ); } diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index c4e3d27a07fa..743c193e61ce 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -552,6 +552,7 @@ export function useSettingsRestore(onRestored?: () => void) { ...(settings.enableAgentBrowserAccess !== DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess ? ["Agent browser access"] : []), + ...(settings.enableCua !== DEFAULT_UNIFIED_SETTINGS.enableCua ? ["Computer use"] : []), ]; const restoreDefaults = async () => { @@ -673,6 +674,7 @@ export function useSettingsRestore(onRestored?: () => void) { browserAutoShowFloatingPreview: DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview, pullRequestLinksOpenInApp: DEFAULT_UNIFIED_SETTINGS.pullRequestLinksOpenInApp, enableAgentBrowserAccess: DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess, + enableCua: DEFAULT_UNIFIED_SETTINGS.enableCua, }); onRestored?.(); }; diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index aabfd0300eda..2d3ed853be53 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -845,6 +845,13 @@ export const SETTINGS_SEARCH_ITEMS = [ targetId: "devices", searchTerms: ["allow simulator emulator ios android drive tools sessions"], }, + { + id: "cua-computer-use", + title: "Computer use", + to: "/settings/devices", + targetId: "computer", + searchTerms: ["cua driver apps desktop screen click type agents control mac linux windows"], + }, { id: "device-hub", title: "Use devices", diff --git a/docs/README.md b/docs/README.md index af3c45ce567b..6a201348658a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,7 @@ - [Appearance and themes](./user/appearance.md) - [Keyboard shortcuts](./user/keybindings.md) - [SnapShots](./user/snap-shot.md) +- [Computer use](./user/computer-use.md) - [Visual replies](./user/html-renders.md) - [Import browser sessions](./user/browser-import.md) - [Use T3 Code as your default browser](./user/default-browser.md) diff --git a/docs/user/computer-use.md b/docs/user/computer-use.md new file mode 100644 index 000000000000..4f1f135d337b --- /dev/null +++ b/docs/user/computer-use.md @@ -0,0 +1,42 @@ +# Computer use + +Computer use lets agents see and use apps on the computer that runs your T3 +Code environment. Every provider gets it: Claude, Codex, Cursor, Grok, +Antigravity, OpenCode, Pi, and ACP agents. It runs through +[Cua Driver](https://cua.ai/docs/cua-driver), which acts on windows in the +background, so agents do not take over your cursor or keyboard while you work. + +It works on macOS, Windows, and Linux. It is off by default. + +## Turn it on + +Open **Settings → Integrations → Computer**, select the environment, and turn +on **Computer use**. Agent sessions started afterwards get the computer use +tools. Turning it off stops the driver; sessions already running lose the tools +on their next start. + +What the host needs depends on its OS: + +- **macOS:** T3 Code needs Accessibility and Screen Recording. On the Mac that + runs the environment, the setup walks you through both. From another computer + or the mobile app, the switch turns on, but someone at that Mac still has to + grant them from T3 Code there. Choose **Setup** next to the switch to check + them again later. +- **Windows:** someone must be signed in to the desktop. No permissions are + needed. Windows Defender may ask once before Cua Driver first runs. +- **Linux:** the host needs a signed-in graphical session. On Wayland, someone + at the computer approves the screen capture prompt the first time an agent + looks at the screen. + +The desktop app includes Cua Driver. A server started with `npx t3` downloads +the same version into T3 Code's home the first time an agent session needs it. +To use your own build instead, set `T3CODE_CUA_DRIVER_PATH` to its executable +before starting the server. + +If you already configured a `cua-driver` MCP server for Codex yourself, Codex +keeps yours and T3 Code does not add a second one. + +## What you see + +Each action an agent takes shows in the thread with the app's icon and a plain +title, such as **Clicked in Safari**. diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index dadaf2dafb7a..a82ce84c56e4 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -1098,6 +1098,11 @@ export function createServerEnvironmentAtoms( ]); }).pipe(Atom.withLabel(`environment-data:server:usage-scan-settings:${environmentId}`)), ); + const computerUseSettingAtom = Atom.family((environmentId: EnvironmentId) => + Atom.make((get) => get(settingsValueAtom(environmentId))?.enableCua ?? false).pipe( + Atom.withLabel(`environment-data:server:computer-use-setting:${environmentId}`), + ), + ); const providersValueAtom = Atom.family((environmentId: EnvironmentId) => Atom.make((get) => get(configValueAtom(environmentId))?.providers ?? null).pipe( Atom.withLabel(`environment-data:server:providers:${environmentId}`), @@ -1247,6 +1252,12 @@ export function createServerEnvironmentAtoms( label: "environment-data:provider:install-remove", tag: WS_METHODS.providerInstallRemove, }), + /** Whether the host can run computer use; rechecked when the switch flips. */ + computerUse: createEnvironmentRpcQueryAtomFamily(runtime, { + label: "environment-data:server:computer-use", + tag: WS_METHODS.serverCheckComputerUse, + refreshTrigger: ({ environmentId }) => computerUseSettingAtom(environmentId), + }), traceDiagnostics: createEnvironmentRpcQueryAtomFamily(runtime, { label: "environment-data:server:trace-diagnostics", tag: WS_METHODS.serverGetTraceDiagnostics, diff --git a/packages/contracts/src/cua.ts b/packages/contracts/src/cua.ts index c3734c193032..eb7b3600185d 100644 --- a/packages/contracts/src/cua.ts +++ b/packages/contracts/src/cua.ts @@ -37,3 +37,17 @@ export const DesktopCuaDriverReport = Schema.Union([ }), ]); export type DesktopCuaDriverReport = typeof DesktopCuaDriverReport.Type; + +/** + * Whether the environment's host can run computer use right now. Setup flows + * show it; the switch itself records intent and is not a readiness signal. + */ +export const CuaHostStatus = Schema.Struct({ + /** The host's OS, which decides what the user must grant. */ + platform: Schema.Literals(["darwin", "linux", "win32", "other"]), + /** The driver started for an agent session and is running. */ + running: Schema.Boolean, + /** Why the last start failed, until a later start succeeds. */ + failure: Schema.optionalKey(Schema.String), +}); +export type CuaHostStatus = typeof CuaHostStatus.Type; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index edc7a9ab852a..a60ccd583612 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -307,6 +307,7 @@ import { DeviceSession, DeviceShutdownInput, } from "./device.ts"; +import { CuaHostStatus } from "./cua.ts"; import { ServerConfigStreamEvent, DesktopUpdateCommitInput, @@ -492,6 +493,7 @@ export const WS_METHODS = { serverProbe: "server.probe", serverGetConfig: "server.getConfig", + serverCheckComputerUse: "server.checkComputerUse", serverRefreshProviders: "server.refreshProviders", serverUpdateProvider: "server.updateProvider", serverUpdateServer: "server.updateServer", @@ -624,6 +626,12 @@ const WsServerProbeRpc = Rpc.make(WS_METHODS.serverProbe, { error: EnvironmentAuthorizationError, }); +const WsServerCheckComputerUseRpc = Rpc.make(WS_METHODS.serverCheckComputerUse, { + payload: Schema.Struct({}), + success: CuaHostStatus, + error: EnvironmentAuthorizationError, +}); + const WsServerGetConfigRpc = Rpc.make(WS_METHODS.serverGetConfig, { payload: Schema.Struct({}), success: ServerConfig, @@ -1954,6 +1962,7 @@ export const WsRpcGroup = RpcGroup.make( WsGuestShareCreateRpc, WsServerProbeRpc, WsServerGetConfigRpc, + WsServerCheckComputerUseRpc, WsServerRefreshProvidersRpc, WsServerUpdateProviderRpc, WsProviderConsumeResetCreditRpc, From a300aa9dce035eb9e2adcc7c38e6e764c1fde876 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 21:50:03 +0000 Subject: [PATCH 2/3] fix(cua): finish host setup and restore settings --- .../SettingsServerControlsRouteScreen.tsx | 2 +- apps/server/src/auth/RpcAuthorization.test.ts | 26 ++++ apps/server/src/cua/CuaDriver.test.ts | 67 +++++++++ apps/server/src/cua/CuaDriver.ts | 1 - .../components/settings/CuaSetupDialog.tsx | 54 +++++--- .../settings/IntegrationsSettings.tsx | 131 ++++++++++++++---- .../components/settings/SettingsPanels.tsx | 4 +- ...rceControlWritingSettings.browser.test.tsx | 4 +- .../components/settings/useScopedSettings.ts | 74 +++++----- docs/user/computer-use.md | 23 ++- packages/client-runtime/src/state/server.ts | 1 - packages/contracts/src/cua.ts | 7 +- 12 files changed, 288 insertions(+), 106 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index 3f16ebe4f0ea..e204001a928e 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -381,7 +381,7 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { write({ enableCua: value })} diff --git a/apps/server/src/auth/RpcAuthorization.test.ts b/apps/server/src/auth/RpcAuthorization.test.ts index 8044740a7e2e..49af649cb521 100644 --- a/apps/server/src/auth/RpcAuthorization.test.ts +++ b/apps/server/src/auth/RpcAuthorization.test.ts @@ -22,6 +22,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; import * as RpcTest from "effect/rpc/RpcTest"; +import { applyServerSettingsPatch } from "@t3tools/shared/serverSettings"; import { RPC_REQUIRED_SCOPES, @@ -147,6 +148,31 @@ describe("settings mutation authorization", () => { instanceId: ProviderInstanceId.make("codex_work"), }; + it.effect("restores computer use alongside duration settings through authorization", () => + Effect.gen(function* () { + const client = yield* RpcTest.makeClient(group).pipe( + Effect.provide( + Layer.mergeAll( + group.toLayerHandler(WS_METHODS.serverUpdateSettings, ({ patch }) => + Effect.succeed( + applyServerSettingsPatch({ ...DEFAULT_SERVER_SETTINGS, enableCua: true }, patch), + ), + ), + RpcAuthorization.layer([AuthSettingsWriteScope]), + ), + ), + ); + const defaults = { + enableCua: DEFAULT_SERVER_SETTINGS.enableCua, + automaticGitFetchInterval: DEFAULT_SERVER_SETTINGS.automaticGitFetchInterval, + providerHealthRefreshInterval: DEFAULT_SERVER_SETTINGS.providerHealthRefreshInterval, + }; + expect(yield* client[WS_METHODS.serverUpdateSettings]({ patch: defaults })).toMatchObject( + defaults, + ); + }).pipe(Effect.scoped), + ); + it.effect("allows provider-only mutations while denying mixed settings without their grant", () => Effect.gen(function* () { let handled = 0; diff --git a/apps/server/src/cua/CuaDriver.test.ts b/apps/server/src/cua/CuaDriver.test.ts index 7d3fc531b18a..ba423c356947 100644 --- a/apps/server/src/cua/CuaDriver.test.ts +++ b/apps/server/src/cua/CuaDriver.test.ts @@ -1,6 +1,7 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { it } from "@effect/vitest"; import type { CuaDriverMcpConfiguration, DesktopCuaDriverReport } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import type { EmbeddedDriverConnection } from "@trycua/cua-driver/embedded"; import { Deferred, Effect, Exit, Fiber, Option, PubSub, Ref, Scope, Stream } from "effect"; import { TestClock } from "effect/testing"; @@ -501,8 +502,10 @@ describe("CuaDriver", () => { yield* Deferred.succeed(f.result, mcp); expect(yield* f.service.acquire).toEqual(Option.some(mcp)); yield* Deferred.await(f.watching); + expect((yield* f.service.status).running).toBe(true); yield* Deferred.succeed(f.exited, undefined); yield* Deferred.await(f.stopped); + expect((yield* f.service.status).running).toBe(false); expect(f.starts()).toBe(1); expect(yield* f.service.acquire).toEqual(Option.some(mcp)); expect(f.starts()).toBe(2); @@ -543,6 +546,70 @@ describe("CuaDriver", () => { }); describe("Cua Driver readiness", () => { + it.effect("reports the host platform and runs only between readiness and disable", () => + Effect.scoped( + Effect.gen(function* () { + const f = yield* fixture(false).pipe(Effect.provideService(HostProcess.Platform, "darwin")); + expect(yield* f.service.status).toEqual({ platform: "darwin", running: false }); + yield* f.setEnabled(true); + const caller = yield* f.service.acquire.pipe(Effect.forkChild); + yield* Deferred.await(f.started); + expect(yield* f.service.status).toEqual({ platform: "darwin", running: false }); + yield* Deferred.succeed(f.result, mcp); + expect(yield* Fiber.join(caller)).toEqual(Option.some(mcp)); + expect(yield* f.service.status).toEqual({ platform: "darwin", running: true }); + yield* f.setEnabled(false); + yield* Deferred.await(f.stopped); + expect(yield* f.service.status).toEqual({ platform: "darwin", running: false }); + expect(yield* f.service.acquire).toEqual(Option.none()); + }), + ), + ); + + it.effect("reports a host start failure until the replacement is ready", () => + Effect.scoped( + Effect.gen(function* () { + const failure = new CuaDriver.CuaDriverStartError({ + binaryPath: "/driver", + cause: new Error("native start failed"), + }); + const restarting = yield* Deferred.make(); + const ready = yield* Deferred.make(); + let attempts = 0; + const service = yield* makeCuaDriver({ + enabled: Effect.succeed(true), + changes: Stream.empty, + createHost: Effect.sync(() => ({ + start: + ++attempts === 1 + ? Effect.fail(failure) + : Deferred.succeed(restarting, undefined).pipe( + Effect.andThen(Deferred.await(ready)), + ), + stop: Effect.void, + waitForExit: Effect.never, + })), + }).pipe(Effect.provideService(HostProcess.Platform, "win32")); + expect(yield* service.acquire).toEqual(Option.none()); + expect(yield* service.status).toEqual({ + platform: "win32", + running: false, + failure: failure.message, + }); + const retry = yield* service.acquire.pipe(Effect.forkChild); + yield* Deferred.await(restarting); + expect(yield* service.status).toEqual({ + platform: "win32", + running: false, + failure: failure.message, + }); + yield* Deferred.succeed(ready, mcp); + expect(yield* Fiber.join(retry)).toEqual(Option.some(mcp)); + expect(yield* service.status).toEqual({ platform: "win32", running: true }); + }), + ), + ); + it.effect("remembers why the last start failed until a start succeeds", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/cua/CuaDriver.ts b/apps/server/src/cua/CuaDriver.ts index 6769569b7435..7d8a3117ae44 100644 --- a/apps/server/src/cua/CuaDriver.ts +++ b/apps/server/src/cua/CuaDriver.ts @@ -149,7 +149,6 @@ export class CuaDriver extends Context.Service< readonly acquire: Effect.Effect>; /** Why the last start failed, until a later start succeeds. */ readonly lastFailure: Effect.Effect>; - /** What setup flows show: the host OS, and whether the driver runs or why not. */ readonly status: Effect.Effect; } >()("t3/cua/CuaDriver") {} diff --git a/apps/web/src/components/settings/CuaSetupDialog.tsx b/apps/web/src/components/settings/CuaSetupDialog.tsx index 44e4f2eec302..1d51e8c82557 100644 --- a/apps/web/src/components/settings/CuaSetupDialog.tsx +++ b/apps/web/src/components/settings/CuaSetupDialog.tsx @@ -1,5 +1,5 @@ import type { CuaHostStatus } from "@t3tools/contracts"; -import { useState } from "react"; +import { useRef, useState } from "react"; import { MacAccessibilityIcon, MacScreenRecordingIcon } from "../Icons"; import { PermissionChecklist, PermissionContinueButton } from "../permissions/PermissionChecklist"; @@ -19,25 +19,19 @@ export type CuaPermission = "accessibility" | "screen-recording"; const CUA_PERMISSIONS: readonly CuaPermission[] = ["accessibility", "screen-recording"]; -/** What the host itself needs before computer use can work there. */ function hostRequirements(platform: CuaHostStatus["platform"], hostLabel: string): string { switch (platform) { case "darwin": - return `T3 Code on ${hostLabel} needs Accessibility and Screen Recording. Open T3 Code on that Mac to grant them; the setting turns on from here either way.`; + return `${hostLabel} needs Accessibility and Screen Recording. Finish setup at that Mac: in System Settings → Privacy & Security, grant access to the app or process running this environment. You can turn the setting on from here.`; case "linux": - return `${hostLabel} needs a signed-in graphical session. On Wayland, someone at ${hostLabel} approves the screen capture prompt the first time an agent looks at the screen.`; + return `${hostLabel} needs a signed-in graphical session. On Wayland, someone at ${hostLabel} may need to approve a screen capture prompt.`; case "win32": - return `${hostLabel} needs a signed-in desktop session. No permissions are needed; Windows Defender may ask once before Cua Driver first runs.`; + return `${hostLabel} needs a signed-in desktop session. If Windows shows a security prompt for Cua Driver, someone at that computer must respond.`; default: return `Computer use needs a desktop on ${hostLabel}.`; } } -/** - * Setup for Cua computer use. On the Mac that hosts the environment it walks - * through Accessibility and Screen Recording, which macOS grants to T3 Code; - * elsewhere it explains what the host needs, since only someone at it can act. - */ export function CuaSetupDialog({ enabled, hostLabel, @@ -51,7 +45,6 @@ export function CuaSetupDialog({ enabled: boolean; hostLabel: string; platform: CuaHostStatus["platform"]; - /** This desktop is the Mac that runs the environment, so it can grant access here. */ localMac: boolean; onCheck: (permission: CuaPermission) => Promise; onAllow: (permission: CuaPermission) => Promise; @@ -59,6 +52,7 @@ export function CuaSetupDialog({ onClose: () => void; }) { const [busy, setBusy] = useState(false); + const pending = useRef(false); const [error, setError] = useState(null); const permission = usePermissionStatus( async () => ({ @@ -69,24 +63,32 @@ export function CuaSetupDialog({ localMac && !busy, ); const ready = !localMac || permission.isReady(CUA_PERMISSIONS); - const allow = (target: CuaPermission) => { - if (busy) return; + const allow = async (target: CuaPermission) => { + if (pending.current) return; + pending.current = true; setBusy(true); setError(null); - void onAllow(target) - .catch(() => setError("Could not open System Settings. Try Allow again.")) - .finally(() => setBusy(false)); + try { + await onAllow(target); + } catch { + setError("Could not open System Settings. Try Allow again."); + } finally { + pending.current = false; + setBusy(false); + } }; const finish = async () => { - if (busy) return; + if (pending.current || !ready) return; + pending.current = true; setBusy(true); setError(null); try { await onEnable(); onClose(); - } catch { - setError("Could not save the setting. Try again."); + } catch (error) { + setError(error instanceof Error ? error.message : "Could not save the setting. Try again."); } finally { + pending.current = false; setBusy(false); } }; @@ -94,7 +96,7 @@ export function CuaSetupDialog({ { - if (!open && !busy) onClose(); + if (!open && !pending.current) onClose(); }} > @@ -116,7 +118,7 @@ export function CuaSetupDialog({ title: "Accessibility", description: "Click, type, and read controls in other apps.", granted: permission.status.accessibility, - onAllow: () => allow("accessibility"), + onAllow: () => void allow("accessibility"), }, { id: "screen-recording", @@ -124,7 +126,7 @@ export function CuaSetupDialog({ title: "Screen Recording", description: "See the screen to decide what to do next.", granted: permission.status["screen-recording"], - onAllow: () => allow("screen-recording"), + onAllow: () => void allow("screen-recording"), }, ]} /> @@ -143,7 +145,13 @@ export function CuaSetupDialog({ ) : null} - void finish()}> diff --git a/apps/web/src/components/settings/IntegrationsSettings.tsx b/apps/web/src/components/settings/IntegrationsSettings.tsx index 0dfe90a45489..54006b50b09d 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.tsx @@ -4,6 +4,7 @@ import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings" import { ScopedSwitch } from "./ScopedSwitch"; import { DeviceSourcesSettings } from "./DeviceSourcesSettings"; import { CuaSetupDialog } from "./CuaSetupDialog"; +import { SettingsScopeNotice } from "./SettingsScopeNotice"; import { AuthSettingsWriteScope, BrowserImportFailureReason, @@ -39,7 +40,7 @@ import { import { isValidSearchUrlTemplate } from "@t3tools/shared/preview"; import { PREVIEW_VIEWPORT_PRESETS } from "@t3tools/shared/previewViewport"; import { MoreVertical, Plus as PlusIcon } from "lucide-react"; -import { useRef, useState } from "react"; +import { useEffect, useEffectEvent, useRef, useState } from "react"; import { ScreenRotationIcon } from "~/browser/ScreenRotationIcon"; import { resolveEnvironmentOptionLabel } from "~/components/BranchToolbar.logic"; @@ -1052,23 +1053,61 @@ function AgentBrowserAccessSetting() { ); } -/** - * Cua computer use for the selected environment. The switch records intent; - * the host status shows whether the driver can actually start there. - */ function ComputerUseSettings() { + const { scope, environment, search } = useSettingsScope(); + const singleHost = + scope.kind === "environment" || (scope.kind === "all" && scope.environmentIds.length === 1); + return ( + + {singleHost && environment ? ( + + ) : ( + + {singleHost + ? "Connect the selected environment to configure computer use." + : "Choose one environment to configure computer use on its host."} + + )} + + ); +} + +function ComputerUseControls() { const { environment: selected } = useSettingsScope(); const settings = useScopedSettings(); const updateSettings = useUpdateScopedSettings(); const [setupOpen, setSetupOpen] = useState(false); + const [saving, setSaving] = useState(false); const environmentId = selected?.connection.phase === "connected" && selected.serverConfig !== null ? selected.environmentId : null; const hostLabel = selected?.label ?? "this computer"; - const { data: status, refresh } = useEnvironmentQuery( + const { + data: status, + error, + isPending, + refresh, + } = useEnvironmentQuery( environmentId === null ? null : serverEnvironment.computerUse({ environmentId, input: {} }), ); + const canConfigure = useEnvironmentScope(environmentId, AuthSettingsWriteScope); + if (setupOpen && !canConfigure) setSetupOpen(false); + const refreshVisible = useEffectEvent(() => { + if (!isPending && document.visibilityState !== "hidden") refresh(); + }); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Polls the host's Cua status through a timer and focus/visibility listeners while this row is mounted. + useEffect(() => { + if (environmentId === null) return; + const timer = window.setInterval(refreshVisible, 3000); + window.addEventListener("focus", refreshVisible); + document.addEventListener("visibilitychange", refreshVisible); + return () => { + window.clearInterval(timer); + window.removeEventListener("focus", refreshVisible); + document.removeEventListener("visibilitychange", refreshVisible); + }; + }, [environmentId]); const platform = status?.platform ?? "other"; // Only the Mac that runs the environment can grant it macOS permissions. const localMac = @@ -1077,51 +1116,76 @@ function ComputerUseSettings() { window.desktopBridge?.getClientPlatform() === "darwin" && selected?.entry.target._tag === "PrimaryConnectionTarget"; const enabled = settings.enableCua; - const statusText = !enabled - ? null - : status?.running - ? "Running" - : status?.failure - ? status.failure - : "Starts with the next agent session"; + const statusText = error + ? `Could not check computer use: ${error}` + : status === null + ? "Checking host…" + : platform === "other" + ? "Computer use is not supported on this host." + : !enabled + ? null + : status.running + ? "Running" + : (status.failure ?? "Starts with the next agent session"); + + const save = async (enableCua: boolean) => { + setSaving(true); + try { + const result = await updateSettings({ enableCua }); + refresh(); + return result; + } finally { + setSaving(false); + } + }; return ( - + <> void updateSettings({ enableCua: DEFAULT_SERVER_SETTINGS.enableCua })} + onClick={() => void save(DEFAULT_SERVER_SETTINGS.enableCua)} /> ) : null } control={ <> - {enabled && platform !== "other" ? ( - ) : null} { if (checked) setSetupOpen(true); - else void updateSettings({ enableCua: false }); + else void save(false); }} /> } /> - {setupOpen ? ( + {setupOpen && canConfigure && environmentId !== null ? ( { - if (!enabled) await updateSettings({ enableCua: true }); + if (!readEnvironmentScope(environmentId, AuthSettingsWriteScope)) { + throw new Error("This connection cannot change settings on this host."); + } + if (!enabled) { + const result = await save(true); + if (!result.saved) + throw new Error(result.message ?? "Could not save the setting. Try again."); + } // A driver started before the grants keeps macOS's cached denial. - if (localMac) await window.desktopBridge?.restartCuaDriver(); + if (localMac) { + try { + await window.desktopBridge?.restartCuaDriver(); + } catch { + throw new Error( + "Computer use is on, but Cua Driver could not restart. Try Setup again.", + ); + } + } }} onClose={() => { setSetupOpen(false); @@ -1146,7 +1225,7 @@ function ComputerUseSettings() { }} /> ) : null} - + ); } diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index 743c193e61ce..b635031bfdda 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -606,7 +606,7 @@ export function useSettingsRestore(onRestored?: () => void) { notifyThemeRestoreFailure(); return; } - updateSettings({ + const result = await updateSettings({ appearanceContrast: DEFAULT_UNIFIED_SETTINGS.appearanceContrast, diffColorScheme: DEFAULT_UNIFIED_SETTINGS.diffColorScheme, chatWidth: DEFAULT_UNIFIED_SETTINGS.chatWidth, @@ -676,7 +676,7 @@ export function useSettingsRestore(onRestored?: () => void) { enableAgentBrowserAccess: DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess, enableCua: DEFAULT_UNIFIED_SETTINGS.enableCua, }); - onRestored?.(); + if (result.saved) onRestored?.(); }; return { diff --git a/apps/web/src/components/settings/SourceControlWritingSettings.browser.test.tsx b/apps/web/src/components/settings/SourceControlWritingSettings.browser.test.tsx index 5a44beb50ac8..c5441cb65a59 100644 --- a/apps/web/src/components/settings/SourceControlWritingSettings.browser.test.tsx +++ b/apps/web/src/components/settings/SourceControlWritingSettings.browser.test.tsx @@ -5,12 +5,11 @@ import { describe, expect, it, vi } from "vite-plus/test"; import { page, userEvent } from "vite-plus/test/browser"; import "../../index.css"; -import type { ScopedSettingsPatch } from "./scopedSettings"; type WritingStyle = typeof DEFAULT_UNIFIED_SETTINGS.sourceControlWritingStyle; const state = vi.hoisted(() => ({ styles: [] as WritingStyle[], - updateSettings: vi.fn<(patch: ScopedSettingsPatch) => void>(), + updateSettings: vi.fn>(), })); vi.mock(import("@tanstack/react-router"), async (original) => ({ @@ -87,6 +86,7 @@ async function setup() { state.updateSettings.mockReset().mockImplementation((patch) => { state.styles = state.styles.map((style) => ({ ...style, ...patch.sourceControlWritingStyle })); void screen.rerender(content()); + return Promise.resolve({ saved: true, message: null }); }); return screen; } diff --git a/apps/web/src/components/settings/useScopedSettings.ts b/apps/web/src/components/settings/useScopedSettings.ts index 5fb9d048a9e3..2853eea1f735 100644 --- a/apps/web/src/components/settings/useScopedSettings.ts +++ b/apps/web/src/components/settings/useScopedSettings.ts @@ -49,7 +49,10 @@ export function useScopedSettingSource(keys: readonly (keyof ServerSettings)[]) function useRunScopedPlan() { const persistServer = useAtomCommand(serverEnvironment.updateSettings, { reportFailure: false }); - return (plan: ReturnType, onFailed?: () => void) => { + return async ( + plan: ReturnType, + onFailed?: () => void, + ): Promise<{ readonly saved: boolean; readonly message: string | null }> => { if (plan.unavailableReason) { toastManager.add({ type: "warning", @@ -57,43 +60,50 @@ function useRunScopedPlan() { description: plan.unavailableReason, }); onFailed?.(); - return; + return { saved: false, message: plan.unavailableReason }; } - void persistScopedSettingsPatch( - plan, - async (request) => { - const missing = requiredScopesForServerSettingsPatch(request.input.patch).find( - (scope) => !readEnvironmentScope(request.environmentId, scope), + try { + const { failedEnvironments, savedEnvironments, savedEnvironmentCount } = + await persistScopedSettingsPatch( + plan, + async (request) => { + const missing = requiredScopesForServerSettingsPatch(request.input.patch).find( + (scope) => !readEnvironmentScope(request.environmentId, scope), + ); + if (missing) + return AsyncResult.failure( + Cause.fail( + new EnvironmentAuthorizationError({ + requiredScope: missing, + message: "This connection lacks permission to change these settings.", + }), + ), + ); + return persistServer(request); + }, + persistClientSettingsPatch, ); - if (missing) - return AsyncResult.failure( - Cause.fail( - new EnvironmentAuthorizationError({ - requiredScope: missing, - message: "This connection lacks permission to change these settings.", - }), - ), - ); - return persistServer(request); - }, - persistClientSettingsPatch, - ).then(({ failedEnvironments, savedEnvironments, savedEnvironmentCount }) => { - if (failedEnvironments.length === 0) return; + if (failedEnvironments.length === 0) return { saved: true, message: null }; onFailed?.(); + const message = [ + ...failedEnvironments.map(({ label, message }) => `Could not save on ${label}: ${message}`), + ...(savedEnvironmentCount > 0 + ? [`Saved on ${savedEnvironments.map(({ label }) => label).join(", ")}.`] + : []), + ].join("\n"); toastManager.add({ type: "error", title: savedEnvironmentCount > 0 ? "Setting saved on some environments" : "Setting not saved", - description: [ - ...failedEnvironments.map( - ({ label, message }) => `Could not save on ${label}: ${message}`, - ), - ...(savedEnvironmentCount > 0 - ? [`Saved on ${savedEnvironments.map(({ label }) => label).join(", ")}.`] - : []), - ].join("\n"), + description: message, }); - }); + return { saved: false, message }; + } catch (error) { + onFailed?.(); + const message = error instanceof Error ? error.message : "The save failed. Try again."; + toastManager.add({ type: "error", title: "Setting not saved", description: message }); + return { saved: false, message }; + } }; } @@ -110,7 +120,7 @@ export function useClearScopedSettings() { const run = useRunScopedPlan(); return (keys: readonly ProjectScopedServerSettingKey[]) => { if (context === null) return; - run(planScopedSettingsClear(context.scope, context.environments, keys)); + void run(planScopedSettingsClear(context.scope, context.environments, keys)); }; } @@ -122,7 +132,7 @@ export function useClearProjectOverrides() { keys: readonly ProjectScopedServerSettingKey[], ) => { if (context === null) return; - run(planProjectOverridesClear(context.environments, entries, keys)); + void run(planProjectOverridesClear(context.environments, entries, keys)); }; } diff --git a/docs/user/computer-use.md b/docs/user/computer-use.md index 4f1f135d337b..991e3b27a2c8 100644 --- a/docs/user/computer-use.md +++ b/docs/user/computer-use.md @@ -1,8 +1,7 @@ # Computer use Computer use lets agents see and use apps on the computer that runs your T3 -Code environment. Every provider gets it: Claude, Codex, Cursor, Grok, -Antigravity, OpenCode, Pi, and ACP agents. It runs through +Code environment. It runs through [Cua Driver](https://cua.ai/docs/cua-driver), which acts on windows in the background, so agents do not take over your cursor or keyboard while you work. @@ -10,23 +9,23 @@ It works on macOS, Windows, and Linux. It is off by default. ## Turn it on -Open **Settings → Integrations → Computer**, select the environment, and turn -on **Computer use**. Agent sessions started afterwards get the computer use -tools. Turning it off stops the driver; sessions already running lose the tools -on their next start. +Open **Settings → Devices → Computer**, choose one environment, and turn +on **Computer use**. On mobile, use **Settings → Threads → Computer**. +Agent sessions started afterwards get the computer use tools. Turning it off +stops the driver immediately; restart existing agent sessions to remove their tools. What the host needs depends on its OS: - **macOS:** T3 Code needs Accessibility and Screen Recording. On the Mac that runs the environment, the setup walks you through both. From another computer or the mobile app, the switch turns on, but someone at that Mac still has to - grant them from T3 Code there. Choose **Setup** next to the switch to check - them again later. -- **Windows:** someone must be signed in to the desktop. No permissions are - needed. Windows Defender may ask once before Cua Driver first runs. + grant them there. For a server run from a terminal or service, grant access to + the process hosting the server in **System Settings → Privacy & Security**. + Choose **Setup** next to the switch in the desktop app to check its grants again. +- **Windows:** someone must be signed in to the desktop and respond to any + security prompts for Cua Driver at that computer. - **Linux:** the host needs a signed-in graphical session. On Wayland, someone - at the computer approves the screen capture prompt the first time an agent - looks at the screen. + at the computer may need to approve a screen capture prompt. The desktop app includes Cua Driver. A server started with `npx t3` downloads the same version into T3 Code's home the first time an agent session needs it. diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index a82ce84c56e4..a8d8fa984456 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -1252,7 +1252,6 @@ export function createServerEnvironmentAtoms( label: "environment-data:provider:install-remove", tag: WS_METHODS.providerInstallRemove, }), - /** Whether the host can run computer use; rechecked when the switch flips. */ computerUse: createEnvironmentRpcQueryAtomFamily(runtime, { label: "environment-data:server:computer-use", tag: WS_METHODS.serverCheckComputerUse, diff --git a/packages/contracts/src/cua.ts b/packages/contracts/src/cua.ts index eb7b3600185d..c0b97a7bd75b 100644 --- a/packages/contracts/src/cua.ts +++ b/packages/contracts/src/cua.ts @@ -38,14 +38,9 @@ export const DesktopCuaDriverReport = Schema.Union([ ]); export type DesktopCuaDriverReport = typeof DesktopCuaDriverReport.Type; -/** - * Whether the environment's host can run computer use right now. Setup flows - * show it; the switch itself records intent and is not a readiness signal. - */ +/** Driver readiness is independent of the enableCua setting. */ export const CuaHostStatus = Schema.Struct({ - /** The host's OS, which decides what the user must grant. */ platform: Schema.Literals(["darwin", "linux", "win32", "other"]), - /** The driver started for an agent session and is running. */ running: Schema.Boolean, /** Why the last start failed, until a later start succeeds. */ failure: Schema.optionalKey(Schema.String), From dbfb88eacaba34bd8a4ba219a425a7a68d37d54a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 22:19:34 +0000 Subject: [PATCH 3/3] fix(web): keep computer use handlers compatible with React Compiler --- apps/web/src/components/settings/CuaSetupDialog.tsx | 10 ++++------ .../src/components/settings/IntegrationsSettings.tsx | 11 ++++------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/apps/web/src/components/settings/CuaSetupDialog.tsx b/apps/web/src/components/settings/CuaSetupDialog.tsx index 1d51e8c82557..7001a6309f00 100644 --- a/apps/web/src/components/settings/CuaSetupDialog.tsx +++ b/apps/web/src/components/settings/CuaSetupDialog.tsx @@ -72,10 +72,9 @@ export function CuaSetupDialog({ await onAllow(target); } catch { setError("Could not open System Settings. Try Allow again."); - } finally { - pending.current = false; - setBusy(false); } + pending.current = false; + setBusy(false); }; const finish = async () => { if (pending.current || !ready) return; @@ -87,10 +86,9 @@ export function CuaSetupDialog({ onClose(); } catch (error) { setError(error instanceof Error ? error.message : "Could not save the setting. Try again."); - } finally { - pending.current = false; - setBusy(false); } + pending.current = false; + setBusy(false); }; return ( { setSaving(true); - try { - const result = await updateSettings({ enableCua }); - refresh(); - return result; - } finally { - setSaving(false); - } + const result = await updateSettings({ enableCua }); + refresh(); + setSaving(false); + return result; }; return (