diff --git a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx index 91b0b1b044e5..e204001a928e 100644 --- a/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsServerControlsRouteScreen.tsx @@ -377,6 +377,16 @@ function ServerSettingsDetail(props: { readonly page: SettingsPage }) { onValueChange={(value) => write({ enableAgentBrowserAccess: value })} /> + + write({ enableCua: value })} + /> + 0} /> ) : null} diff --git a/apps/server/src/auth/RpcAuthorization.test.ts b/apps/server/src/auth/RpcAuthorization.test.ts index 8044740a7e2e..49af649cb521 100644 --- a/apps/server/src/auth/RpcAuthorization.test.ts +++ b/apps/server/src/auth/RpcAuthorization.test.ts @@ -22,6 +22,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; import * as RpcTest from "effect/rpc/RpcTest"; +import { applyServerSettingsPatch } from "@t3tools/shared/serverSettings"; import { RPC_REQUIRED_SCOPES, @@ -147,6 +148,31 @@ describe("settings mutation authorization", () => { instanceId: ProviderInstanceId.make("codex_work"), }; + it.effect("restores computer use alongside duration settings through authorization", () => + Effect.gen(function* () { + const client = yield* RpcTest.makeClient(group).pipe( + Effect.provide( + Layer.mergeAll( + group.toLayerHandler(WS_METHODS.serverUpdateSettings, ({ patch }) => + Effect.succeed( + applyServerSettingsPatch({ ...DEFAULT_SERVER_SETTINGS, enableCua: true }, patch), + ), + ), + RpcAuthorization.layer([AuthSettingsWriteScope]), + ), + ), + ); + const defaults = { + enableCua: DEFAULT_SERVER_SETTINGS.enableCua, + automaticGitFetchInterval: DEFAULT_SERVER_SETTINGS.automaticGitFetchInterval, + providerHealthRefreshInterval: DEFAULT_SERVER_SETTINGS.providerHealthRefreshInterval, + }; + expect(yield* client[WS_METHODS.serverUpdateSettings]({ patch: defaults })).toMatchObject( + defaults, + ); + }).pipe(Effect.scoped), + ); + it.effect("allows provider-only mutations while denying mixed settings without their grant", () => Effect.gen(function* () { let handled = 0; diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index bf2b8d49daf8..956f978bb974 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -61,6 +61,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.projectsMutate]: AuthOrchestrationOperateScope, [WS_METHODS.serverProbe]: AuthOrchestrationReadScope, [WS_METHODS.serverGetConfig]: AuthOrchestrationReadScope, + [WS_METHODS.serverCheckComputerUse]: AuthOrchestrationReadScope, [WS_METHODS.serverRefreshProviders]: AuthOrchestrationReadScope, [WS_METHODS.serverUpdateProvider]: AuthProvidersManageScope, [WS_METHODS.providerAuthStart]: AuthProvidersManageScope, diff --git a/apps/server/src/cua/CuaDriver.test.ts b/apps/server/src/cua/CuaDriver.test.ts index 7d3fc531b18a..ba423c356947 100644 --- a/apps/server/src/cua/CuaDriver.test.ts +++ b/apps/server/src/cua/CuaDriver.test.ts @@ -1,6 +1,7 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { it } from "@effect/vitest"; import type { CuaDriverMcpConfiguration, DesktopCuaDriverReport } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import type { EmbeddedDriverConnection } from "@trycua/cua-driver/embedded"; import { Deferred, Effect, Exit, Fiber, Option, PubSub, Ref, Scope, Stream } from "effect"; import { TestClock } from "effect/testing"; @@ -501,8 +502,10 @@ describe("CuaDriver", () => { yield* Deferred.succeed(f.result, mcp); expect(yield* f.service.acquire).toEqual(Option.some(mcp)); yield* Deferred.await(f.watching); + expect((yield* f.service.status).running).toBe(true); yield* Deferred.succeed(f.exited, undefined); yield* Deferred.await(f.stopped); + expect((yield* f.service.status).running).toBe(false); expect(f.starts()).toBe(1); expect(yield* f.service.acquire).toEqual(Option.some(mcp)); expect(f.starts()).toBe(2); @@ -543,6 +546,70 @@ describe("CuaDriver", () => { }); describe("Cua Driver readiness", () => { + it.effect("reports the host platform and runs only between readiness and disable", () => + Effect.scoped( + Effect.gen(function* () { + const f = yield* fixture(false).pipe(Effect.provideService(HostProcess.Platform, "darwin")); + expect(yield* f.service.status).toEqual({ platform: "darwin", running: false }); + yield* f.setEnabled(true); + const caller = yield* f.service.acquire.pipe(Effect.forkChild); + yield* Deferred.await(f.started); + expect(yield* f.service.status).toEqual({ platform: "darwin", running: false }); + yield* Deferred.succeed(f.result, mcp); + expect(yield* Fiber.join(caller)).toEqual(Option.some(mcp)); + expect(yield* f.service.status).toEqual({ platform: "darwin", running: true }); + yield* f.setEnabled(false); + yield* Deferred.await(f.stopped); + expect(yield* f.service.status).toEqual({ platform: "darwin", running: false }); + expect(yield* f.service.acquire).toEqual(Option.none()); + }), + ), + ); + + it.effect("reports a host start failure until the replacement is ready", () => + Effect.scoped( + Effect.gen(function* () { + const failure = new CuaDriver.CuaDriverStartError({ + binaryPath: "/driver", + cause: new Error("native start failed"), + }); + const restarting = yield* Deferred.make(); + const ready = yield* Deferred.make(); + let attempts = 0; + const service = yield* makeCuaDriver({ + enabled: Effect.succeed(true), + changes: Stream.empty, + createHost: Effect.sync(() => ({ + start: + ++attempts === 1 + ? Effect.fail(failure) + : Deferred.succeed(restarting, undefined).pipe( + Effect.andThen(Deferred.await(ready)), + ), + stop: Effect.void, + waitForExit: Effect.never, + })), + }).pipe(Effect.provideService(HostProcess.Platform, "win32")); + expect(yield* service.acquire).toEqual(Option.none()); + expect(yield* service.status).toEqual({ + platform: "win32", + running: false, + failure: failure.message, + }); + const retry = yield* service.acquire.pipe(Effect.forkChild); + yield* Deferred.await(restarting); + expect(yield* service.status).toEqual({ + platform: "win32", + running: false, + failure: failure.message, + }); + yield* Deferred.succeed(ready, mcp); + expect(yield* Fiber.join(retry)).toEqual(Option.some(mcp)); + expect(yield* service.status).toEqual({ platform: "win32", running: true }); + }), + ), + ); + it.effect("remembers why the last start failed until a start succeeds", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/cua/CuaDriver.ts b/apps/server/src/cua/CuaDriver.ts index 928b09c450b6..7d8a3117ae44 100644 --- a/apps/server/src/cua/CuaDriver.ts +++ b/apps/server/src/cua/CuaDriver.ts @@ -1,6 +1,11 @@ import * as NodeModule from "node:module"; -import type { CuaDriverMcpConfiguration, DesktopCuaDriverReport } from "@t3tools/contracts"; +import type { + CuaDriverMcpConfiguration, + CuaHostStatus, + DesktopCuaDriverReport, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import type { EmbeddedCuaDriverHost, EmbeddedDriverConnection } from "@trycua/cua-driver/embedded"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; @@ -144,6 +149,7 @@ export class CuaDriver extends Context.Service< readonly acquire: Effect.Effect>; /** Why the last start failed, until a later start succeeds. */ readonly lastFailure: Effect.Effect>; + readonly status: Effect.Effect; } >()("t3/cua/CuaDriver") {} @@ -175,6 +181,8 @@ export const make = Effect.fn("CuaDriver.make")(function* ( stopped?: boolean; /** Message of the typed error that ended this attempt, for the readiness signal. */ failure?: string; + /** Set once the host reported its MCP descriptor. */ + running?: boolean; }; let current: Attempt | undefined; let closed = false; @@ -241,6 +249,7 @@ export const make = Effect.fn("CuaDriver.make")(function* ( return; } lastFailure = undefined; + attempt.running = true; yield* Deferred.succeed(attempt.ready, Option.some(mcp)); }), ); @@ -290,10 +299,17 @@ export const make = Effect.fn("CuaDriver.make")(function* ( return attempt ? yield* Deferred.await(attempt.ready) : Option.none(); }); + const platform = yield* HostProcess.Platform; return CuaDriver.of({ enabled: enabled, acquire, lastFailure: Effect.sync(() => Option.fromNullishOr(lastFailure)), + status: Effect.sync(() => ({ + platform: + platform === "darwin" || platform === "linux" || platform === "win32" ? platform : "other", + running: current?.running === true, + ...(lastFailure === undefined ? {} : { failure: lastFailure }), + })), }); }); diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index cb8b0b6a86cf..e5a1ae6e3982 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -32,6 +32,7 @@ const RPC_AGGREGATES = { [ORCHESTRATION_V2_WS_METHODS.subscribeCommandOutput]: "orchestrationV2", [WS_METHODS.projectsMutate]: "orchestration", [WS_METHODS.serverProbe]: "server", + [WS_METHODS.serverCheckComputerUse]: "server", [WS_METHODS.serverGetConfig]: "server", [WS_METHODS.serverRefreshProviders]: "server", [WS_METHODS.serverUpdateProvider]: "server", diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index cfe49ad39d52..ea601276ad83 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -187,6 +187,7 @@ import * as TerminalManager from "./terminal/Manager.ts"; import { terminalOutputStream, withTerminalOutputWindow } from "./terminal/OutputProtocol.ts"; import * as ServerBrowser from "./preview/ServerBrowser.ts"; import * as DeviceService from "./device/DeviceService.ts"; +import * as CuaDriver from "./cua/CuaDriver.ts"; import { remoteSshDeviceHosts } from "./device/localSshDeviceHost.ts"; import * as PreviewManager from "./preview/Manager.ts"; import { issueAssetUrl } from "./assets/AssetAccess.ts"; @@ -2176,6 +2177,14 @@ export const layerWsRpc = ( Effect.andThen(scheduledTasks.getWebhookDelivery(input)), ), [WS_METHODS.serverProbe]: (_input) => Effect.succeed({}), + [WS_METHODS.serverCheckComputerUse]: (_input) => + Effect.serviceOption(CuaDriver.CuaDriver).pipe( + Effect.flatMap((driver) => + Option.isSome(driver) + ? driver.value.status + : Effect.succeed({ platform: "other", running: false } as const), + ), + ), [WS_METHODS.serverGetConfig]: (_input) => loadServerConfig({ usageLimitsCommand: false, workspaceSnapshotsOnDemand: false }), [WS_METHODS.serverSearchAcpRegistry]: (input) => diff --git a/apps/web/src/components/settings/CuaSetupDialog.tsx b/apps/web/src/components/settings/CuaSetupDialog.tsx new file mode 100644 index 000000000000..7001a6309f00 --- /dev/null +++ b/apps/web/src/components/settings/CuaSetupDialog.tsx @@ -0,0 +1,162 @@ +import type { CuaHostStatus } from "@t3tools/contracts"; +import { useRef, useState } from "react"; + +import { MacAccessibilityIcon, MacScreenRecordingIcon } from "../Icons"; +import { PermissionChecklist, PermissionContinueButton } from "../permissions/PermissionChecklist"; +import { usePermissionStatus } from "../permissions/usePermissionStatus"; +import { Button } from "../ui/button"; +import { + Dialog, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPanel, + DialogPopup, + DialogTitle, +} from "../ui/dialog"; + +export type CuaPermission = "accessibility" | "screen-recording"; + +const CUA_PERMISSIONS: readonly CuaPermission[] = ["accessibility", "screen-recording"]; + +function hostRequirements(platform: CuaHostStatus["platform"], hostLabel: string): string { + switch (platform) { + case "darwin": + return `${hostLabel} needs Accessibility and Screen Recording. Finish setup at that Mac: in System Settings → Privacy & Security, grant access to the app or process running this environment. You can turn the setting on from here.`; + case "linux": + return `${hostLabel} needs a signed-in graphical session. On Wayland, someone at ${hostLabel} may need to approve a screen capture prompt.`; + case "win32": + return `${hostLabel} needs a signed-in desktop session. If Windows shows a security prompt for Cua Driver, someone at that computer must respond.`; + default: + return `Computer use needs a desktop on ${hostLabel}.`; + } +} + +export function CuaSetupDialog({ + enabled, + hostLabel, + platform, + localMac, + onCheck, + onAllow, + onEnable, + onClose, +}: { + enabled: boolean; + hostLabel: string; + platform: CuaHostStatus["platform"]; + localMac: boolean; + onCheck: (permission: CuaPermission) => Promise; + onAllow: (permission: CuaPermission) => Promise; + onEnable: () => Promise; + onClose: () => void; +}) { + const [busy, setBusy] = useState(false); + const pending = useRef(false); + const [error, setError] = useState(null); + const permission = usePermissionStatus( + async () => ({ + accessibility: await onCheck("accessibility"), + "screen-recording": await onCheck("screen-recording"), + }), + { accessibility: false, "screen-recording": false }, + localMac && !busy, + ); + const ready = !localMac || permission.isReady(CUA_PERMISSIONS); + const allow = async (target: CuaPermission) => { + if (pending.current) return; + pending.current = true; + setBusy(true); + setError(null); + try { + await onAllow(target); + } catch { + setError("Could not open System Settings. Try Allow again."); + } + pending.current = false; + setBusy(false); + }; + const finish = async () => { + if (pending.current || !ready) return; + pending.current = true; + setBusy(true); + setError(null); + try { + await onEnable(); + onClose(); + } catch (error) { + setError(error instanceof Error ? error.message : "Could not save the setting. Try again."); + } + pending.current = false; + setBusy(false); + }; + return ( + { + if (!open && !pending.current) onClose(); + }} + > + + + Let agents use {hostLabel} + + Cua Driver lets agents see and use apps on {hostLabel} in the background, without moving + your cursor. Agent sessions started afterwards get the computer use tools. + + + + {localMac ? ( + , + title: "Accessibility", + description: "Click, type, and read controls in other apps.", + granted: permission.status.accessibility, + onAllow: () => void allow("accessibility"), + }, + { + id: "screen-recording", + icon: , + title: "Screen Recording", + description: "See the screen to decide what to do next.", + granted: permission.status["screen-recording"], + onAllow: () => void allow("screen-recording"), + }, + ]} + /> + ) : ( +

{hostRequirements(platform, hostLabel)}

+ )} + {error || (localMac && permission.error) ? ( +

+ {error ?? permission.error} +

+ ) : null} + {localMac && !ready ? ( +

+ If a permission stays off after you allow it, quit and reopen T3 Code. +

+ ) : null} + + + + void finish()}> + {enabled ? "Done" : "Turn on"} + + + + + ); +} diff --git a/apps/web/src/components/settings/IntegrationsSettings.tsx b/apps/web/src/components/settings/IntegrationsSettings.tsx index 5239f57bfdc3..6a0d35c87a16 100644 --- a/apps/web/src/components/settings/IntegrationsSettings.tsx +++ b/apps/web/src/components/settings/IntegrationsSettings.tsx @@ -3,6 +3,8 @@ import { DeviceToolVersions } from "../device/DeviceToolVersions"; import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings"; import { ScopedSwitch } from "./ScopedSwitch"; import { DeviceSourcesSettings } from "./DeviceSourcesSettings"; +import { CuaSetupDialog } from "./CuaSetupDialog"; +import { SettingsScopeNotice } from "./SettingsScopeNotice"; import { AuthSettingsWriteScope, BrowserImportFailureReason, @@ -38,7 +40,7 @@ import { import { isValidSearchUrlTemplate } from "@t3tools/shared/preview"; import { PREVIEW_VIEWPORT_PRESETS } from "@t3tools/shared/previewViewport"; import { MoreVertical, Plus as PlusIcon } from "lucide-react"; -import { useRef, useState } from "react"; +import { useEffect, useEffectEvent, useRef, useState } from "react"; import { ScreenRotationIcon } from "~/browser/ScreenRotationIcon"; import { resolveEnvironmentOptionLabel } from "~/components/BranchToolbar.logic"; @@ -48,7 +50,8 @@ import { useEnvironments, usePrimaryEnvironment } from "~/state/environments"; import { deviceEnvironment, useDeviceState } from "~/state/device"; import { readEnvironmentScope, useEnvironmentScope } from "~/state/session"; import { useAtomCommand } from "~/state/use-atom-command"; -import { formatEnvironmentQueryError } from "~/state/query"; +import { formatEnvironmentQueryError, useEnvironmentQuery } from "~/state/query"; +import { serverEnvironment } from "~/state/server"; import { previewEnvironment } from "~/state/preview"; import { useServerConfigs } from "~/state/entities"; import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; @@ -1050,6 +1053,179 @@ function AgentBrowserAccessSetting() { ); } +function ComputerUseSettings() { + const { scope, environment, search } = useSettingsScope(); + const singleHost = + scope.kind === "environment" || (scope.kind === "all" && scope.environmentIds.length === 1); + return ( + + {singleHost && environment ? ( + + ) : ( + + {singleHost + ? "Connect the selected environment to configure computer use." + : "Choose one environment to configure computer use on its host."} + + )} + + ); +} + +function ComputerUseControls() { + const { environment: selected } = useSettingsScope(); + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + const [setupOpen, setSetupOpen] = useState(false); + const [saving, setSaving] = useState(false); + const environmentId = + selected?.connection.phase === "connected" && selected.serverConfig !== null + ? selected.environmentId + : null; + const hostLabel = selected?.label ?? "this computer"; + const { + data: status, + error, + isPending, + refresh, + } = useEnvironmentQuery( + environmentId === null ? null : serverEnvironment.computerUse({ environmentId, input: {} }), + ); + const canConfigure = useEnvironmentScope(environmentId, AuthSettingsWriteScope); + if (setupOpen && !canConfigure) setSetupOpen(false); + const refreshVisible = useEffectEvent(() => { + if (!isPending && document.visibilityState !== "hidden") refresh(); + }); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Polls the host's Cua status through a timer and focus/visibility listeners while this row is mounted. + useEffect(() => { + if (environmentId === null) return; + const timer = window.setInterval(refreshVisible, 3000); + window.addEventListener("focus", refreshVisible); + document.addEventListener("visibilitychange", refreshVisible); + return () => { + window.clearInterval(timer); + window.removeEventListener("focus", refreshVisible); + document.removeEventListener("visibilitychange", refreshVisible); + }; + }, [environmentId]); + const platform = status?.platform ?? "other"; + // Only the Mac that runs the environment can grant it macOS permissions. + const localMac = + isElectron && + platform === "darwin" && + window.desktopBridge?.getClientPlatform() === "darwin" && + selected?.entry.target._tag === "PrimaryConnectionTarget"; + const enabled = settings.enableCua; + const statusText = error + ? `Could not check computer use: ${error}` + : status === null + ? "Checking host…" + : platform === "other" + ? "Computer use is not supported on this host." + : !enabled + ? null + : status.running + ? "Running" + : (status.failure ?? "Starts with the next agent session"); + + const save = async (enableCua: boolean) => { + setSaving(true); + const result = await updateSettings({ enableCua }); + refresh(); + setSaving(false); + return result; + }; + + return ( + <> + void save(DEFAULT_SERVER_SETTINGS.enableCua)} + /> + ) : null + } + control={ + <> + {enabled && status !== null && !error && platform !== "other" ? ( + + ) : null} + { + if (checked) setSetupOpen(true); + else void save(false); + }} + /> + + } + /> + {setupOpen && canConfigure && environmentId !== null ? ( + + window.desktopBridge?.checkSystemPermission(permission) ?? Promise.resolve(false) + } + onAllow={async (permission) => { + const api = readLocalApi(); + if (!api) throw new Error("Unable to open System Settings."); + await api.shell.openSystemSettings(permission); + }} + onEnable={async () => { + if (!readEnvironmentScope(environmentId, AuthSettingsWriteScope)) { + throw new Error("This connection cannot change settings on this host."); + } + if (!enabled) { + const result = await save(true); + if (!result.saved) + throw new Error(result.message ?? "Could not save the setting. Try again."); + } + // A driver started before the grants keeps macOS's cached denial. + if (localMac) { + try { + await window.desktopBridge?.restartCuaDriver(); + } catch { + throw new Error( + "Computer use is on, but Cua Driver could not restart. Try Setup again.", + ); + } + } + }} + onClose={() => { + setSetupOpen(false); + refresh(); + }} + /> + ) : null} + + ); +} + function BrowserAutoShowFloatingPreviewSetting({ disabled }: { readonly disabled: boolean }) { const autoShow = useClientSettings((settings) => settings.browserAutoShowFloatingPreview); const updateSettings = useUpdatePrimarySettings(); @@ -1591,6 +1767,7 @@ export function DevicesSettingsPanel() { return ( + ); } diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index c4e3d27a07fa..b635031bfdda 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -552,6 +552,7 @@ export function useSettingsRestore(onRestored?: () => void) { ...(settings.enableAgentBrowserAccess !== DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess ? ["Agent browser access"] : []), + ...(settings.enableCua !== DEFAULT_UNIFIED_SETTINGS.enableCua ? ["Computer use"] : []), ]; const restoreDefaults = async () => { @@ -605,7 +606,7 @@ export function useSettingsRestore(onRestored?: () => void) { notifyThemeRestoreFailure(); return; } - updateSettings({ + const result = await updateSettings({ appearanceContrast: DEFAULT_UNIFIED_SETTINGS.appearanceContrast, diffColorScheme: DEFAULT_UNIFIED_SETTINGS.diffColorScheme, chatWidth: DEFAULT_UNIFIED_SETTINGS.chatWidth, @@ -673,8 +674,9 @@ export function useSettingsRestore(onRestored?: () => void) { browserAutoShowFloatingPreview: DEFAULT_UNIFIED_SETTINGS.browserAutoShowFloatingPreview, pullRequestLinksOpenInApp: DEFAULT_UNIFIED_SETTINGS.pullRequestLinksOpenInApp, enableAgentBrowserAccess: DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess, + enableCua: DEFAULT_UNIFIED_SETTINGS.enableCua, }); - onRestored?.(); + if (result.saved) onRestored?.(); }; return { diff --git a/apps/web/src/components/settings/SourceControlWritingSettings.browser.test.tsx b/apps/web/src/components/settings/SourceControlWritingSettings.browser.test.tsx index 5a44beb50ac8..c5441cb65a59 100644 --- a/apps/web/src/components/settings/SourceControlWritingSettings.browser.test.tsx +++ b/apps/web/src/components/settings/SourceControlWritingSettings.browser.test.tsx @@ -5,12 +5,11 @@ import { describe, expect, it, vi } from "vite-plus/test"; import { page, userEvent } from "vite-plus/test/browser"; import "../../index.css"; -import type { ScopedSettingsPatch } from "./scopedSettings"; type WritingStyle = typeof DEFAULT_UNIFIED_SETTINGS.sourceControlWritingStyle; const state = vi.hoisted(() => ({ styles: [] as WritingStyle[], - updateSettings: vi.fn<(patch: ScopedSettingsPatch) => void>(), + updateSettings: vi.fn>(), })); vi.mock(import("@tanstack/react-router"), async (original) => ({ @@ -87,6 +86,7 @@ async function setup() { state.updateSettings.mockReset().mockImplementation((patch) => { state.styles = state.styles.map((style) => ({ ...style, ...patch.sourceControlWritingStyle })); void screen.rerender(content()); + return Promise.resolve({ saved: true, message: null }); }); return screen; } diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index aabfd0300eda..2d3ed853be53 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -845,6 +845,13 @@ export const SETTINGS_SEARCH_ITEMS = [ targetId: "devices", searchTerms: ["allow simulator emulator ios android drive tools sessions"], }, + { + id: "cua-computer-use", + title: "Computer use", + to: "/settings/devices", + targetId: "computer", + searchTerms: ["cua driver apps desktop screen click type agents control mac linux windows"], + }, { id: "device-hub", title: "Use devices", diff --git a/apps/web/src/components/settings/useScopedSettings.ts b/apps/web/src/components/settings/useScopedSettings.ts index 5fb9d048a9e3..2853eea1f735 100644 --- a/apps/web/src/components/settings/useScopedSettings.ts +++ b/apps/web/src/components/settings/useScopedSettings.ts @@ -49,7 +49,10 @@ export function useScopedSettingSource(keys: readonly (keyof ServerSettings)[]) function useRunScopedPlan() { const persistServer = useAtomCommand(serverEnvironment.updateSettings, { reportFailure: false }); - return (plan: ReturnType, onFailed?: () => void) => { + return async ( + plan: ReturnType, + onFailed?: () => void, + ): Promise<{ readonly saved: boolean; readonly message: string | null }> => { if (plan.unavailableReason) { toastManager.add({ type: "warning", @@ -57,43 +60,50 @@ function useRunScopedPlan() { description: plan.unavailableReason, }); onFailed?.(); - return; + return { saved: false, message: plan.unavailableReason }; } - void persistScopedSettingsPatch( - plan, - async (request) => { - const missing = requiredScopesForServerSettingsPatch(request.input.patch).find( - (scope) => !readEnvironmentScope(request.environmentId, scope), + try { + const { failedEnvironments, savedEnvironments, savedEnvironmentCount } = + await persistScopedSettingsPatch( + plan, + async (request) => { + const missing = requiredScopesForServerSettingsPatch(request.input.patch).find( + (scope) => !readEnvironmentScope(request.environmentId, scope), + ); + if (missing) + return AsyncResult.failure( + Cause.fail( + new EnvironmentAuthorizationError({ + requiredScope: missing, + message: "This connection lacks permission to change these settings.", + }), + ), + ); + return persistServer(request); + }, + persistClientSettingsPatch, ); - if (missing) - return AsyncResult.failure( - Cause.fail( - new EnvironmentAuthorizationError({ - requiredScope: missing, - message: "This connection lacks permission to change these settings.", - }), - ), - ); - return persistServer(request); - }, - persistClientSettingsPatch, - ).then(({ failedEnvironments, savedEnvironments, savedEnvironmentCount }) => { - if (failedEnvironments.length === 0) return; + if (failedEnvironments.length === 0) return { saved: true, message: null }; onFailed?.(); + const message = [ + ...failedEnvironments.map(({ label, message }) => `Could not save on ${label}: ${message}`), + ...(savedEnvironmentCount > 0 + ? [`Saved on ${savedEnvironments.map(({ label }) => label).join(", ")}.`] + : []), + ].join("\n"); toastManager.add({ type: "error", title: savedEnvironmentCount > 0 ? "Setting saved on some environments" : "Setting not saved", - description: [ - ...failedEnvironments.map( - ({ label, message }) => `Could not save on ${label}: ${message}`, - ), - ...(savedEnvironmentCount > 0 - ? [`Saved on ${savedEnvironments.map(({ label }) => label).join(", ")}.`] - : []), - ].join("\n"), + description: message, }); - }); + return { saved: false, message }; + } catch (error) { + onFailed?.(); + const message = error instanceof Error ? error.message : "The save failed. Try again."; + toastManager.add({ type: "error", title: "Setting not saved", description: message }); + return { saved: false, message }; + } }; } @@ -110,7 +120,7 @@ export function useClearScopedSettings() { const run = useRunScopedPlan(); return (keys: readonly ProjectScopedServerSettingKey[]) => { if (context === null) return; - run(planScopedSettingsClear(context.scope, context.environments, keys)); + void run(planScopedSettingsClear(context.scope, context.environments, keys)); }; } @@ -122,7 +132,7 @@ export function useClearProjectOverrides() { keys: readonly ProjectScopedServerSettingKey[], ) => { if (context === null) return; - run(planProjectOverridesClear(context.environments, entries, keys)); + void run(planProjectOverridesClear(context.environments, entries, keys)); }; } diff --git a/docs/README.md b/docs/README.md index af3c45ce567b..6a201348658a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,7 @@ - [Appearance and themes](./user/appearance.md) - [Keyboard shortcuts](./user/keybindings.md) - [SnapShots](./user/snap-shot.md) +- [Computer use](./user/computer-use.md) - [Visual replies](./user/html-renders.md) - [Import browser sessions](./user/browser-import.md) - [Use T3 Code as your default browser](./user/default-browser.md) diff --git a/docs/user/computer-use.md b/docs/user/computer-use.md new file mode 100644 index 000000000000..991e3b27a2c8 --- /dev/null +++ b/docs/user/computer-use.md @@ -0,0 +1,41 @@ +# Computer use + +Computer use lets agents see and use apps on the computer that runs your T3 +Code environment. It runs through +[Cua Driver](https://cua.ai/docs/cua-driver), which acts on windows in the +background, so agents do not take over your cursor or keyboard while you work. + +It works on macOS, Windows, and Linux. It is off by default. + +## Turn it on + +Open **Settings → Devices → Computer**, choose one environment, and turn +on **Computer use**. On mobile, use **Settings → Threads → Computer**. +Agent sessions started afterwards get the computer use tools. Turning it off +stops the driver immediately; restart existing agent sessions to remove their tools. + +What the host needs depends on its OS: + +- **macOS:** T3 Code needs Accessibility and Screen Recording. On the Mac that + runs the environment, the setup walks you through both. From another computer + or the mobile app, the switch turns on, but someone at that Mac still has to + grant them there. For a server run from a terminal or service, grant access to + the process hosting the server in **System Settings → Privacy & Security**. + Choose **Setup** next to the switch in the desktop app to check its grants again. +- **Windows:** someone must be signed in to the desktop and respond to any + security prompts for Cua Driver at that computer. +- **Linux:** the host needs a signed-in graphical session. On Wayland, someone + at the computer may need to approve a screen capture prompt. + +The desktop app includes Cua Driver. A server started with `npx t3` downloads +the same version into T3 Code's home the first time an agent session needs it. +To use your own build instead, set `T3CODE_CUA_DRIVER_PATH` to its executable +before starting the server. + +If you already configured a `cua-driver` MCP server for Codex yourself, Codex +keeps yours and T3 Code does not add a second one. + +## What you see + +Each action an agent takes shows in the thread with the app's icon and a plain +title, such as **Clicked in Safari**. diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index dadaf2dafb7a..a8d8fa984456 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -1098,6 +1098,11 @@ export function createServerEnvironmentAtoms( ]); }).pipe(Atom.withLabel(`environment-data:server:usage-scan-settings:${environmentId}`)), ); + const computerUseSettingAtom = Atom.family((environmentId: EnvironmentId) => + Atom.make((get) => get(settingsValueAtom(environmentId))?.enableCua ?? false).pipe( + Atom.withLabel(`environment-data:server:computer-use-setting:${environmentId}`), + ), + ); const providersValueAtom = Atom.family((environmentId: EnvironmentId) => Atom.make((get) => get(configValueAtom(environmentId))?.providers ?? null).pipe( Atom.withLabel(`environment-data:server:providers:${environmentId}`), @@ -1247,6 +1252,11 @@ export function createServerEnvironmentAtoms( label: "environment-data:provider:install-remove", tag: WS_METHODS.providerInstallRemove, }), + computerUse: createEnvironmentRpcQueryAtomFamily(runtime, { + label: "environment-data:server:computer-use", + tag: WS_METHODS.serverCheckComputerUse, + refreshTrigger: ({ environmentId }) => computerUseSettingAtom(environmentId), + }), traceDiagnostics: createEnvironmentRpcQueryAtomFamily(runtime, { label: "environment-data:server:trace-diagnostics", tag: WS_METHODS.serverGetTraceDiagnostics, diff --git a/packages/contracts/src/cua.ts b/packages/contracts/src/cua.ts index c3734c193032..c0b97a7bd75b 100644 --- a/packages/contracts/src/cua.ts +++ b/packages/contracts/src/cua.ts @@ -37,3 +37,12 @@ export const DesktopCuaDriverReport = Schema.Union([ }), ]); export type DesktopCuaDriverReport = typeof DesktopCuaDriverReport.Type; + +/** Driver readiness is independent of the enableCua setting. */ +export const CuaHostStatus = Schema.Struct({ + platform: Schema.Literals(["darwin", "linux", "win32", "other"]), + running: Schema.Boolean, + /** Why the last start failed, until a later start succeeds. */ + failure: Schema.optionalKey(Schema.String), +}); +export type CuaHostStatus = typeof CuaHostStatus.Type; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index edc7a9ab852a..a60ccd583612 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -307,6 +307,7 @@ import { DeviceSession, DeviceShutdownInput, } from "./device.ts"; +import { CuaHostStatus } from "./cua.ts"; import { ServerConfigStreamEvent, DesktopUpdateCommitInput, @@ -492,6 +493,7 @@ export const WS_METHODS = { serverProbe: "server.probe", serverGetConfig: "server.getConfig", + serverCheckComputerUse: "server.checkComputerUse", serverRefreshProviders: "server.refreshProviders", serverUpdateProvider: "server.updateProvider", serverUpdateServer: "server.updateServer", @@ -624,6 +626,12 @@ const WsServerProbeRpc = Rpc.make(WS_METHODS.serverProbe, { error: EnvironmentAuthorizationError, }); +const WsServerCheckComputerUseRpc = Rpc.make(WS_METHODS.serverCheckComputerUse, { + payload: Schema.Struct({}), + success: CuaHostStatus, + error: EnvironmentAuthorizationError, +}); + const WsServerGetConfigRpc = Rpc.make(WS_METHODS.serverGetConfig, { payload: Schema.Struct({}), success: ServerConfig, @@ -1954,6 +1962,7 @@ export const WsRpcGroup = RpcGroup.make( WsGuestShareCreateRpc, WsServerProbeRpc, WsServerGetConfigRpc, + WsServerCheckComputerUseRpc, WsServerRefreshProvidersRpc, WsServerUpdateProviderRpc, WsProviderConsumeResetCreditRpc,