diff --git a/packages/core/native/macos/GodmodeComputer.swift b/packages/core/native/macos/GodmodeComputer.swift index c5f907a2..4c354aba 100644 --- a/packages/core/native/macos/GodmodeComputer.swift +++ b/packages/core/native/macos/GodmodeComputer.swift @@ -19,7 +19,7 @@ import ImageIO import ScreenCaptureKit import UniformTypeIdentifiers -let HELPER_VERSION = "2" +let HELPER_VERSION = "3" // MARK: - Output @@ -1696,12 +1696,81 @@ func openApp(_ p: Params) throws -> [String: Any] { return ["pid": pid ?? NSNull(), "path": appURL.path] } +// MARK: - Focus guard (agent browsers) + +/// Chromium activates itself when a page opens a tab or popup, or when a tool opens a tab in the foreground — taking +/// the keyboard from whatever the human is doing. While the core has armed an agent browser (it just opened a tab), +/// an activation the human didn't make by clicking one of its windows is handed straight back. Main thread only. +final class FocusGuard { + static let shared = FocusGuard() + private var armedUntil: [pid_t: Date] = [:] + private var current: NSRunningApplication? + private var lastTaken: (pid: pid_t, at: Date, from: NSRunningApplication?, byHuman: Bool)? + private var observing = false + + func arm(pid: pid_t, ms: Int) { + observe() + let now = Date() + armedUntil[pid] = max(armedUntil[pid] ?? now, now.addingTimeInterval(Double(ms) / 1000)) + // The tab's activation can come before the core heard of the tab. + if ms > 0, let t = lastTaken, t.pid == pid, !t.byHuman, now.timeIntervalSince(t.at) < 1.5, + NSWorkspace.shared.frontmostApplication?.processIdentifier == pid { + giveBack(to: t.from) + } + } + + private func observe() { + if observing { return } + observing = true + current = NSWorkspace.shared.frontmostApplication + NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.didActivateApplicationNotification, object: nil, queue: .main) { [weak self] note in + guard let app = note.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication else { return } + self?.activated(app) + } + } + + private func activated(_ app: NSRunningApplication) { + let before = current + current = app + let pid = app.processIdentifier + guard before?.processIdentifier != pid, let until = armedUntil[pid] else { return } + let byHuman = clickedInto(pid) + lastTaken = (pid, Date(), before, byHuman) + if !byHuman, until > Date() { giveBack(to: before) } + } + + private func giveBack(to app: NSRunningApplication?) { + lastTaken = nil + guard let app, !app.isTerminated, app.processIdentifier != getpid() else { return } + app.activate(options: []) + } + + /// Did the human just click one of the app's windows? + private func clickedInto(_ pid: pid_t) -> Bool { + let since = min( + CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: .leftMouseDown), + CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: .rightMouseDown)) + guard since < 1 else { return false } + let point = CGEvent(source: nil)?.location ?? .zero + let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? [] + for w in list where (w[kCGWindowLayer as String] as? Int) == 0 { + guard let owner = w[kCGWindowOwnerPID as String] as? Int, owner != Int(getpid()), + (w[kCGWindowAlpha as String] as? Double ?? 1) > 0, + let bounds = w[kCGWindowBounds as String] as? NSDictionary, + let rect = CGRect(dictionaryRepresentation: bounds), rect.contains(point) + else { continue } + return owner == Int(pid) + } + return false + } +} + // MARK: - Dispatch /// Input runs one command at a time (event order matters); lookups run concurrently so a long `type` doesn't block them. let inputQueue = DispatchQueue(label: "godmode.computer.input") let readQueue = DispatchQueue(label: "godmode.computer.read", attributes: .concurrent) -let readCommands: Set = ["hello", "permissions", "displays", "windows", "window", "apps", "cursor"] +let readCommands: Set = ["hello", "permissions", "displays", "windows", "window", "apps", "cursor", "guardFocus"] func handle(_ line: String) { guard let data = line.data(using: .utf8), @@ -1758,6 +1827,11 @@ func handle(_ line: String) { case "activate": return try activate(p) case "ensureKeyWindow": return try ensureKeyWindow(p) case "openApp": return try openApp(p) + case "guardFocus": + let pid = pid_t(try p.requireInt("pid")) + let ms = max(0, min(p.int("ms") ?? 0, 10_000)) + DispatchQueue.main.async { FocusGuard.shared.arm(pid: pid, ms: ms) } + return ["ok": true] case "agentCursor": if p.bool("hide") == true { let window = p.int("window") diff --git a/packages/core/src/browser/focusGuard.ts b/packages/core/src/browser/focusGuard.ts new file mode 100644 index 00000000..95a9d38a --- /dev/null +++ b/packages/core/src/browser/focusGuard.ts @@ -0,0 +1,47 @@ +/** + * macOS: Chromium activates itself when a page opens a tab or popup, or when a script connected straight to its + * DevTools port opens a tab in the foreground — taking the keyboard from whatever the human is doing. Whenever an + * agent browser gets a new tab, the native helper is armed to hand such an activation straight back (`FocusGuard` in + * native/macos/GodmodeComputer.swift). A window the human clicks into stays theirs. + */ +import { getHelper, NativeHelper } from "../computer/helper"; +import { logger } from "../log"; +import { onBrowserState, type RunningBrowser } from "./state"; + +const log = logger("browser-focus"); + +/** How long after a new tab the browser may not take the focus (pages open popups after loading a little). */ +const ARM_MS = 3000; + +const watching = new Map void>(); +let initialized = false; +let warned = false; + +export function initFocusGuard() { + if (initialized || process.platform !== "darwin") return; + initialized = true; + onBrowserState((profileId, rb) => { + watching.get(profileId)?.(); + watching.delete(profileId); + if (rb && !rb.headless && rb.pid) watching.set(profileId, watch(rb, rb.pid)); + }); +} + +function watch(rb: RunningBrowser, pid: number): () => void { + // Started now, the helper knows which app the human is in before the browser takes it. + void arm(pid, 0); + return rb.client.on("Target.targetCreated", (p) => { + if ((p.targetInfo as { type?: string } | undefined)?.type === "page") void arm(pid, ARM_MS); + }); +} + +async function arm(pid: number, ms: number) { + try { + const helper = await getHelper(); + if (helper instanceof NativeHelper) await helper.call("guardFocus", { pid, ms }, 5000); + } catch (err) { + if (warned) return; + warned = true; + log.warn(`the browser can't be kept from taking the focus: ${err instanceof Error ? err.message : String(err)}`); + } +} diff --git a/packages/core/src/browser/manager.ts b/packages/core/src/browser/manager.ts index 8e4c93e8..a2943865 100644 --- a/packages/core/src/browser/manager.ts +++ b/packages/core/src/browser/manager.ts @@ -25,6 +25,7 @@ import { stealthArgs, stopProbes, windowedUserAgent } from "./stealth"; import { botCheckReport } from "./botCheck"; import { allRunning, getRegistered, getRunning, registerBrowser, touchBrowser, unregisterBrowser, type RunningBrowser } from "./state"; import { initLiveView, pauseLiveViews, resumeLiveViews } from "./screencast"; +import { initFocusGuard } from "./focusGuard"; import { TabRegistry } from "./tabs"; import { leasedChats, openChatLease, releaseChatLease, stopChatProxy } from "./proxy"; import * as importer from "./importer"; @@ -133,6 +134,7 @@ export function listProfiles(): BrowserProfile[] { /** Ensure a global default profile exists (called at startup). */ export function ensureDefaultProfile(): BrowserProfile { initLiveView(); + initFocusGuard(); if (!idleTimer) void adoptOrphans(); startIdleWatcher(); const existing = get("SELECT * FROM browser_profiles WHERE workspace_id IS NULL AND is_default = 1 ORDER BY created_at LIMIT 1"); diff --git a/packages/core/test/browser-focus-guard.test.ts b/packages/core/test/browser-focus-guard.test.ts new file mode 100644 index 00000000..726a7d56 --- /dev/null +++ b/packages/core/test/browser-focus-guard.test.ts @@ -0,0 +1,111 @@ +/** + * macOS: a visible agent browser gets the native helper's focus guard (a stand-in helper via GODMODE_COMPUTER_HELPER), + * armed whenever it opens a page. + */ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { CdpClient, CdpParams } from "../src/browser/cdp"; +import { initFocusGuard } from "../src/browser/focusGuard"; +import { registerBrowser, unregisterBrowser, type RunningBrowser } from "../src/browser/state"; +import { stopHelper } from "../src/computer/helper"; + +const suite = process.platform === "darwin" ? describe : describe.skip; + +const FAKE_HELPER = ` +const { appendFileSync } = require("node:fs"); +const logFile = process.argv[2]; +process.stdout.write(JSON.stringify({ ready: true, version: "3" }) + "\\n"); +let buf = ""; +process.stdin.on("data", (chunk) => { + buf += chunk; + let i; + while ((i = buf.indexOf("\\n")) >= 0) { + const line = buf.slice(0, i); + buf = buf.slice(i + 1); + appendFileSync(logFile, line + "\\n"); + process.stdout.write(JSON.stringify({ id: JSON.parse(line).id, ok: true, result: { ok: true } }) + "\\n"); + } +}); +`; + +function fakeBrowser(profileId: string, opts: { headless?: boolean; pid?: number } = {}) { + const handlers = new Map void>>(); + const client = { + closed: false, + on(method: string, fn: (p: CdpParams) => void) { + if (!handlers.has(method)) handlers.set(method, new Set()); + handlers.get(method)!.add(fn); + return () => handlers.get(method)!.delete(fn); + }, + }; + const rb = { profileId, headless: opts.headless ?? false, pid: opts.pid ?? 4242, client: client as unknown as CdpClient, stopping: false } as RunningBrowser; + const created = (type: string) => { + for (const fn of handlers.get("Target.targetCreated") ?? []) fn({ targetInfo: { targetId: `T${Math.random()}`, type, url: "about:blank", title: "" } }); + }; + return { rb, created }; +} + +suite("browser focus guard (macOS)", () => { + let dir: string; + let logFile: string; + const calls = (): Record[] => + existsSync(logFile) + ? readFileSync(logFile, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((l) => JSON.parse(l)) + : []; + const settle = async (count: number) => { + for (let i = 0; i < 100 && calls().length < count; i++) await Bun.sleep(20); + await Bun.sleep(50); + }; + + beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), "godmode-focus-guard-")); + logFile = join(dir, "calls.jsonl"); + const script = join(dir, "godmode-computer"); + writeFileSync(script, `#!${process.execPath}\nprocess.argv.push(${JSON.stringify(logFile)});\n${FAKE_HELPER}`); + chmodSync(script, 0o755); + await stopHelper(); + process.env.GODMODE_COMPUTER_HELPER = script; + initFocusGuard(); + }); + + afterAll(async () => { + await stopHelper(); + delete process.env.GODMODE_COMPUTER_HELPER; + if (dir) rmSync(dir, { recursive: true, force: true }); + }); + + beforeEach(() => rmSync(logFile, { force: true })); + + test("a visible browser is watched from its start and armed for every page it opens", async () => { + const { rb, created } = fakeBrowser("bpr_visible", { pid: 4242 }); + registerBrowser(rb); + await settle(1); + expect(calls()).toEqual([expect.objectContaining({ cmd: "guardFocus", pid: 4242, ms: 0 })]); + + created("page"); + created("iframe"); + created("service_worker"); + await settle(2); + expect(calls().slice(1)).toEqual([expect.objectContaining({ cmd: "guardFocus", pid: 4242, ms: 3000 })]); + + unregisterBrowser(rb); + created("page"); + await Bun.sleep(150); + expect(calls()).toHaveLength(2); + }); + + test("headless browsers have no window to guard", async () => { + const { rb, created } = fakeBrowser("bpr_headless", { headless: true }); + registerBrowser(rb); + created("page"); + await Bun.sleep(150); + expect(calls()).toEqual([]); + unregisterBrowser(rb); + }); +});