Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 76 additions & 2 deletions packages/core/native/macos/GodmodeComputer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import ImageIO
import ScreenCaptureKit
import UniformTypeIdentifiers

let HELPER_VERSION = "2"
let HELPER_VERSION = "3"

// MARK: - Output

Expand Down Expand Up @@ -1696,12 +1696,81 @@ func openApp(_ p: Params) throws -> [String: Any] {
return ["pid": pid ?? NSNull(), "path": appURL.path]
}

// MARK: - Focus guard (agent browsers)

/// Chromium activates itself when a page opens a tab or popup, or when a tool opens a tab in the foreground — taking
/// the keyboard from whatever the human is doing. While the core has armed an agent browser (it just opened a tab),
/// an activation the human didn't make by clicking one of its windows is handed straight back. Main thread only.
final class FocusGuard {
static let shared = FocusGuard()
private var armedUntil: [pid_t: Date] = [:]
private var current: NSRunningApplication?
private var lastTaken: (pid: pid_t, at: Date, from: NSRunningApplication?, byHuman: Bool)?
private var observing = false

func arm(pid: pid_t, ms: Int) {
observe()
let now = Date()
armedUntil[pid] = max(armedUntil[pid] ?? now, now.addingTimeInterval(Double(ms) / 1000))
// The tab's activation can come before the core heard of the tab.
if ms > 0, let t = lastTaken, t.pid == pid, !t.byHuman, now.timeIntervalSince(t.at) < 1.5,
NSWorkspace.shared.frontmostApplication?.processIdentifier == pid {
giveBack(to: t.from)
}
}

private func observe() {
if observing { return }
observing = true
current = NSWorkspace.shared.frontmostApplication
NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.didActivateApplicationNotification, object: nil, queue: .main) { [weak self] note in
guard let app = note.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication else { return }
self?.activated(app)
}
}

private func activated(_ app: NSRunningApplication) {
let before = current
current = app
let pid = app.processIdentifier
guard before?.processIdentifier != pid, let until = armedUntil[pid] else { return }
let byHuman = clickedInto(pid)
lastTaken = (pid, Date(), before, byHuman)
if !byHuman, until > Date() { giveBack(to: before) }
}

private func giveBack(to app: NSRunningApplication?) {
lastTaken = nil
guard let app, !app.isTerminated, app.processIdentifier != getpid() else { return }
app.activate(options: [])
}

/// Did the human just click one of the app's windows?
private func clickedInto(_ pid: pid_t) -> Bool {
let since = min(
CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: .leftMouseDown),
CGEventSource.secondsSinceLastEventType(.combinedSessionState, eventType: .rightMouseDown))
guard since < 1 else { return false }
let point = CGEvent(source: nil)?.location ?? .zero
let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
for w in list where (w[kCGWindowLayer as String] as? Int) == 0 {
guard let owner = w[kCGWindowOwnerPID as String] as? Int, owner != Int(getpid()),
(w[kCGWindowAlpha as String] as? Double ?? 1) > 0,
let bounds = w[kCGWindowBounds as String] as? NSDictionary,
let rect = CGRect(dictionaryRepresentation: bounds), rect.contains(point)
else { continue }
return owner == Int(pid)
}
return false
}
}

// MARK: - Dispatch

/// Input runs one command at a time (event order matters); lookups run concurrently so a long `type` doesn't block them.
let inputQueue = DispatchQueue(label: "godmode.computer.input")
let readQueue = DispatchQueue(label: "godmode.computer.read", attributes: .concurrent)
let readCommands: Set<String> = ["hello", "permissions", "displays", "windows", "window", "apps", "cursor"]
let readCommands: Set<String> = ["hello", "permissions", "displays", "windows", "window", "apps", "cursor", "guardFocus"]

func handle(_ line: String) {
guard let data = line.data(using: .utf8),
Expand Down Expand Up @@ -1758,6 +1827,11 @@ func handle(_ line: String) {
case "activate": return try activate(p)
case "ensureKeyWindow": return try ensureKeyWindow(p)
case "openApp": return try openApp(p)
case "guardFocus":
let pid = pid_t(try p.requireInt("pid"))
let ms = max(0, min(p.int("ms") ?? 0, 10_000))
DispatchQueue.main.async { FocusGuard.shared.arm(pid: pid, ms: ms) }
return ["ok": true]
case "agentCursor":
if p.bool("hide") == true {
let window = p.int("window")
Expand Down
47 changes: 47 additions & 0 deletions packages/core/src/browser/focusGuard.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/**
* macOS: Chromium activates itself when a page opens a tab or popup, or when a script connected straight to its
* DevTools port opens a tab in the foreground — taking the keyboard from whatever the human is doing. Whenever an
* agent browser gets a new tab, the native helper is armed to hand such an activation straight back (`FocusGuard` in
* native/macos/GodmodeComputer.swift). A window the human clicks into stays theirs.
*/
import { getHelper, NativeHelper } from "../computer/helper";
import { logger } from "../log";
import { onBrowserState, type RunningBrowser } from "./state";

const log = logger("browser-focus");

/** How long after a new tab the browser may not take the focus (pages open popups after loading a little). */
const ARM_MS = 3000;

const watching = new Map<string, () => void>();
let initialized = false;
let warned = false;

export function initFocusGuard() {
if (initialized || process.platform !== "darwin") return;
initialized = true;
onBrowserState((profileId, rb) => {
watching.get(profileId)?.();
watching.delete(profileId);
if (rb && !rb.headless && rb.pid) watching.set(profileId, watch(rb, rb.pid));
});
}

function watch(rb: RunningBrowser, pid: number): () => void {
// Started now, the helper knows which app the human is in before the browser takes it.
void arm(pid, 0);
return rb.client.on("Target.targetCreated", (p) => {
if ((p.targetInfo as { type?: string } | undefined)?.type === "page") void arm(pid, ARM_MS);
});
}

async function arm(pid: number, ms: number) {
try {
const helper = await getHelper();
if (helper instanceof NativeHelper) await helper.call("guardFocus", { pid, ms }, 5000);
} catch (err) {
if (warned) return;
warned = true;
log.warn(`the browser can't be kept from taking the focus: ${err instanceof Error ? err.message : String(err)}`);
}
}
2 changes: 2 additions & 0 deletions packages/core/src/browser/manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import { stealthArgs, stopProbes, windowedUserAgent } from "./stealth";
import { botCheckReport } from "./botCheck";
import { allRunning, getRegistered, getRunning, registerBrowser, touchBrowser, unregisterBrowser, type RunningBrowser } from "./state";
import { initLiveView, pauseLiveViews, resumeLiveViews } from "./screencast";
import { initFocusGuard } from "./focusGuard";
import { TabRegistry } from "./tabs";
import { leasedChats, openChatLease, releaseChatLease, stopChatProxy } from "./proxy";
import * as importer from "./importer";
Expand Down Expand Up @@ -133,6 +134,7 @@ export function listProfiles(): BrowserProfile[] {
/** Ensure a global default profile exists (called at startup). */
export function ensureDefaultProfile(): BrowserProfile {
initLiveView();
initFocusGuard();
if (!idleTimer) void adoptOrphans();
startIdleWatcher();
const existing = get<ProfileRow>("SELECT * FROM browser_profiles WHERE workspace_id IS NULL AND is_default = 1 ORDER BY created_at LIMIT 1");
Expand Down
111 changes: 111 additions & 0 deletions packages/core/test/browser-focus-guard.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/**
* macOS: a visible agent browser gets the native helper's focus guard (a stand-in helper via GODMODE_COMPUTER_HELPER),
* armed whenever it opens a page.
*/
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { CdpClient, CdpParams } from "../src/browser/cdp";
import { initFocusGuard } from "../src/browser/focusGuard";
import { registerBrowser, unregisterBrowser, type RunningBrowser } from "../src/browser/state";
import { stopHelper } from "../src/computer/helper";

const suite = process.platform === "darwin" ? describe : describe.skip;

const FAKE_HELPER = `
const { appendFileSync } = require("node:fs");
const logFile = process.argv[2];
process.stdout.write(JSON.stringify({ ready: true, version: "3" }) + "\\n");
let buf = "";
process.stdin.on("data", (chunk) => {
buf += chunk;
let i;
while ((i = buf.indexOf("\\n")) >= 0) {
const line = buf.slice(0, i);
buf = buf.slice(i + 1);
appendFileSync(logFile, line + "\\n");
process.stdout.write(JSON.stringify({ id: JSON.parse(line).id, ok: true, result: { ok: true } }) + "\\n");
}
});
`;

function fakeBrowser(profileId: string, opts: { headless?: boolean; pid?: number } = {}) {
const handlers = new Map<string, Set<(p: CdpParams) => void>>();
const client = {
closed: false,
on(method: string, fn: (p: CdpParams) => void) {
if (!handlers.has(method)) handlers.set(method, new Set());
handlers.get(method)!.add(fn);
return () => handlers.get(method)!.delete(fn);
},
};
const rb = { profileId, headless: opts.headless ?? false, pid: opts.pid ?? 4242, client: client as unknown as CdpClient, stopping: false } as RunningBrowser;
const created = (type: string) => {
for (const fn of handlers.get("Target.targetCreated") ?? []) fn({ targetInfo: { targetId: `T${Math.random()}`, type, url: "about:blank", title: "" } });
};
return { rb, created };
}

suite("browser focus guard (macOS)", () => {
let dir: string;
let logFile: string;
const calls = (): Record<string, unknown>[] =>
existsSync(logFile)
? readFileSync(logFile, "utf8")
.trim()
.split("\n")
.filter(Boolean)
.map((l) => JSON.parse(l))
: [];
const settle = async (count: number) => {
for (let i = 0; i < 100 && calls().length < count; i++) await Bun.sleep(20);
await Bun.sleep(50);
};

beforeAll(async () => {
dir = mkdtempSync(join(tmpdir(), "godmode-focus-guard-"));
logFile = join(dir, "calls.jsonl");
const script = join(dir, "godmode-computer");
writeFileSync(script, `#!${process.execPath}\nprocess.argv.push(${JSON.stringify(logFile)});\n${FAKE_HELPER}`);
chmodSync(script, 0o755);
await stopHelper();
process.env.GODMODE_COMPUTER_HELPER = script;
initFocusGuard();
});

afterAll(async () => {
await stopHelper();
delete process.env.GODMODE_COMPUTER_HELPER;
if (dir) rmSync(dir, { recursive: true, force: true });
});

beforeEach(() => rmSync(logFile, { force: true }));

test("a visible browser is watched from its start and armed for every page it opens", async () => {
const { rb, created } = fakeBrowser("bpr_visible", { pid: 4242 });
registerBrowser(rb);
await settle(1);
expect(calls()).toEqual([expect.objectContaining({ cmd: "guardFocus", pid: 4242, ms: 0 })]);

created("page");
created("iframe");
created("service_worker");
await settle(2);
expect(calls().slice(1)).toEqual([expect.objectContaining({ cmd: "guardFocus", pid: 4242, ms: 3000 })]);

unregisterBrowser(rb);
created("page");
await Bun.sleep(150);
expect(calls()).toHaveLength(2);
});

test("headless browsers have no window to guard", async () => {
const { rb, created } = fakeBrowser("bpr_headless", { headless: true });
registerBrowser(rb);
created("page");
await Bun.sleep(150);
expect(calls()).toEqual([]);
unregisterBrowser(rb);
});
});
Loading