earakely-scale · Nov 5, 2026 · Nov 4, 2026 · Nov 4, 2026 · Nov 4, 2026 · Nov 4, 2026
@@ -52,6 +52,8 @@
 GAP_S = 2.0
 #: How long a stream that stopped or failed waits before it is started again.
 RETRY_S = 1.0
+#: How long a recording's start waits for its first frame, so a recording a start answered for is recording.
+FIRST_FRAME_S = 5.0
 #: How often free disk is checked while frames arrive.
 DISK_CHECK_S = 5.0
 KEEP_ENDED = 64
@@ -127,8 +129,9 @@ def recording_params(recording: Recording, streams: bool) -> dict:
 
 
 class Recordings:
-    """The recordings of one driver: started by request id, open ones counted against ``max_concurrent``, ended
-    ones kept with their manifests so a repeated ``end`` answers the same."""
+    """The recordings of one driver: started by request id, each start answering once its recording is recording, open
+    ones counted against ``max_concurrent``, ended ones kept with their manifests so a repeated ``end`` answers the
+    same."""
 
     def __init__(self, runtime: ComputerRuntime) -> None:
         self.runtime = runtime
@@ -144,7 +147,9 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O
             raise InvalidArgument("request_id is a non-empty string")
         async with self._lock:
             if request_id in self.started:
-                return self.started[request_id]
+                answer = self.started[request_id]
+                await self._live(self.running.get(answer["recording_id"]))
+                return answer
             hints, grant = self._hints(features, scope, window, fps, max_edge, cursor, upload)
             stopped = [rid for rid, r in self.running.items() if r.done]
             for rid in stopped[:max(0, len(stopped) - KEEP_ENDED)]:  # stopped by itself and never ended
@@ -164,8 +169,16 @@ async def start(self, request_id: str, scope: str, window: Optional[str], fps: O
             self.started[request_id] = answer
             while len(self.started) > KEEP_STARTED:
                 self.started.popitem(last=False)
+            await self._live(recording)
             return answer
 
+    @staticmethod
+    async def _live(recording: Optional[_Recording]) -> None:
+        """Wait up to FIRST_FRAME_S for ``recording``, one still running, to have its first frame."""
+        if recording is not None:
+            with contextlib.suppress(TimeoutError):
+                await asyncio.wait_for(recording.live.wait(), FIRST_FRAME_S)
+
     async def end(self, recording_id: str, upload: Optional[dict], reason: str) -> dict:
         if reason not in REASONS:
             raise InvalidArgument(f"reason is one of {list(REASONS)}, got {reason!r}")
@@ -219,6 +232,7 @@ def __init__(self, recording_id: str, hints: RecordHints, features: Recording, r
         self.ended_by: Optional[str] = None
         self.error: Optional[str] = None
         self.done = False
+        self.live = asyncio.Event()
         self.task: Optional[asyncio.Task] = None
         self.timer: Optional[asyncio.TimerHandle] = None
         self.released: Optional[asyncio.Future] = None
@@ -228,7 +242,11 @@ def __init__(self, recording_id: str, hints: RecordHints, features: Recording, r
         self._last_disk_check = 0.0
 
     async def run(self) -> None:
+        """Frames or segments from the source, recorded until the recording ends; ``live`` once it's recording: a
+        stream from its start, the fallback's polling from its first frame, which a busy computer is slow to take."""
         failing_since: Optional[float] = None
+        if self.streams:
+            self.live.set()
         try:
             while True:
                 try:
@@ -255,6 +273,7 @@ async def run(self) -> None:
                 await self._close()
             finally:
                 self.done = True
+                self.live.set()
                 self.released = asyncio.ensure_future(self._release())
 
     def expire(self) -> None:
@@ -315,6 +334,7 @@ async def _add_frame(self, item: RecordedFrame) -> None:
             path = self.hints.directory / f"{len(self.segments):04d}.mp4"
             self.encoder = await self._in_thread(_Encoder, path, frame.width, frame.height, self.hints.fps, at)
         await self._in_thread(self.encoder.add, frame, at)
+        self.live.set()
         self._check_disk()
 
     async def _add_segment(self, item: RecordedSegment) -> None:

@@ -176,6 +176,27 @@ async def test_the_fallback_polls_capture_and_draws_the_pointer(home):
     assert _at(manifest["started_at"]) <= _at(manifest["ended_at"])
 
 
+@pytest.mark.asyncio
+async def test_a_start_answers_once_its_recording_has_its_first_frame_however_slow_the_capture(home):
+    driver = Recorded()
+    driver.slow["capture"] = 0.5
+    async with http_client(driver) as http:
+        began = time.monotonic()
+        started = (await http.post(START, json={"request_id": "r", "fps": 10})).json()
+        took = time.monotonic() - began
+        recording = driver._computer.recordings.running[started["recording_id"]]
+        assert took >= 0.5 and recording.encoder is not None and recording.encoder.frames >= 1
+        await http.post(END, json={"recording_id": started["recording_id"]})
+
+        recordings = driver._computer.recordings
+        with pytest.raises(TimeoutError):
+            await asyncio.wait_for(recordings.start("cancelled", "display", None, 10, None, True, None), 0.1)
+        retried = await recordings.start("cancelled", "display", None, 10, None, True, None)
+        running = [r for r in recordings.running.values() if not r.done]
+        assert [r.id for r in running] == [retried["recording_id"]] and running[0].encoder.frames >= 1
+        await http.post(END, json={"recording_id": retried["recording_id"]})
+
+
 @pytest.mark.asyncio
 async def test_a_driver_without_the_pointer_group_records_without_a_cursor(home):
     _, manifest = await _record(MinimalDriver(), 0.4)

@@ -89,18 +89,24 @@ a VM bundle already on the Mac that boots it, as composing leaves one.
 
 A macOS computer env is composed on a Mac, by a sandbox provider that boots macOS VM bundles there, `lume`: the base's
 VM bundle is cloned under a name of the compose's key, booted with a build folder shared read-write, and readied for
-agent-env (`vm_bundle/macos/prepare.sh`): `/opt/agentenv` for the layers; a Python 3.12 with Tk, `python3` first on the
-`PATH` of what services run, which their venvs and the probe's fixture run on; `agentenv-service`, which starts each
+agent-env (`vm_bundle/macos/prepare.sh`): `/opt/agentenv` for the layers; a pinned Python, python-build-standalone's
+CPython 3.12 with Tk, checked by its sha256 (`vm_bundle/macos/lib.sh`), `python3` first on the `PATH` of what services
+run, which their venvs and the probe's fixture run on; `agentenv-service`, which starts each
 service with the deploy's environment; `set-display`, which puts the display in the virtual display's size at login,
 since a guest boots in the mode its disk was last in, with the sRGB colour profile, so a screenshot holds the pixels as
 drawn; and Cua Spacesd and the Login Items banner off. Then each layer, the driver's and each app's, installs as its
 `[install.macos]` says, by a script composing writes (`env/envs/computer/utils/macos_vm_bundle.py`): its files under its
-prefix, its downloads checked by sha256, its venv (its requirements and this release's protocol in one resolve), its
+prefix, its downloads checked by sha256, its venv, installed offline from the wheels of its lock, its
 run scripts and install steps, as root or the session user, its `grants`, the TCC permissions of an app bundle it
 installs, bound to the bundle's code signature, and its services as LaunchAgents of the login session. A service runs
 through `agentenv-service`, or, when `direct`, as its own command, with the variables it names, as an app bundle whose
 TCC permissions macOS gives only to the executable launchd starts itself must. macOS has no apt: a layer's
-`[install.macos]` names no `packages` or `sources`. Then the VM shuts down, is kept on that Mac as the VM bundle of its
+`[install.macos]` names no `packages` or `sources`. The venvs' wheels are locked as a Linux layer's packages are: the
+first compose of given inputs (each venv's requirements, the protocol's source, the base, the pinned Python and the
+resolve script) resolves them once in a VM of the base, which builds every venv's wheels, its requirements and the
+protocol in one resolve, into the folder it shares and is deleted; they're stored in the object store with a lock
+naming them, whose digest joins the VM bundle's key. Every compose of those inputs, on any Mac, stages them and
+installs offline, so one name is one set of wheels. Then the VM shuts down, is kept on that Mac as the VM bundle of its
 name, and gate 3 probes it at full, a check its driver declares it fails on macOS (`known_failures`) recorded and not
 failing the build; a compose of the same inputs on that Mac reuses it. A VM bundle is kept on the Mac that composed it
 alone, so a deploy on a Mac that doesn't hold the one its env's record names composes it there first, from the same

@@ -1,5 +1,6 @@
-# lib.sh, sourced by a composed VM bundle's provision script in the VM, as its session user: root through sudo, with
-# the password the build folder holds, and TCC permissions granted to an app bundle.
+# lib.sh, sourced by a composed VM bundle's provision script, and by the script that resolves its venvs' wheels, in the
+# VM, as its session user: root through sudo, with the password the build folder holds, the pinned Python agent-env
+# runs on, and TCC permissions granted to an app bundle.
 
 build="/Volumes/My Shared Files/agentenv-build"
 export PATH="/opt/agentenv/bin:/opt/homebrew/bin:/usr/local/bin:$HOME/.local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
@@ -9,6 +10,25 @@ fail() { echo "agentenv-compose: $*" >&2; exit 1; }
 # as_root COMMAND...: COMMAND as root.
 as_root() { sudo -S -p '' "$@" < "$build/.password"; }
 
+# The Python agent-env runs on: python-build-standalone's CPython 3.12.16 for Apple silicon, with Tk, by its sha256.
+PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20261026/cpython-3.12.16%2B20261026-aarch64-apple-darwin-install_only.tar.gz"
+PYTHON_SHA256=fbf4afef53c2a6b72bb741bc06d5671a9dfaf87bd84e5aa73ac9b3d0d589f14e
+
+# pinned_python: that Python at /opt/agentenv/python, python3 first on agent-env's PATH.
+pinned_python() {
+    local tarball
+    tarball=$(mktemp)
+    curl -fsSL --retry 3 -o "$tarball" "$PYTHON_URL"
+    echo "$PYTHON_SHA256  $tarball" | shasum -a 256 -c - > /dev/null || fail "the Python downloaded isn't the one pinned"
+    as_root mkdir -p /opt/agentenv
+    as_root chown "$(id -un):staff" /opt/agentenv
+    rm -rf /opt/agentenv/python
+    mkdir -p /opt/agentenv/python /opt/agentenv/bin
+    tar -xzf "$tarball" -C /opt/agentenv/python --strip-components 1
+    rm -f "$tarball"
+    ln -sf /opt/agentenv/python/bin/python3 /opt/agentenv/bin/python3
+}
+
 # grant APP SERVICE...: each TCC SERVICE (accessibility, post_event, screen_capture) for the app bundle APP, by its
 # identifier, bound to the code signature it has now, so another app of that identifier gets none.
 grant() {

@@ -1,9 +1,10 @@
 #!/bin/bash
 # prepare.sh: a macOS base readied for agent-env, once a composed VM bundle, before its layers, in the VM as its session
-# user. /opt/agentenv holds the layers; a Python with a current Tk, which layers' venvs and the probe's fixture run on,
-# is python3 first on the PATH of what services run; agentenv-service starts each service with the deploy's
-# environment, and set-display follows the virtual display's size at login; and nothing of the base's own is in an
-# agent's way: Cua Spacesd and the Login Items banner are off.
+# user. /opt/agentenv holds the layers; the pinned Python, with a current Tk, which layers' venvs and the probe's fixture
+# run on, is python3 first on the PATH of what services run; agentenv-service starts each service with the deploy's
+# environment, and set-display follows the virtual display's size at login; host.docker.internal is the VM's own
+# loopback, where a deploy forwards this Mac's grant server; and nothing of the base's own is in an agent's way: Cua
+# Spacesd and the Login Items banner are off.
 set -euo pipefail
 . "/Volumes/My Shared Files/agentenv-build/prepare/lib.sh"
 
@@ -13,11 +14,11 @@ as_root mkdir -p /opt/agentenv/bin /opt/agentenv/drivers /opt/agentenv/apps /etc
 as_root chown -R "$(id -un):staff" /opt/agentenv
 mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs/agentenv"
 
-command -v uv > /dev/null || fail "the base has no uv, which installs the Python agent-env runs on"
-uv python install -q 3.12
-ln -sf "$(uv python find 3.12)" /opt/agentenv/bin/python3
+pinned_python
 /opt/agentenv/bin/python3 -c 'import tkinter; assert tkinter.TkVersion >= 8.6' || fail "its Python has no Tk"
 install -m 755 "$build/prepare/agentenv-service" /opt/agentenv/bin/agentenv-service
+grep -q "[[:space:]]host[.]docker[.]internal$" /etc/hosts ||
+    as_root sh -c 'echo "127.0.0.1 host.docker.internal" >> /etc/hosts'
 command -v swiftc > /dev/null || fail "the base has no swiftc, which builds set-display"
 swiftc -O "$build/prepare/set-display.swift" -o /opt/agentenv/bin/set-display
 

@@ -21,6 +21,7 @@
 import subprocess
 import sys
 import tempfile
+import threading
 import time
 import uuid
 from pathlib import Path
@@ -31,6 +32,10 @@
 from mcp.types import CallToolResult, TextContent
 from pydantic import AnyUrl
 
+if sys.platform != "darwin":
+    from Xlib import display as x_display
+    from Xlib.error import ConnectionClosedError, DisplayConnectionError
+
 from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext, resource, server_instructions
 from agentenv_protocol.computer import (
     PRINTABLE_KEYS,
@@ -71,6 +76,8 @@
 #: where the pointer is when it handles the press, and a button the pointer hasn't yet entered loses the click; and a
 #: drag's steps, about one a pixel, within cua-driver's 1 to 200.
 SETTLE_S, DRAG_STEPS = 0.2, (20, 200)
+#: The first code point past ASCII and its controls, where a character is off a US keymap.
+LATIN_1 = 0xA0
 #: The TCC permissions cua-driver reports on macOS, which it needs.
 PERMISSIONS = ("accessibility", "screen_recording")
 #: What keeps cua-driver from reaching trycua: no telemetry and no update checks.
@@ -105,6 +112,7 @@ def __init__(self) -> None:
         self._pinned: list[ReadyCheck] | None = None
         self._skill: tuple[str, str, dict[str, str]] | None = None
         self._texts: dict[str, str] | None = None
+        self._keymap: _Keymap | None = None
 
     def forwarded_tools(self) -> list[ForwardedTool]:
         return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], inputSchema=t["inputSchema"]),
@@ -141,7 +149,8 @@ async def screen(self) -> Display:
 
     @capture
     async def grab(self) -> Image:
-        result = await self._call("get_desktop_state", DeviceUnavailable)
+        """The whole display at its size, whose long edge cua-driver caps by default on macOS."""
+        result = await self._call("get_desktop_state", DeviceUnavailable, max_image_dimension=0)
         shot, state = next(c for c in result["content"] if c["type"] == "image"), result["structuredContent"]
         return Image(state["screenshot_width"], state["screenshot_height"], base64.b64decode(shot["data"]), "png")
 
@@ -170,6 +179,8 @@ async def scroll_at(self, x, y, direction, amount, modifiers) -> None:
 
     @type_text
     async def type_in(self, text) -> None:
+        if not MACOS and any(ord(ch) >= LATIN_1 for ch in text):
+            await asyncio.to_thread(self._hold, text)
         await self._send("type_text", text=text)
 
     @press
@@ -192,6 +203,20 @@ async def _rest(self, x: int, y: int) -> None:
         await self._send("move_cursor", x=x, y=y, **FOREGROUND)
         await asyncio.sleep(SETTLE_S)
 
+    def _hold(self, text: str) -> None:
+        """``text``'s characters on X's keymap, through a connection opened afresh once when X dropped the last, as it
+        does when it restarts."""
+        try:
+            self._x_keymap().hold(text)
+        except (ConnectionClosedError, DisplayConnectionError, OSError):
+            self._keymap = None
+            self._x_keymap().hold(text)
+
+    def _x_keymap(self) -> _Keymap:
+        if self._keymap is None:
+            self._keymap = _Keymap()
+        return self._keymap
+
     async def _send(self, name: str, **args: Any) -> None:
         await self._call(name, DeliveryFailed, scope="desktop", **args)
 
@@ -275,6 +300,43 @@ def _manifest() -> dict:
     return json.loads(MANIFEST.read_text())
 
 
+class _Keymap:
+    """X's keymap, given each character typed text needs that it lacks on a spare keycode, kept there. cua-driver maps
+    such a character onto a spare keycode for its press and back at once, and an app that reads the press after the
+    mapping is back, as a busy one does, loses the character; one on the keymap it types as any other key."""
+
+    def __init__(self) -> None:
+        self.d, self.lock, self.ours = x_display.Display(), threading.Lock(), {}
+
+    def hold(self, text: str) -> None:
+        """``text``'s characters past ASCII on the keymap: on spare keycodes, then on ours its text doesn't need, oldest
+        first. When those are too few, ours are given back, so cua-driver maps the characters itself."""
+        needed = {_keysym(ch) for ch in text if ord(ch) >= LATIN_1}
+        with self.lock:
+            first = self.d.display.info.min_keycode
+            mapping = self.d.get_keyboard_mapping(first, self.d.display.info.max_keycode - first + 1)
+            width = len(mapping[0])
+            if not (wanted := needed - {sym for syms in mapping for sym in syms}):
+                return
+            free = [first + i for i, syms in enumerate(mapping) if not any(syms)]
+            free += [code for sym, code in self.ours.items() if sym not in needed]
+            if len(wanted) > len(free):
+                for code in self.ours.values():
+                    self.d.change_keyboard_mapping(code, [(0,) * width])
+                self.ours.clear()
+            else:
+                for sym, code in zip(sorted(wanted), free):
+                    self.ours = {s: c for s, c in self.ours.items() if c != code}
+                    self.d.change_keyboard_mapping(code, [(sym,) * width])
+                    self.ours[sym] = code
+            self.d.sync()
+
+
+def _keysym(ch: str) -> int:
+    """X's keysym for ``ch``: its code point in Latin-1, else in X's Unicode range."""
+    return ord(ch) if ord(ch) < 0x100 else 0x1000000 | ord(ch)
+
+
 def _modifiers(keys) -> list[str]:
     if held := [key for key in keys if key not in MODIFIERS]:
         raise Unsupported(f"cua-driver holds only modifier keys, not {held}")

@@ -1,5 +1,6 @@
 # What a composed computer installs into the driver's venv. A release pins agentenv-framework-protocol to the version
 # it ships, which serves the driver with agentenv_protocol.computer.serve. cua-driver's wheel carries its executable for
-# each arch, whose sha256 the driver checks.
+# each arch, whose sha256 the driver checks; python-xlib keeps the characters typed text needs on X's keymap.
 agentenv-framework-protocol[computer]>=0.1.381
 cua-driver==0.38.0
+python-xlib>=0.33,<0.34