From 8b2a455ee9d142f3ceb1efa2a972373b69b0413b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 00:40:46 -0700 Subject: [PATCH 1/6] feat(computer): launch computer apps and wait until they're ready A computer app's launch and readiness, which M1.1 declares, now act. The composer writes each app's .agentenv/app.json and, with a launch, a launch script, a launcher and, for a boot app, an XDG autostart entry. The SDK runtime counts each app that starts with the computer in computer-ready: by a window matching launch.window (a new list_windows operation, with a Linux default over xprop and the reference driver's over python-xlib), a port, a command, or its @ready_check methods, run in the app's venv by _ready. A deploy, a reset and the probe wait up to the slowest boot app's own timeout. Declarations whose readiness can't be checked are refused. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- packages/agentenv-protocol/README.md | 30 +++-- .../agentenv_protocol/computer/__init__.py | 3 +- .../agentenv_protocol/computer/_declare.py | 6 + .../src/agentenv_protocol/computer/_ready.py | 31 +++++ .../src/agentenv_protocol/computer/app.py | 31 ++++- .../src/agentenv_protocol/computer/apps.py | 107 ++++++++++++++++++ .../src/agentenv_protocol/computer/driver.py | 4 + .../src/agentenv_protocol/computer/linux.py | 23 ++++ .../agentenv_protocol/computer/operations.py | 4 +- .../src/agentenv_protocol/computer/runtime.py | 5 + .../tests/test_computer_declare.py | 44 ++++++- .../tests/test_computer_driver.py | 66 +++++++++++ .../drivers/envs/reference/driver.py | 25 +++- src/agent_env/env/envs/computer/app_env.py | 10 ++ src/agent_env/env/envs/computer/utils/boot.py | 14 ++- .../env/envs/computer/utils/composer.py | 54 +++++++-- .../env_providers/env_computer_provider.py | 6 +- .../computer/app/artifacts/check_notes/run.py | 11 +- .../computer/app/envs/notes/app.py | 14 ++- .../computer/app/envs/notes/window.py | 9 ++ tst/integration/computer/computer_app_test.py | 9 +- tst/unit/drivers/reference_driver_test.py | 34 +++++- tst/unit/env/envs/test_computer_env.py | 38 ++++++- 24 files changed, 531 insertions(+), 49 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/_ready.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py create mode 100644 tst/integration/computer/app/envs/notes/window.py diff --git a/AGENTS.md b/AGENTS.md index 4daa66a93..915e986d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,7 +107,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`: files, a `python` requirements file for a venv, and the services the base's init runs; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile and build context so any env with the same inputs reuses it; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`: files, a `python` requirements file for a venv, and the services the base's init runs; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile and build context so any env with the same inputs reuses it; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs, a launcher for its `launch` (an XDG autostart entry for a `boot` one) and `.agentenv/app.json`, from which the driver counts an app that starts with the computer in `computer-ready`, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 27a9a4635..03068cca9 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -393,6 +393,7 @@ pixels), at the display's size times its scale. Input operations may return an ` | `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools | | `@run_in_session` | no; a Linux default | `computer-exec/v1` | | `@write_files` | no; a Linux default | `data/add` | +| `@list_windows` | no; a Linux default | a computer app's readiness by its window | | `@record` | no | the stream `computer-recording/v1` records from; without it the base class polls `@capture` | The class is checked when it is defined: a missing required operation, half a group, two methods @@ -400,12 +401,13 @@ for one operation that some platform could both match, or features that don't ma decorators raise `TypeError`. Predicates (`os`, `distribution`, `version`, `arch`, `display`) choose a method per platform, read from `/etc/os-release`, the machine and `XDG_SESSION_TYPE` or `DISPLAY`: `@capture(display="x11")` and `@capture(display="wayland")` can sit side by side. A -driver's own `@run_in_session` or `@write_files` replaces the base class's Linux default, which runs -as the driver's user inside the desktop session. The base class keeps its state in `self._computer` -and names its own members `_computer_*` and `computer_*`; a driver that redefines one of them, or -another private member of the base class, is refused when it is defined, so a helper of the -driver's can't silently replace one. What the optional operations give is served only when the -driver has them, through the `served_tools`, `served_extensions` and `served_operations` hooks. +driver's own `@run_in_session`, `@write_files` or `@list_windows` replaces the base class's Linux +default, which runs as the driver's user inside the desktop session. The base class keeps its state +in `self._computer` and names its own members `_computer_*` and `computer_*`; a driver that +redefines one of them, or another private member of the base class, is refused when it is defined, +so a helper of the driver's can't silently replace one. What the optional operations give is +served only when the driver has them, through the `served_tools`, `served_extensions` and +`served_operations` hooks. **What the base class serves:** @@ -428,8 +430,8 @@ driver has them, through the `served_tools`, `served_extensions` and `served_ope the live screen size, what its input takes, its OS's shortcuts) and ending with the driver class's `notes`, markdown for what its card can't say. - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a - capture, plus the driver's own `ready_checks()`; tools refuse `device_unavailable` until they - pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool + capture, the driver's own `ready_checks()`, and one per app that starts with the computer; tools + refuse `device_unavailable` until they pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool descriptions), `computer-keyboard/v1`, `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?, request_id}`: output capped per stream, a repeated `request_id` answered with the first result) and, with `recording`, `computer-recording/v1` (below). @@ -566,10 +568,14 @@ class MailApp(ComputerAppEnvironment): - **`linux = Linux(...)`** is how a Linux computer installs it: `files`, a folder of the app's copied under its prefix; `python`, a requirements file there installed into a venv beside them; `services`, programs the computer's init runs from the prefix as the session user. -- **`launch = Launch(...)`** says how it starts in the desktop session, with it (`at="boot"`) or when - the agent opens it (`"demand"`), and `ready=Ready(...)` when it's ready; `@ready_check` marks a - method for readiness that needs logic, and `@install_step(scope=...)` one a computer runs as it - installs the app. +- **`launch = Launch(...)`** says how it starts in the desktop session: with it (`at="boot"`, an XDG + autostart entry) or when the agent opens it (`"demand"`); either way the computer gets a launcher + for it. `ready=Ready(...)` says when it's ready: a window matching `launch.window` by its class, + instance or title, a port answering on loopback, or a command exiting 0. `@ready_check` marks a + method for readiness that needs logic, run in the app's own venv, so an app with one names its + `python` requirements. A `boot` app counts in `computer-ready`, as does an app with no launch whose + `@ready_check` methods watch its services; a `demand` app is checked once something launches it. + `@install_step(scope=...)` marks a method a computer runs as it installs the app. - **Its skill** is a file of its folder (`skill = "skill.md"`) or an `@agent_skill` method, not both; its data hooks are the usual `@add_data`, `@get_data` and `@reset_data`. - **`Settings`** is typed; a computer hands each app the values a deploy gives it, as one JSON value diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py index 45af04bf8..19b8ab07f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -48,6 +48,7 @@ drag, key_down, key_up, + list_windows, pointer_down, pointer_move, pointer_position, @@ -101,7 +102,7 @@ def __getattr__(name: str) -> Any: "APP_SETTINGS_ENV", "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV", "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press", - "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", + "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", "list_windows", "run_in_session", "write_files", "record", "Recording", "RecordHints", "RecordedFrame", "RecordedSegment", "frames_in_thread", "RecordingManifest", "ManifestSegment", "ComputerError", "InvalidArgument", "OutOfBounds", "StaleFrame", "InputBusy", "PermissionDenied", "Unsupported", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py index 8f8d723ac..c0c043d42 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py @@ -14,6 +14,8 @@ import types from pathlib import Path +from pydantic import ValidationError + from .app import AppDeclaration, ComputerAppEnvironment from .card import COMPUTER from .driver import ComputerDriverEnvironment, DriverDeclaration, DriverPlatform @@ -95,6 +97,10 @@ def main(argv: list[str]) -> int: try: with contextlib.redirect_stdout(sys.stderr): declared = declare(Path(folder), file, name) + except ValidationError as e: + print("ValueError: " + "; ".join(error["msg"].removeprefix("Value error, ") for error in e.errors()), + file=sys.stderr) + return 1 except Exception as e: print(f"{type(e).__name__}: {e}", file=sys.stderr) return 1 diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_ready.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_ready.py new file mode 100644 index 000000000..387c14f9b --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_ready.py @@ -0,0 +1,31 @@ +"""Run one of an app's ``@ready_check`` methods in the app's own venv, where its dependencies are installed: +``python -m agentenv_protocol.computer._ready `` exits 0 when the method returns true, +1 when it returns false, and 2, saying why on stderr, when it can't run. The class reads its settings as it does when +it serves.""" +from __future__ import annotations + +import asyncio +import importlib.util +import inspect +import sys +from pathlib import Path + + +def main(argv: list[str]) -> int: + folder, file, name, method = argv + try: + sys.path.insert(0, folder) + spec = importlib.util.spec_from_file_location("computer_app_module", Path(folder) / file) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + result = getattr(getattr(module, name)(), method)() + if inspect.isawaitable(result): + result = asyncio.run(result) + except Exception as e: + print(f"{type(e).__name__}: {e}", file=sys.stderr) + return 2 + return 0 if result else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index d72a78738..b60003392 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -10,7 +10,7 @@ import os from typing import Any, Callable, ClassVar, Literal, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from ..agent_env_environment import _EXT_ATTR, _OP_ATTR, AgentEnvEnvironment from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, SKILL_EXTENSION_URI @@ -20,6 +20,8 @@ _INSTALL_STEP_ATTR = "_aee_install_step" _READY_CHECK_ATTR = "_aee_ready_check" _DATA_METHODS = {"reset_data": METHOD_RESET, "add_data": METHOD_ADD, "get_data": METHOD_GET} +#: What a launch's window can be matched by, as a driver's list_windows describes each window. +WINDOW_KEYS = frozenset({"class", "instance", "title"}) class Service(BaseModel): @@ -53,6 +55,12 @@ class Ready(BaseModel): command: Optional[list[str]] = None timeout_s: float = 60 + @model_validator(mode="after") + def _waits_on_something(self) -> Ready: + if not self.window and self.port is None and not self.command: + raise ValueError("Ready waits on nothing: set window, port or command") + return self + class Launch(BaseModel): """How the app starts in the desktop session: with it (``boot``) or when the agent opens it (``demand``).""" @@ -61,9 +69,23 @@ class Launch(BaseModel): at: Literal["boot", "demand"] = "boot" command: list[str] + #: Matches the app's window by its WM_CLASS ``class`` or ``instance`` and its ``title``, as given. window: Optional[dict[str, str]] = None ready: Optional[Ready] = None + @field_validator("window") + @classmethod + def _matchable(cls, window: Optional[dict[str, str]]) -> Optional[dict[str, str]]: + if window is not None and (not window or set(window) - WINDOW_KEYS): + raise ValueError(f"window matches by {', '.join(sorted(WINDOW_KEYS))}, not {window!r}") + return window + + @model_validator(mode="after") + def _window_to_wait_for(self) -> Launch: + if self.ready is not None and self.ready.window and self.window is None: + raise ValueError("ready waits on a window: name it with Launch(window={'class': ...})") + return self + class AppSkill(BaseModel): """Where an app's skill for agents comes from: a ``file`` of its folder, or the ``@agent_skill`` ``method`` that @@ -101,6 +123,13 @@ class AppDeclaration(BaseModel): install_steps: list[InstallStep] ready_checks: list[str] + @model_validator(mode="after") + def _venv_for_ready_checks(self) -> AppDeclaration: + if self.ready_checks and not any(install.python for install in self.install.values()): + raise ValueError(f"@ready_check {', '.join(self.ready_checks)} runs in the app's own venv: name its " + "requirements with Linux(python=...)") + return self + def computer_app(*, supports: Optional[list] = None, permissions: Optional[list[str]] = None) -> Callable: """Declare what an app is: the platforms it supports, as a driver names them, and the permissions it needs of the diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py new file mode 100644 index 000000000..6dc4d4cdf --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py @@ -0,0 +1,107 @@ +"""The apps composed onto a computer, as its driver finds them, and whether each is ready. The composer writes each +app's ``.agentenv/app.json`` into its folder under ``APPS``. An app that starts with the computer counts toward the +computer's readiness: a ``boot`` app, or one with no launch whose ``@ready_check`` methods watch its services.""" +from __future__ import annotations + +import asyncio +import json +import shlex +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +from .app import Launch, Ready +from .operations import ReadyCheck + +if TYPE_CHECKING: + from .runtime import ComputerRuntime + +#: Where a composed computer installs its apps, a folder each. +APPS = Path("/opt/agentenv/apps") +#: An app's description of itself for the driver, in its folder. +APP_FILE = ".agentenv/app.json" +#: How long a readiness command or ``@ready_check`` may run. +CHECK_TIMEOUT_S = 10.0 +#: How long a port gets to answer. +CONNECT_TIMEOUT_S = 1.0 + + +@dataclass(frozen=True) +class InstalledApp: + name: str + folder: Path + module: str + launch: Optional[Launch] + ready_checks: tuple[str, ...] + + @property + def counted(self) -> bool: + """Whether the computer is ready only once this app is, because it starts with the computer.""" + return self.launch.at == "boot" if self.launch is not None else bool(self.ready_checks) + + +def installed() -> list[InstalledApp]: + """The apps composed onto this computer, by their folders' names.""" + found = [] + for path in sorted(APPS.glob(f"*/{APP_FILE}")): + data = json.loads(path.read_text()) + found.append(InstalledApp(name=data["name"], folder=path.parent.parent, module=data["module"], + launch=Launch.model_validate(data["launch"]) if data.get("launch") else None, + ready_checks=tuple(data.get("ready_checks", ())))) + return found + + +async def check(runtime: ComputerRuntime, app: InstalledApp, since: float) -> ReadyCheck: + """Whether ``app`` is ready, by its launch's ``ready`` and its ``@ready_check`` methods; once past its timeout, the + check says so with what it still waits on.""" + ready = app.launch.ready if app.launch is not None else None + problems = [] + if ready is not None and ready.window: + problems += await _window(runtime, app.launch.window) + if ready is not None and ready.port is not None: + problems += await _port(ready.port) + if ready is not None and ready.command: + problems += await _run(runtime, app, ready.command, shlex.join(ready.command)) + for method in app.ready_checks: + file, _, name = app.module.partition(":") + argv = [str(app.folder / "venv/bin/python"), "-m", "agentenv_protocol.computer._ready", str(app.folder), file, + name, method] + problems += await _run(runtime, app, argv, f"@ready_check {method}") + if not problems: + return ReadyCheck(f"app {app.name}", True, "ready") + waited = time.monotonic() - since + limit = ready.timeout_s if ready is not None else Ready.model_fields["timeout_s"].default + when = f"not ready after {limit:.0f} s" if waited > limit else f"not ready yet, {waited:.0f} s in" + return ReadyCheck(f"app {app.name}", False, f"{when}: {'; '.join(problems)}") + + +async def _window(runtime: ComputerRuntime, match: dict[str, str]) -> list[str]: + if not runtime.has("list_windows"): + return ["this driver can't list windows"] + windows = await runtime.call("list_windows") + if any(all(window.get(key) == value for key, value in match.items()) for window in windows): + return [] + return [f"no window with {', '.join(f'{key} {value!r}' for key, value in match.items())}"] + + +async def _port(port: int) -> list[str]: + try: + _, writer = await asyncio.wait_for(asyncio.open_connection("127.0.0.1", port), CONNECT_TIMEOUT_S) + except (OSError, TimeoutError): + return [f"nothing answers on port {port}"] + writer.close() + return [] + + +async def _run(runtime: ComputerRuntime, app: InstalledApp, argv: list[str], what: str) -> list[str]: + if not runtime.has("run_in_session"): + return [f"this driver can't run {what}"] + result = await runtime.call("run_in_session", argv, str(app.folder), {"ENVIRONMENT_NAME": app.name}, + CHECK_TIMEOUT_S) + if result.exit_code == 0: + return [] + if result.timed_out: + return [f"{what} ran past {CHECK_TIMEOUT_S:.0f} s"] + reason = (result.stderr.decode(errors="replace").strip().splitlines() or [""])[-1] + return [f"{what} exited {result.exit_code}" + (f": {reason}" if reason else "")] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index 172b57703..f58543545 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -150,6 +150,10 @@ async def _computer_linux_run_in_session(self, argv: list, cwd: str | None, env: async def _computer_linux_write_files(self, files: dict) -> None: await asyncio.to_thread(linux.write_files, files) + @default_for("list_windows", os="linux") + async def _computer_linux_list_windows(self) -> list[dict[str, str]]: + return await linux.list_windows() + def computer_driver(name: str | None = None, *, supports: list | None = None) -> Callable: """Name a driver, its card's name unless ENVIRONMENT_NAME gives one, and declare the platforms it supports.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py index 021fbc346..a45d55109 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py @@ -5,6 +5,7 @@ import asyncio import contextlib import os +import re import signal import tempfile from pathlib import Path @@ -81,6 +82,28 @@ async def read(self, stream: asyncio.StreamReader) -> None: self.kept += chunk[:max(room, 0)] +async def list_windows() -> list[dict[str, str]]: + """The window manager's top-level windows, each as its WM_CLASS ``instance`` and ``class`` and its ``title``, read + with ``xprop``; a window that closes while it's read is left out.""" + listed = await _xprop("-root", "_NET_CLIENT_LIST") + windows = [] + for window in re.findall(r"0x[0-9a-f]+", listed.partition("#")[2]): + props = await _xprop("-id", window, "WM_CLASS", "_NET_WM_NAME") + names = dict(re.findall(r'^(WM_CLASS|_NET_WM_NAME)[^=]*= (.*)$', props, re.MULTILINE)) + parts = re.findall(r'"((?:[^"\\]|\\.)*)"', names.get("WM_CLASS", "")) + if len(parts) == 2: + title = re.findall(r'"((?:[^"\\]|\\.)*)"', names.get("_NET_WM_NAME", "")) + windows.append({"instance": parts[0], "class": parts[1], "title": title[0] if title else ""}) + return windows + + +async def _xprop(*args: str) -> str: + process = await asyncio.create_subprocess_exec("xprop", *args, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL) + out, _ = await process.communicate() + return out.decode(errors="replace") if process.returncode == 0 else "" + + def write_files(files: dict[str, bytes]) -> None: """Write each file under the home atomically: a temporary file beside it, then a rename.""" home = Path.home().resolve() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py index 1e9936ec1..c742c4430 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py @@ -11,6 +11,7 @@ pointer_move(x, y), pointer_down(button), pointer_up(button), pointer_position() -> (x, y) key_down(keys), key_up(keys) run_in_session(argv, cwd, env, timeout_s) -> ExecResult write_files({path: bytes}) + list_windows() -> [{"class": ..., "instance": ..., "title": ...}, ...], the desktop's top-level windows record(hints) -> an async iterator of RecordedFrame or RecordedSegment, until it is closed """ from __future__ import annotations @@ -26,7 +27,7 @@ "pointer": ("pointer_move", "pointer_down", "pointer_up", "pointer_position"), "keyboard": ("key_down", "key_up"), } -OPTIONAL = ("set_display", "run_in_session", "write_files", "record") +OPTIONAL = ("set_display", "run_in_session", "write_files", "record", "list_windows") INPUT = frozenset({"click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", "key_down", "key_up"}) @@ -116,6 +117,7 @@ def deco(f: Callable) -> Callable: run_in_session = _marker("run_in_session") write_files = _marker("write_files") record = _marker("record") +list_windows = _marker("list_windows") def default_for(operation: str, **predicates: Any) -> Callable: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index d4e43bfbe..8ee333c47 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -5,6 +5,7 @@ import asyncio import contextlib +import time from collections import OrderedDict from collections.abc import AsyncIterator from typing import TYPE_CHECKING, Any, Awaitable, Callable @@ -13,6 +14,7 @@ from ..errors import AgentEnvError from ..tool_context import ToolContext +from . import apps from .errors import DeliveryFailed, DeviceUnavailable, InputBusy, Internal from .frames import Frame, FrameStore, to_pil from .operations import GROUPS, INPUT, Display, ReadyCheck, resolve @@ -40,6 +42,8 @@ def __init__(self, driver: ComputerDriverEnvironment) -> None: self._commands_running: dict[str, asyncio.Task] = {} self._commands_finished: OrderedDict = OrderedDict() self.recordings = Recordings(self) + self.apps = apps.installed() + self.started = time.monotonic() def has(self, capability: str) -> bool: if capability in GROUPS: @@ -175,6 +179,7 @@ async def ready_checks(self) -> list[ReadyCheck]: checks += list(await self.driver.ready_checks()) except Exception as e: checks.append(ReadyCheck("driver", False, f"{type(e).__name__}: {e}")) + checks += [await apps.check(self, app, self.started) for app in self.apps if app.counted] if all(c.ok for c in checks): self.ready = True return checks diff --git a/packages/agentenv-protocol/tests/test_computer_declare.py b/packages/agentenv-protocol/tests/test_computer_declare.py index a0fbccda3..f01bbb5bc 100644 --- a/packages/agentenv-protocol/tests/test_computer_declare.py +++ b/packages/agentenv-protocol/tests/test_computer_declare.py @@ -16,6 +16,7 @@ from agentenv_protocol.computer import AppDeclaration # noqa: E402 from agentenv_protocol.computer._declare import _Stub, _StubModule, declare, main # noqa: E402 +from agentenv_protocol.computer._ready import main as ready # noqa: E402 from agentenv_protocol.computer.platform import Platform # noqa: E402 DRIVER = '''import not_installed_here @@ -68,7 +69,8 @@ def press_keys(self, keys): pass @computer_app(supports={supports}, permissions=["camera"]) class Mail(ComputerAppEnvironment): linux = Linux(files="linux", python="requirements.txt", services=[Service(name="imap", command="bin/imap")]) - launch = Launch(at="demand", command=["thunderbird"], ready=Ready(window=True, timeout_s=30)) + launch = Launch(at="demand", command=["thunderbird"], window={{"class": "Thunderbird"}}, + ready=Ready(window=True, timeout_s=30)) class Settings(BaseModel): account: str = "user@example.com" @@ -174,7 +176,8 @@ def test_an_app_class_whose_imports_arent_installed_declares_what_it_is_to_a_com "supports": LINUX, "permissions": ["camera"], "install": {"linux": {"files": "linux", "python": "requirements.txt", "services": [{"name": "imap", "command": "bin/imap"}]}}, - "launch": {"at": "demand", "command": ["thunderbird"], "ready": {"window": True, "timeout_s": 30.0}}, + "launch": {"at": "demand", "command": ["thunderbird"], "window": {"class": "Thunderbird"}, + "ready": {"window": True, "timeout_s": 30.0}}, "skill": {"method": "guide"}, "data": ["data/add", "data/reset"], "install_steps": [{"name": "packages", "scope": "system"}, {"name": "profile", "scope": "session"}], "ready_checks": ["inbox_open"]} @@ -189,6 +192,43 @@ def test_an_app_declaring_its_skill_twice_is_refused(folder): declare(folder, "app.py", "Twice") +@pytest.mark.parametrize("change, problem", [ + ('launch = Launch(command=["mail"], ready=Ready(timeout_s=5))', "Ready waits on nothing: set window, port or command"), + ('launch = Launch(command=["mail"], ready=Ready(window=True))', "ready waits on a window: name it with Launch(window="), + ('launch = Launch(command=["mail"], window={"role": "main"})', "window matches by class, instance, title, not"), + ('linux = Linux(services=[Service(name="imap", command="bin/imap")])', + "@ready_check inbox_open runs in the app's own venv: name its requirements with Linux(python=...)"), +]) +def test_an_app_whose_launch_or_readiness_cant_be_checked_is_refused_saying_why(folder, capsys, change, problem): + (folder / "app.py").write_text(APP.format(supports=LINUX) + f""" + +@computer_app(supports={LINUX}) +class Changed(Mail): + {change} +""") + + assert main([str(folder), "app.py", "Changed"]) == 1 + assert problem in capsys.readouterr().err.splitlines()[-1] + + +@pytest.mark.parametrize("body, code", [("return True", 0), ("return False", 1), + ("raise RuntimeError('mailbox locked')", 2)]) +def test_a_ready_check_runs_from_the_apps_folder_and_exits_by_what_it_returns(tmp_path, monkeypatch, capsys, body, + code): + (tmp_path / "app.py").write_text(f"""from agentenv_protocol.computer import ComputerAppEnvironment, ready_check + + +class Mail(ComputerAppEnvironment): + @ready_check + async def inbox_open(self): + {body} +""") + monkeypatch.setenv("ENVIRONMENT_NAME", "mail") + + assert ready([str(tmp_path), "app.py", "Mail", "inbox_open"]) == code + assert capsys.readouterr().err == ("RuntimeError: mailbox locked\n" if code == 2 else "") + + def test_an_app_reads_the_settings_its_computer_gives_it_by_its_name(monkeypatch): from pydantic import BaseModel diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index c6ceec511..b326132f3 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -10,17 +10,20 @@ import asyncio # noqa: E402 import base64 # noqa: E402 import io # noqa: E402 +import json # noqa: E402 from computer_fake import LINUX, FakeDriver, FakeScreen, MinimalDriver, http_client, mcp_session # noqa: E402 from mcp.server.fastmcp import FastMCP # noqa: E402 from mcp.shared.memory import create_connected_server_and_client_session # noqa: E402 from PIL import Image as PILImage # noqa: E402 +from agentenv_protocol.computer import apps, linux # noqa: E402 from agentenv_protocol.computer import ( # noqa: E402 SETTINGS_ENV, ComputerDriverEnvironment, ComputerSettings, Display, + ExecResult, Image, InvalidArgument, OutOfBounds, @@ -31,6 +34,7 @@ computer_driver, display, drag, + list_windows, pointer_move, press, run_in_session, @@ -278,6 +282,68 @@ async def ready_checks(self): assert raised["checks"][-1] == {"name": "driver", "ok": False, "detail": "RuntimeError: probe crashed"} +@pytest.mark.asyncio +async def test_each_app_that_starts_with_the_computer_counts_until_it_is_ready(tmp_path, monkeypatch): + listener = await asyncio.start_server(lambda reader, writer: writer.close(), "127.0.0.1", 0) + port = listener.sockets[0].getsockname()[1] + for name, launch, checks in [ + ("mail", {"at": "boot", "command": ["mail"], "window": {"class": "Mail"}, "ready": {"window": True}}, []), + ("notes", {"at": "boot", "command": ["notes"], "ready": {"port": port}}, []), + ("music", {"at": "demand", "command": ["music"], "window": {"class": "Music"}, "ready": {"window": True}}, []), + ("sync", None, ["synced"]), + ]: + (tmp_path / name / ".agentenv").mkdir(parents=True) + (tmp_path / name / ".agentenv/app.json").write_text(json.dumps( + {"name": name, "module": "app.py:App", "launch": launch, "ready_checks": checks})) + monkeypatch.setattr(apps, "APPS", tmp_path) + + class Desktop(FakeDriver): + windows, synced, ran = [], False, [] + + @list_windows + async def listed(self): + return self.windows + + @run_in_session + async def run(self, argv, cwd, env, timeout_s): + self.ran.append((argv, cwd, env)) + return ExecResult(0 if self.synced else 1, stderr=b"no sync yet\n") + + driver = Desktop() + async with http_client(driver) as http: + before = (await http.get("/agentenv/ext/computer-ready/status")).json() + driver._computer.started -= 120 + late = (await http.get("/agentenv/ext/computer-ready/status")).json() + driver.windows, driver.synced = [{"instance": "mail", "class": "Mail", "title": "Inbox"}], True + after = (await http.get("/agentenv/ext/computer-ready/status")).json() + listener.close() + + assert before["ready"] is False and [c for c in before["checks"] if c["name"].startswith("app ")] == [ + {"name": "app mail", "ok": False, "detail": "not ready yet, 0 s in: no window with class 'Mail'"}, + {"name": "app notes", "ok": True, "detail": "ready"}, + {"name": "app sync", "ok": False, "detail": "not ready yet, 0 s in: @ready_check synced exited 1: no sync yet"}, + ] + assert late["checks"][-1]["detail"].startswith("not ready after 60 s: @ready_check synced exited 1") + assert after["ready"] is True + assert driver.ran[0] == ([str(tmp_path / "sync/venv/bin/python"), "-m", "agentenv_protocol.computer._ready", + str(tmp_path / "sync"), "app.py", "App", "synced"], str(tmp_path / "sync"), + {"ENVIRONMENT_NAME": "sync"}) + + +@pytest.mark.asyncio +async def test_the_linux_default_lists_windows_by_their_class_instance_and_title(monkeypatch): + replies = {("-root", "_NET_CLIENT_LIST"): "_NET_CLIENT_LIST(WINDOW): window id # 0x1e00004, 0x2000003\n", + ("-id", "0x1e00004", "WM_CLASS", "_NET_WM_NAME"): + '_NET_WM_NAME(UTF8_STRING) = "Inbox \\"work\\""\nWM_CLASS(STRING) = "mail", "Mail"\n', + ("-id", "0x2000003", "WM_CLASS", "_NET_WM_NAME"): ""} + + async def xprop(*args): + return replies[args] + + monkeypatch.setattr(linux, "_xprop", xprop) + assert await linux.list_windows() == [{"instance": "mail", "class": "Mail", "title": 'Inbox \\"work\\"'}] + + @pytest.mark.asyncio @pytest.mark.parametrize("captured, error", [ (lambda screen: "pixels", "@capture returned str, not an Image"), diff --git a/src/agent_env/drivers/envs/reference/driver.py b/src/agent_env/drivers/envs/reference/driver.py index a8de1b3b0..2b2cfd628 100644 --- a/src/agent_env/drivers/envs/reference/driver.py +++ b/src/agent_env/drivers/envs/reference/driver.py @@ -18,7 +18,7 @@ import mss from Xlib import X, XK from Xlib import display as xdisplay -from Xlib.error import ConnectionClosedError, DisplayConnectionError +from Xlib.error import BadWindow, ConnectionClosedError, DisplayConnectionError from Xlib.ext import xtest from agentenv_protocol.computer import ( @@ -40,6 +40,7 @@ frames_in_thread, key_down, key_up, + list_windows, pointer_down, pointer_move, pointer_position, @@ -185,6 +186,10 @@ async def down(self, button) -> None: async def up(self, button) -> None: await self._send(lambda c: c.button(button, False)) + @list_windows + async def windows(self) -> list[dict[str, str]]: + return await self._read(_windows) + @pointer_position async def where(self) -> tuple[int, int]: def read(c): @@ -267,6 +272,24 @@ def _connected(self) -> _Connection: raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None +def _windows(c: _Connection) -> list[dict[str, str]]: + """The window manager's top-level windows by WM_CLASS and title; a window that closes while it's read is left + out.""" + listed = c.d.screen().root.get_full_property(c.d.intern_atom("_NET_CLIENT_LIST"), X.AnyPropertyType) + windows = [] + for wid in listed.value if listed is not None else (): + window = c.d.create_resource_object("window", wid) + try: + names = window.get_wm_class() + title = window.get_full_property(c.d.intern_atom("_NET_WM_NAME"), c.d.intern_atom("UTF8_STRING")) + except BadWindow: + continue + if names: + windows.append({"instance": names[0], "class": names[1], + "title": title.value.decode(errors="replace") if title is not None else ""}) + return windows + + class _Connection: """One connection to the X server, with XTEST input on it. A character the keymap lacks is typed through a spare keycode mapped to it, kept for that character and reused; once all are taken, the least recently used one that diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index a05ca8719..6d622c5fc 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -4,6 +4,7 @@ from __future__ import annotations +from pathlib import PurePosixPath from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self from agentenv_protocol.computer import AppDeclaration @@ -82,6 +83,7 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact app = cls(id=id, version=None, environment_name=environment_name, module=module, files=files, declaration=declaration, computer=computer, driver=driver) _, problems = _installs(declaration) + problems += _ready_checks_beyond_reach(module, declaration) if (computer is None) != (driver is None): problems.append("a computer app deploys alone on a computer and a driver; name both or neither") elif computer is not None and not problems: @@ -145,3 +147,11 @@ def deploy_refusal(self, *, sandbox_type: str | None = None, env_params: Optiona def _installs(declaration: AppDeclaration) -> tuple[dict[str, AppInstall], list[str]]: """Its installs by OS type, each checked as the composer installs it, and the problems with them.""" return installs({os_type: linux.model_dump() for os_type, linux in declaration.install.items()}, AppInstall) + + +def _ready_checks_beyond_reach(module: str, declaration: AppDeclaration) -> list[str]: + """An install that leaves out the class's file, from which the computer runs its ``@ready_check`` methods.""" + file = PurePosixPath(module.partition(":")[0]) + return [f"install.{os_type}.files: @ready_check runs from the app's folder, which needs {str(file)!r} in " + f"{linux.files!r}" for os_type, linux in declaration.install.items() + if declaration.ready_checks and linux.files != "." and not file.is_relative_to(linux.files)] diff --git a/src/agent_env/env/envs/computer/utils/boot.py b/src/agent_env/env/envs/computer/utils/boot.py index 2f114e46b..66d309ea3 100644 --- a/src/agent_env/env/envs/computer/utils/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -27,6 +27,8 @@ #: How long a computer may take from its container starting to its driver saying it's ready. READY_TIMEOUT_S = 180.0 +#: How much longer than the slowest starting app's own readiness timeout a computer may take, for the rest of it first. +APP_READY_MARGIN_S = 30.0 _POLL_S = 0.5 #: How a setting a record keeps shows a secret's or a generated one's value, which it never holds. WITHHELD = "" @@ -82,7 +84,8 @@ def recorded_settings(computer: ComputerArtifact, values: dict[str, Any]) -> dic async def boot(sandbox_provider: SandboxProvider, *, image: str, port: int, environment: dict[str, str], network_policy: Optional[NetworkPolicy], timeout: int, attribution: Optional[Attribution], - disk_size_gb: float = 10, cpu: Optional[float] = None, memory_mb: Optional[int] = None) -> Booted: + disk_size_gb: float = 10, cpu: Optional[float] = None, memory_mb: Optional[int] = None, + ready_s: float = READY_TIMEOUT_S) -> Booted: """``image`` booted with ``environment`` on ``sandbox_provider``, which the caller chose to run its arch natively, its driver's ``port`` reached on the sandbox, once the driver says the computer is ready; a boot that fails is torn down.""" @@ -91,7 +94,7 @@ async def boot(sandbox_provider: SandboxProvider, *, image: str, port: int, envi disk_size_gb=disk_size_gb, timeout=timeout, attribution=dict(attribution or {}), **_size_kwargs(cpu, memory_mb)) try: url = sandbox.tunnel_urls[port] - return Booted(sandbox, url, await ready(url, READY_TIMEOUT_S)) + return Booted(sandbox, url, await ready(url, ready_s)) except BaseException: await discard(sandbox) raise @@ -109,6 +112,13 @@ async def discard(sandbox: Sandbox) -> None: await asyncio.to_thread(remove_local_work_dir, sandbox.sandbox_id) +def ready_timeout(apps: list[ComputerAppEnv]) -> float: + """How long a computer of ``apps`` may take to be ready: ``READY_TIMEOUT_S``, or longer when an app that starts with + it gives itself longer.""" + return max([READY_TIMEOUT_S, *(launch.ready.timeout_s + APP_READY_MARGIN_S for app in apps + if (launch := app.declaration.launch) and launch.at == "boot" and launch.ready)]) + + async def ready(url: str, timeout: float) -> dict: """The card of the driver at ``url`` once its ``computer-ready`` status says it's ready; raises TimeoutError, naming what it last saw, when it isn't within ``timeout`` seconds.""" diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 495f682b5..e4312c63a 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -14,6 +14,7 @@ import json import logging import re +import shlex import shutil import subprocess import tarfile @@ -27,6 +28,7 @@ from collections.abc import Callable from typing import TYPE_CHECKING, Any +from agentenv_protocol.computer import apps as installed_apps from agentenv_protocol.computer.conformance import run_probe from agent_env.artifact.artifacts.computer import ComputerArtifact @@ -60,7 +62,12 @@ REPOSITORY = "agentenv-computers" #: Where a driver's files go in a composed computer, in a folder named for the driver; an app's, under APPS. DRIVERS = "/opt/agentenv/drivers" -APPS = "/opt/agentenv/apps" +APPS = str(installed_apps.APPS) +#: Where a desktop session finds its launchers, and the entries it starts with it. +LAUNCHERS = "/usr/share/applications" +AUTOSTART = "/etc/xdg/autostart" +#: The script in an app's folder that its launcher and autostart entry run. +LAUNCH_SCRIPT = ".agentenv/launch" KEY_LABEL = "dev.agentenv.computer-key" # What a build installed and whether it was emulated, kept on its image for an env that reuses it. INSTALLED_LABEL, EMULATED_LABEL = "dev.agentenv.installed", "dev.agentenv.emulated" @@ -168,7 +175,8 @@ async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: Compute for link in links: try: booted = await boot.boot(link, image=image, port=driver.port, environment=environment, - network_policy=None, timeout=_PROBE_TTL_S, attribution=None) + network_policy=None, timeout=_PROBE_TTL_S, attribution=None, + ready_s=boot.ready_timeout(env.composed_apps())) except Exception as e: problems.append(f"{type(link).__name__}: {e}") continue @@ -269,12 +277,14 @@ def _ready_base(base: ComputerArtifact, arch: str) -> None: @dataclass(frozen=True) class _Layer: """A driver's or an app's part of a build: its folder in the build context, its prefix in the image, what it - installs, and the file of its supervisord programs.""" + installs, the file of its supervisord programs, and files of the build context it places elsewhere in the image, + as (source, target).""" folder: str prefix: str install: Install conf: str + placed: tuple[tuple[str, str], ...] = () def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifact) -> _Layer: @@ -287,14 +297,40 @@ def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifa def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact) -> _Layer: - """Write the app's install files under ``context/apps//`` and its supervisord programs beside them, run under - its name, its folder under ``APPS`` named for its id's last part.""" - install, name = app.install[base.os.type], _name(app.id, "app") - layer = _Layer(f"apps/{name}", f"{APPS}/{name}", install, f"agentenv-app-{name}.conf") + """Write the app's install files under ``context/apps//``, its supervisord programs beside them, run under + its name, and the description of it its computer's driver reads; with a launch, its launcher, which a ``boot`` app + also places among the desktop session's autostart entries. Its folder under ``APPS`` is named for its id's last + part.""" + install, name, launch = app.install[base.os.type], _name(app.id, "app"), app.declaration.launch + prefix, placed = f"{APPS}/{name}", () + if launch is not None: + entry = f"desktop/agentenv-app-{name}.desktop" + (context / "desktop").mkdir(exist_ok=True) + (context / entry).write_text(f"[Desktop Entry]\nType=Application\nName={app.environment_name}\n" + f"Exec={prefix}/{LAUNCH_SCRIPT}\nTerminal=false\n") + placed = ((entry, f"{LAUNCHERS}/agentenv-app-{name}.desktop"), + *([(entry, f"{AUTOSTART}/agentenv-app-{name}.desktop")] if launch.at == "boot" else [])) + layer = _Layer(f"apps/{name}", prefix, install, f"agentenv-app-{name}.conf", placed) _stage(context, layer, f"app {app.id}", app.files, f"app-{name}", {"ENVIRONMENT_NAME": app.environment_name}) + described = context / layer.folder / installed_apps.APP_FILE + described.parent.mkdir(parents=True, exist_ok=True) + if launch is not None: + script = context / layer.folder / LAUNCH_SCRIPT + script.write_text(_launcher(app.environment_name, prefix, launch.command)) + script.chmod(_EXECUTABLE) + described.write_text(json.dumps({ + "name": app.environment_name, "module": app.module, + "launch": launch.model_dump(mode="json", exclude_none=True) if launch is not None else None, + "ready_checks": app.declaration.ready_checks}, indent=1)) return layer +def _launcher(name: str, prefix: str, command: list[str]) -> str: + """The script a desktop entry runs: ``command`` from the app's folder, as its services run, with + ``ENVIRONMENT_NAME`` set. A desktop session may ignore an entry's own working directory.""" + return f"#!/bin/sh\ncd {prefix} && ENVIRONMENT_NAME={shlex.quote(name)} exec {shlex.join(command)}\n" + + def _name(env_id: str, fallback: str) -> str: return re.sub(r"[^a-z0-9-]+", "-", env_id.rsplit("/", 1)[-1].lower()).strip("-") or fallback @@ -378,7 +414,9 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build lines.append(f"RUN python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip install --no-cache-dir {packages} " f"&& {prefix}/venv/bin/pip list --format=json > {prefix}/installed.json") lines.append(f"COPY {layer.conf} /etc/supervisor/conf.d/{layer.conf}") - checks = [f"test -d {prefix}", f"test -f /etc/supervisor/conf.d/{layer.conf}"] + lines += [f"COPY {source} {target}" for source, target in layer.placed] + checks = [f"test -d {prefix}", f"test -f /etc/supervisor/conf.d/{layer.conf}", + *(f"test -f {target}" for _, target in layer.placed)] if install.python is not None: checks.append(f"test -x {prefix}/venv/bin/python") lines.append(f"RUN {' && '.join(checks)}") diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index ce0794fc3..5dc11e489 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -15,11 +15,11 @@ from agent_env.env.env import DeployedSandboxEnv, EnvCapabilityUnsupported from agent_env.env.envs.computer.utils import params as computer_params from agent_env.env.envs.computer.utils.boot import ( - READY_TIMEOUT_S, app_settings_environment, boot, discard, ready, + ready_timeout, recorded_settings, settings_environment, ) @@ -96,7 +96,7 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par booted = await boot( sandbox_provider, image=build["image"], port=env.driver.port, network_policy=policy, environment=_environment(env, computer, values, params), timeout=ttl_seconds, attribution=attribution, - disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb) + disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb, ready_s=ready_timeout(env.composed_apps())) self._container_sandboxes.append(booted.sandbox) self._environment_sandboxes[name] = booted.sandbox try: @@ -136,7 +136,7 @@ async def reboot(self, env: ComputerEnv, deployed: DeployedSandboxEnv) -> None: env=_environment(env, computer, values, params)) except NotImplementedError as e: # a reset the env supports, which this provider can't carry out raise RuntimeError(f"'{env.id}' can't be reset on {type(sandbox_provider).__name__}: {e}") from None - _require_name(env, await ready(deployed.environment_url, READY_TIMEOUT_S)) + _require_name(env, await ready(deployed.environment_url, ready_timeout(env.composed_apps()))) await _require_display(deployed, values.get("resolution")) diff --git a/tst/integration/computer/app/artifacts/check_notes/run.py b/tst/integration/computer/app/artifacts/check_notes/run.py index 38b72f11b..9190cd4c3 100644 --- a/tst/integration/computer/app/artifacts/check_notes/run.py +++ b/tst/integration/computer/app/artifacts/check_notes/run.py @@ -1,5 +1,5 @@ -"""What the notes app's service wrote, which it can only once its settings reached it, and the name the computer's -card gives, asked on loopback from inside the computer.""" +"""What the notes app's service wrote, which it can only once its settings reached it, the name the computer's card +gives and what its readiness says of the app, asked on loopback from inside the computer, and where its launcher is.""" import json import time import urllib.request @@ -14,4 +14,9 @@ def run(input): time.sleep(0.5) with urllib.request.urlopen(f"{input['env']['url']}/.well-known/agent-env.json", timeout=10) as response: card = json.load(response) - return {**json.loads(written.read_text()), "card": card["name"]} + with urllib.request.urlopen(f"{input['env']['url']}/agentenv/ext/computer-ready/status", timeout=30) as response: + ready = json.load(response) + return {**json.loads(written.read_text()), "card": card["name"], + "app_ready": [check for check in ready["checks"] if check["name"].startswith("app ")], + "launchers": sorted(p.name for p in Path("/usr/share/applications").glob("agentenv-app-*.desktop")), + "autostart": sorted(p.name for p in Path("/etc/xdg/autostart").glob("agentenv-app-*.desktop"))} diff --git a/tst/integration/computer/app/envs/notes/app.py b/tst/integration/computer/app/envs/notes/app.py index d96ccfeef..285e71105 100644 --- a/tst/integration/computer/app/envs/notes/app.py +++ b/tst/integration/computer/app/envs/notes/app.py @@ -1,15 +1,23 @@ -"""A notes app for the computer app tests: its service writes the settings a deploy gave it to ~/.notes.json.""" +"""A notes app for the computer app tests: its service writes the settings a deploy gave it to ~/.notes.json, and its +window opens with the desktop session; it's ready once both have happened.""" +from pathlib import Path + from pydantic import BaseModel, SecretStr -from agentenv_protocol.computer import ComputerAppEnvironment, Launch, Linux, Service, computer_app +from agentenv_protocol.computer import ComputerAppEnvironment, Launch, Linux, Ready, Service, computer_app, ready_check @computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) class Notes(ComputerAppEnvironment): linux = Linux(python="requirements.txt", services=[Service(name="notes", command="venv/bin/python serve.py")]) - launch = Launch(at="demand", command=["xdg-open", ".notes.json"]) + launch = Launch(at="boot", command=["venv/bin/python", "window.py"], window={"class": "Notes"}, + ready=Ready(window=True)) skill = "skill.md" class Settings(BaseModel): signature: str = "-- notes" token: SecretStr = SecretStr("") + + @ready_check + def written(self) -> bool: + return (Path.home() / ".notes.json").exists() diff --git a/tst/integration/computer/app/envs/notes/window.py b/tst/integration/computer/app/envs/notes/window.py new file mode 100644 index 000000000..da9b997da --- /dev/null +++ b/tst/integration/computer/app/envs/notes/window.py @@ -0,0 +1,9 @@ +"""The notes app's window, of class Notes, showing the signature its settings give.""" +import tkinter as tk + +from app import Notes + +root = tk.Tk(className="notes") +root.title("Notes") +tk.Label(root, text=Notes().settings.signature).pack(padx=40, pady=40) +root.mainloop() diff --git a/tst/integration/computer/computer_app_test.py b/tst/integration/computer/computer_app_test.py index 7a6903043..b699feb2a 100644 --- a/tst/integration/computer/computer_app_test.py +++ b/tst/integration/computer/computer_app_test.py @@ -1,6 +1,7 @@ """A computer app deployed alone through ``agent-env run``: a one-app computer of the computer and driver its env.toml names, its card named after it, its service running from its folder in a venv of the plain protocol package, on the -settings the deploy gave it, a secret resolved at boot. Needs Docker and the local registry on localhost:5000.""" +settings the deploy gave it, a secret resolved at boot, its window opened with the desktop session and the computer ready +only once its window and its @ready_check say so. Needs Docker and the local registry on localhost:5000.""" from pathlib import Path @@ -24,5 +25,7 @@ def test_an_app_deployed_alone_runs_its_service_on_its_settings_on_a_computer_na context = run_context(ran.output) (deployed,) = context["deployed_envs"] assert deployed["env_id"].endswith("/notes") and deployed["environment_card"]["name"] == "notes" - assert context["metadata"]["script_results"]["service"] == {"signature": "-- Ada", "token": "t0ken", - "card": "notes"} + assert context["metadata"]["script_results"]["service"] == { + "signature": "-- Ada", "token": "t0ken", "card": "notes", + "app_ready": [{"name": "app notes", "ok": True, "detail": "ready"}], + "launchers": ["agentenv-app-notes.desktop"], "autostart": ["agentenv-app-notes.desktop"]} diff --git a/tst/unit/drivers/reference_driver_test.py b/tst/unit/drivers/reference_driver_test.py index cbed28526..42db3d6fe 100644 --- a/tst/unit/drivers/reference_driver_test.py +++ b/tst/unit/drivers/reference_driver_test.py @@ -32,6 +32,10 @@ class ConnectionClosedError(Exception): pass +class BadWindow(Exception): + pass + + class FakeX: """An X display: its screen and pointer, a US-like keymap whose capital letters are the second level of their keys, the keys down, the XTEST events sent, ``drops`` syncs and the ``fail_at``-th event that fail, and the @@ -47,11 +51,25 @@ def __init__(self) -> None: self.fail_at = None self.spare = {8} self.display = types.SimpleNamespace(info=types.SimpleNamespace(min_keycode=8, max_keycode=255)) + #: The window manager's windows by id, each its WM_CLASS instance and class and its title, or None once closed. + self.windows: dict[int, tuple[str, str, str] | None] = {} def screen(self): query = lambda: types.SimpleNamespace(root_x=self.pointer[0], root_y=self.pointer[1]) # noqa: E731 + listed = lambda atom, kind: types.SimpleNamespace(value=list(self.windows)) # noqa: E731 return types.SimpleNamespace(width_in_pixels=WIDTH, height_in_pixels=HEIGHT, - root=types.SimpleNamespace(query_pointer=query)) + root=types.SimpleNamespace(query_pointer=query, get_full_property=listed)) + + def intern_atom(self, name: str) -> str: + return name + + def create_resource_object(self, kind: str, wid: int): + def named(): + if self.windows[wid] is None: + raise BadWindow(wid) + return self.windows[wid][:2] + title = lambda atom, kind: types.SimpleNamespace(value=self.windows[wid][2].encode()) # noqa: E731 + return types.SimpleNamespace(get_wm_class=named, get_full_property=title) def sync(self) -> None: if self.drops: @@ -156,10 +174,11 @@ def module(name: str, **attrs) -> types.ModuleType: return m X = module("Xlib.X", MotionNotify="move", ButtonPress="button-down", ButtonRelease="button-up", - KeyPress="key-down", KeyRelease="key-up") + KeyPress="key-down", KeyRelease="key-up", AnyPropertyType=0) XK = module("Xlib.XK", string_to_keysym=lambda name: keysyms.setdefault(name, 0xff00 + len(keysyms))) display = module("Xlib.display", Display=open_display) - module("Xlib.error", ConnectionClosedError=ConnectionClosedError, DisplayConnectionError=ConnectionClosedError) + module("Xlib.error", ConnectionClosedError=ConnectionClosedError, DisplayConnectionError=ConnectionClosedError, + BadWindow=BadWindow) xtest = module("Xlib.ext.xtest", fake_input=fake_input) module("Xlib.ext", xtest=xtest) module("Xlib", X=X, XK=XK, display=display) @@ -397,3 +416,12 @@ async def test_the_reference_driver_records_from_its_stream_at_up_to_30_fps(x11) card = (await http.get("/.well-known/agent-env.json")).json() recording = next(e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING) assert recording["params"]["source"] == "stream" and recording["params"]["fps"] == {"default": 10, "max": 30} + + +@pytest.mark.asyncio +async def test_the_desktops_windows_are_listed_by_class_and_title_leaving_out_one_that_closed(x11): + driver, fake, _ = x11 + fake.windows = {0x1e00004: ("notes", "Notes", "Notes"), 0x2000003: None, 0x2200001: ("xfdesktop", "Xfdesktop", "")} + + assert await driver.windows() == [{"instance": "notes", "class": "Notes", "title": "Notes"}, + {"instance": "xfdesktop", "class": "Xfdesktop", "title": ""}] diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 470b5d1cf..07f556281 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -84,7 +84,12 @@ def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_ self.builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), "files": sorted(p.relative_to(context).as_posix() for p in context.rglob("*") if p.is_file()), "conf": next(context.glob("agentenv-*.conf")).read_text(), - "confs": {conf.name: conf.read_text() for conf in context.glob("agentenv-*.conf")}}) + "confs": {conf.name: conf.read_text() for conf in context.glob("agentenv-*.conf")}, + "desktop": {entry.name: entry.read_text() for entry in context.glob("desktop/*")}, + "described": {path.parent.parent.name: json.loads(path.read_text()) + for path in context.glob("apps/*/.agentenv/app.json")}, + "launchers": {path.parent.parent.name: (path.read_text(), bool(path.stat().st_mode & 0o111)) + for path in context.glob("apps/*/.agentenv/launch")}}) def label(self, local, platform, extra): self.labels[local] = extra @@ -934,7 +939,8 @@ async def test_a_load_outside_the_home_is_refused_before_anything_is_sent(base, "theme": {"enum": ["light", "dark"], "type": "string", "default": "light"}}, "type": "object"} -def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, **changes): +def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, launch=None, + ready_checks=(), **changes): """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements.""" folder.mkdir(parents=True, exist_ok=True) (folder / "app.py").write_text("class Mail: ...\n") @@ -943,8 +949,8 @@ def _app(base, driver, folder, compose=True, supports=None, install=None, settin name: folder / name for name in ("app.py", "requirements.txt")}) declaration = AppDeclaration.model_validate({ "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], - "permissions": [], "launch": None, "settings": settings, "skill": None, - "data": [], "install_steps": [], "ready_checks": [], + "permissions": [], "launch": launch, "settings": settings, "skill": None, + "data": [], "install_steps": [], "ready_checks": list(ready_checks), "install": {"linux": install or {"python": "requirements.txt", "services": [ {"name": "imap", "command": "venv/bin/python imap.py"}]}}}) return ComputerAppEnv.put(**{"id": "@local/~/kit/mail", "environment_name": "mail", "module": "app.py:Mail", @@ -975,6 +981,26 @@ def test_an_app_deployed_alone_is_composed_with_its_files_venv_and_programs_unde "container/arm64": "ready"} +@pytest.mark.parametrize("at", ["boot", "demand"]) +def test_an_apps_launch_reaches_the_desktop_session_and_its_driver(base, driver, docker, tmp_path, at): + launch = {"at": at, "command": ["bin/mail", "--folder", "Work mail"], "window": {"class": "Mail"}, + "ready": {"window": True, "timeout_s": 240.0}} + app = _app(base, driver, tmp_path / "mail", launch=launch, ready_checks=["inbox_open"]) + + build = docker.builds[0] + assert build["desktop"] == {"agentenv-app-mail.desktop": ( + "[Desktop Entry]\nType=Application\nName=mail\nExec=/opt/agentenv/apps/mail/.agentenv/launch\nTerminal=false\n")} + assert build["launchers"] == {"mail": ( + "#!/bin/sh\ncd /opt/agentenv/apps/mail && ENVIRONMENT_NAME=mail exec bin/mail --folder 'Work mail'\n", True)} + copies = [line for line in build["dockerfile"].splitlines() if line.startswith("COPY desktop/")] + assert copies == ["COPY desktop/agentenv-app-mail.desktop /usr/share/applications/agentenv-app-mail.desktop", + *(["COPY desktop/agentenv-app-mail.desktop /etc/xdg/autostart/agentenv-app-mail.desktop"] + if at == "boot" else [])] + assert build["described"] == {"mail": {"name": "mail", "module": "app.py:Mail", "launch": launch, + "ready_checks": ["inbox_open"]}} + assert boot.ready_timeout([app]) == (270.0 if at == "boot" else boot.READY_TIMEOUT_S) + + def test_an_app_is_composed_only_for_the_arches_it_supports(base, driver, docker, tmp_path): _app(base, driver, tmp_path / "mail", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["arm64"]}]) @@ -988,7 +1014,9 @@ def test_an_app_is_composed_only_for_the_arches_it_supports(base, driver, docker ({"install": {"services": [{"name": "imap", "command": "a"}, {"name": "imap", "command": "b"}]}}, "install.linux.services: Value error, names a service twice"), ({"driver": None}, "a computer app deploys alone on a computer and a driver; name both or neither"), -], ids=["platform", "arch", "install", "no-driver"]) + ({"install": {"files": "linux", "python": "requirements.txt", "services": []}, "ready_checks": ["inbox_open"]}, + "install.linux.files: @ready_check runs from the app's folder, which needs 'app.py' in 'linux'"), +], ids=["platform", "arch", "install", "no-driver", "ready-check-file"]) def test_an_app_that_doesnt_fit_its_computer_and_driver_is_refused_before_anything_is_stored(base, driver, docker, tmp_path, changes, problem): with pytest.raises(ValueError, match=problem.replace(".", "[.]")): From 1f63738183bfa35429c1f728434fb74a03b66c02 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 09:31:29 -0700 Subject: [PATCH 2/6] feat(computer): each computer app runs its own server An app's ComputerAppEnvironment class now runs on the computer as the app's own server (serve_app), in the app's venv, on .agentenv/server.sock in its folder. It serves the app's card with computer-app/v1: the app's launch as params, launch, and ready, which checks the app's window, port, command and @ready_check methods in-process, each within 10 s. Once the desktop session is up it adds the app's launcher to the session user's menu, which launches through the server, and launches a boot app once per boot. What it does in the session is the SDK's per-OS session module. The driver finds apps by their sockets: computer-ready needs every app's server to answer, each boot app and each app with no launch to be ready, and a demand app once launched. list_windows leaves the driver contract, and app.json, the launch script, the XDG entries and _ready go. The composer gives every app a venv with the protocol's new app extra (mcp, uvicorn) and its server as a supervisord program, and put refuses an app whose install leaves out its class's file. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- packages/agentenv-protocol/README.md | 58 ++-- packages/agentenv-protocol/pyproject.toml | 6 + .../agentenv_protocol/computer/__init__.py | 3 +- .../src/agentenv_protocol/computer/_load.py | 22 ++ .../src/agentenv_protocol/computer/_ready.py | 31 -- .../src/agentenv_protocol/computer/app.py | 135 +++++++- .../src/agentenv_protocol/computer/apps.py | 128 ++------ .../src/agentenv_protocol/computer/driver.py | 4 - .../src/agentenv_protocol/computer/launch.py | 23 ++ .../src/agentenv_protocol/computer/linux.py | 23 -- .../agentenv_protocol/computer/operations.py | 4 +- .../src/agentenv_protocol/computer/runtime.py | 6 +- .../src/agentenv_protocol/computer/serve.py | 14 +- .../agentenv_protocol/computer/serve_app.py | 41 +++ .../src/agentenv_protocol/computer/session.py | 64 ++++ .../tests/test_computer_app_server.py | 291 ++++++++++++++++++ .../tests/test_computer_declare.py | 21 -- .../tests/test_computer_driver.py | 66 ---- src/agent_env/computers/README.md | 4 +- .../drivers/envs/reference/driver.py | 25 +- src/agent_env/env/envs/computer/app_env.py | 12 +- .../env/envs/computer/utils/composer.py | 109 +++---- .../computer/app/artifacts/check_notes/run.py | 7 +- .../computer/app/envs/notes/app.py | 5 +- .../computer/app/envs/notes/requirements.txt | 1 - tst/integration/computer/computer_app_test.py | 9 +- tst/unit/drivers/reference_driver_test.py | 34 +- tst/unit/env/envs/test_computer_env.py | 72 +++-- uv.lock | 8 +- 30 files changed, 769 insertions(+), 459 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/_load.py delete mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/_ready.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/launch.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/session.py create mode 100644 packages/agentenv-protocol/tests/test_computer_app_server.py delete mode 100644 tst/integration/computer/app/envs/notes/requirements.txt diff --git a/AGENTS.md b/AGENTS.md index 915e986d0..c523d41be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,7 +107,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`: files, a `python` requirements file for a venv, and the services the base's init runs; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile and build context so any env with the same inputs reuses it; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs, a launcher for its `launch` (an XDG autostart entry for a `boot` one) and `.agentenv/app.json`, from which the driver counts an app that starts with the computer in `computer-ready`, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`: files, a `python` requirements file for a venv, and the services the base's init runs; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile and build context so any env with the same inputs reuses it; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 03068cca9..adcc94760 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -393,7 +393,6 @@ pixels), at the display's size times its scale. Input operations may return an ` | `@key_down`, `@key_up` | both or none | `computer-keyboard/v1` and its two tools | | `@run_in_session` | no; a Linux default | `computer-exec/v1` | | `@write_files` | no; a Linux default | `data/add` | -| `@list_windows` | no; a Linux default | a computer app's readiness by its window | | `@record` | no | the stream `computer-recording/v1` records from; without it the base class polls `@capture` | The class is checked when it is defined: a missing required operation, half a group, two methods @@ -401,13 +400,12 @@ for one operation that some platform could both match, or features that don't ma decorators raise `TypeError`. Predicates (`os`, `distribution`, `version`, `arch`, `display`) choose a method per platform, read from `/etc/os-release`, the machine and `XDG_SESSION_TYPE` or `DISPLAY`: `@capture(display="x11")` and `@capture(display="wayland")` can sit side by side. A -driver's own `@run_in_session`, `@write_files` or `@list_windows` replaces the base class's Linux -default, which runs as the driver's user inside the desktop session. The base class keeps its state -in `self._computer` and names its own members `_computer_*` and `computer_*`; a driver that -redefines one of them, or another private member of the base class, is refused when it is defined, -so a helper of the driver's can't silently replace one. What the optional operations give is -served only when the driver has them, through the `served_tools`, `served_extensions` and -`served_operations` hooks. +driver's own `@run_in_session` or `@write_files` replaces the base class's Linux default, which runs +as the driver's user inside the desktop session. The base class keeps its state in `self._computer` +and names its own members `_computer_*` and `computer_*`; a driver that redefines one of them, or +another private member of the base class, is refused when it is defined, so a helper of the +driver's can't silently replace one. What the optional operations give is served only when the +driver has them, through the `served_tools`, `served_extensions` and `served_operations` hooks. **What the base class serves:** @@ -430,8 +428,8 @@ served only when the driver has them, through the `served_tools`, `served_extens the live screen size, what its input takes, its OS's shortcuts) and ending with the driver class's `notes`, markdown for what its card can't say. - **Framework extensions.** `computer-ready/v1` (`status`: real checks of the display and a - capture, the driver's own `ready_checks()`, and one per app that starts with the computer; tools - refuse `device_unavailable` until they pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool + capture, the driver's own `ready_checks()`, and one per app on the computer, asked of the app's + own server (see Computer apps); tools refuse `device_unavailable` until they pass once), `computer-display/v1`, `computer-spaces/v1` (the default space, written into the tool descriptions), `computer-keyboard/v1`, `computer-exec/v1` (`run {argv, cwd?, env?, timeout_s?, request_id}`: output capped per stream, a repeated `request_id` answered with the first result) and, with `recording`, `computer-recording/v1` (below). @@ -543,9 +541,10 @@ and gives it back. A computer app is software an agent uses on a computer, beside the driver: a desktop app, a service or both. Write one as a `ComputerAppEnvironment` subclass from `agentenv_protocol.computer`, an -`AgentEnvEnvironment` whose class also says what it is to a computer. It needs only the plain -package: `agentenv_protocol.computer` imports what a driver needs (mcp, uvicorn, Pillow, PyAV) only -when a driver's names are used. +`AgentEnvEnvironment` whose class also says what it is to a computer. Writing one and reading what +it declares need only the plain package: `agentenv_protocol.computer` imports what a driver needs +(mcp, uvicorn, Pillow, PyAV) only when a driver's names are used. On a computer it runs as its own +server, which needs the `app` extra (below). ```python from pydantic import BaseModel, SecretStr @@ -566,22 +565,35 @@ class MailApp(ComputerAppEnvironment): `permissions` it needs of the computer. Its env.toml names the rest: the app itself (`environment_name`) and the `computer` and `driver` it deploys alone on. - **`linux = Linux(...)`** is how a Linux computer installs it: `files`, a folder of the app's - copied under its prefix; `python`, a requirements file there installed into a venv beside them; - `services`, programs the computer's init runs from the prefix as the session user. -- **`launch = Launch(...)`** says how it starts in the desktop session: with it (`at="boot"`, an XDG - autostart entry) or when the agent opens it (`"demand"`); either way the computer gets a launcher - for it. `ready=Ready(...)` says when it's ready: a window matching `launch.window` by its class, - instance or title, a port answering on loopback, or a command exiting 0. `@ready_check` marks a - method for readiness that needs logic, run in the app's own venv, so an app with one names its - `python` requirements. A `boot` app counts in `computer-ready`, as does an app with no launch whose - `@ready_check` methods watch its services; a `demand` app is checked once something launches it. - `@install_step(scope=...)` marks a method a computer runs as it installs the app. + copied under its prefix, which holds the class's file; `python`, a requirements file there, + installed into the venv every app gets beside them; `services`, programs the computer's init runs + from the prefix as the session user. +- **`launch = Launch(...)`** says how it starts in the desktop session: with it (`at="boot"`) or when + the agent opens it (`"demand"`). `ready=Ready(...)` says when it's ready, within `timeout_s` of its + launch: a window matching `launch.window` by its class, instance or title, a port answering on + loopback, or a command exiting 0. `@ready_check` marks a method for readiness that needs logic (a + sync one runs in a thread), and `@install_step(scope=...)` one a computer runs as it installs the + app. - **Its skill** is a file of its folder (`skill = "skill.md"`) or an `@agent_skill` method, not both; its data hooks are the usual `@add_data`, `@get_data` and `@reset_data`. - **`Settings`** is typed; a computer hands each app the values a deploy gives it, as one JSON value in `APP_SETTINGS_ENV` (`AGENTENV_APP_SETTINGS`, the apps' values by name), and the base class reads its own, by `ENVIRONMENT_NAME`, as `self.settings`. A `SecretStr` field is a secret. +On a computer each app is its own server: `python -m agentenv_protocol.computer.serve_app +:`, a program the computer's init runs from the app's folder in the app's venv, which +has the `app` extra (`agentenv-framework-protocol[app]`: mcp and uvicorn). It serves the class's +card and tools on the Unix socket `.agentenv/server.sock` in that folder, with `computer-app/v1`: +the app's `launch` as its params, and the methods `launch` (POST), which starts the app in the +desktop session, and `ready` (GET: `ready`, `problems`, `launched`, `waited_s` and `timeout_s`), +which runs the app's readiness checks then, each within 10 s. Once the session is up the server adds +the app's launcher to the session user's menu, which runs `python -m +agentenv_protocol.computer.launch ` to ask the server, so the server starts every copy of +the app, and launches a `boot` app, once per boot. The driver finds each app by the socket in its +folder: the computer is ready once every app's server answers and each `boot` app, and each app +with no launch, says it's ready; a `demand` app counts once something launches it. What the server +does in the session (`agentenv_protocol.computer.session`) is Linux's. + `MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app` env stores, and `python -m agentenv_protocol.computer._declare ` prints it as JSON, or a driver's `DriverDeclaration`, without the class's own dependencies installed. diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index de516de1d..f9adb24a6 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -35,6 +35,12 @@ agent = [ "regex>=2023.12.25", "uvicorn>=0.30", ] +# A computer app's own server: ComputerAppEnvironment serves its card and computer-app/v1 over mcp's FastMCP, on a Unix +# socket through uvicorn. +app = [ + "mcp>=1.25,<2", + "uvicorn>=0.30", +] # A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP, crops and encodes frames, and # records the display as H.264 with PyAV (computer-recording/v1). computer = [ diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py index 19b8ab07f..45af04bf8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -48,7 +48,6 @@ drag, key_down, key_up, - list_windows, pointer_down, pointer_move, pointer_position, @@ -102,7 +101,7 @@ def __getattr__(name: str) -> Any: "APP_SETTINGS_ENV", "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV", "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press", - "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", "list_windows", + "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", "run_in_session", "write_files", "record", "Recording", "RecordHints", "RecordedFrame", "RecordedSegment", "frames_in_thread", "RecordingManifest", "ManifestSegment", "ComputerError", "InvalidArgument", "OutOfBounds", "StaleFrame", "InputBusy", "PermissionDenied", "Unsupported", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_load.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_load.py new file mode 100644 index 000000000..c31c7f0dc --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_load.py @@ -0,0 +1,22 @@ +"""Import a class from a file of a driver's or an app's folder, as its computer's program starts it.""" +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +def load_class(file: Path, name: str, base: type) -> type: + """The class ``name`` in ``file``, imported as a module of its own with the file's folder importable; exits saying + why when it isn't a subclass of ``base``.""" + if not file.is_file(): + raise SystemExit(f"{file} isn't a file") + sys.path.insert(0, str(file.resolve().parent)) + spec = importlib.util.spec_from_file_location(file.stem, file) + module = importlib.util.module_from_spec(spec) + sys.modules[file.stem] = module + spec.loader.exec_module(module) + cls = getattr(module, name, None) + if not (isinstance(cls, type) and issubclass(cls, base)): + raise SystemExit(f"{file} has no {base.__name__} subclass named {name}") + return cls diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_ready.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_ready.py deleted file mode 100644 index 387c14f9b..000000000 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/_ready.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Run one of an app's ``@ready_check`` methods in the app's own venv, where its dependencies are installed: -``python -m agentenv_protocol.computer._ready `` exits 0 when the method returns true, -1 when it returns false, and 2, saying why on stderr, when it can't run. The class reads its settings as it does when -it serves.""" -from __future__ import annotations - -import asyncio -import importlib.util -import inspect -import sys -from pathlib import Path - - -def main(argv: list[str]) -> int: - folder, file, name, method = argv - try: - sys.path.insert(0, folder) - spec = importlib.util.spec_from_file_location("computer_app_module", Path(folder) / file) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - result = getattr(getattr(module, name)(), method)() - if inspect.isawaitable(result): - result = asyncio.run(result) - except Exception as e: - print(f"{type(e).__name__}: {e}", file=sys.stderr) - return 2 - return 0 if result else 1 - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index b60003392..de3a1e0e5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -2,26 +2,42 @@ computer: the platforms it supports and the permissions it needs (``@computer_app``), how each OS installs it (``linux = Linux(...)``), how it launches (``launch``), its install steps and ready checks, its skill, and its typed ``Settings``. A ``computer_app`` env's put reads that declaration from the class; its data hooks are the base class's. -Its name is its env's ``environment_name``.""" +Its name is its env's ``environment_name``. On the computer the class runs as the app's own server (``serve_app``), +which launches the app, adds its launcher and says whether it's ready (``computer-app/v1``).""" from __future__ import annotations +import asyncio import inspect import json import os +import shlex +import time +from collections.abc import Awaitable +from pathlib import Path from typing import Any, Callable, ClassVar, Literal, Optional from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from ..agent_env_environment import _EXT_ATTR, _OP_ATTR, AgentEnvEnvironment +from ..agent_env_environment import _EXT_ATTR, _OP_ATTR, AgentEnvEnvironment, extension from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, SKILL_EXTENSION_URI +from . import session +from .errors import InvalidArgument #: The environment variable a computer delivers its apps' settings in: JSON of each app's values by its name. APP_SETTINGS_ENV = "AGENTENV_APP_SETTINGS" _INSTALL_STEP_ATTR = "_aee_install_step" _READY_CHECK_ATTR = "_aee_ready_check" _DATA_METHODS = {"reset_data": METHOD_RESET, "add_data": METHOD_ADD, "get_data": METHOD_GET} -#: What a launch's window can be matched by, as a driver's list_windows describes each window. +#: What a launch's window can be matched by, as the session lists each window. WINDOW_KEYS = frozenset({"class", "instance", "title"}) +#: The framework extension an app's own server serves: its launch as params, and its ``launch`` and ``ready`` methods. +COMPUTER_APP = "urn:agentenv:computer-app/v1" +#: Where an app's own server listens, in the app's folder. +SOCKET = Path(".agentenv/server.sock") +#: Where an app's server marks that it launched the app since the computer booted. +LAUNCHED = Path("/run/agentenv/apps") +#: How long one readiness check may take. +CHECK_TIMEOUT_S = 10.0 class Service(BaseModel): @@ -123,13 +139,6 @@ class AppDeclaration(BaseModel): install_steps: list[InstallStep] ready_checks: list[str] - @model_validator(mode="after") - def _venv_for_ready_checks(self) -> AppDeclaration: - if self.ready_checks and not any(install.python for install in self.install.values()): - raise ValueError(f"@ready_check {', '.join(self.ready_checks)} runs in the app's own venv: name its " - "requirements with Linux(python=...)") - return self - def computer_app(*, supports: Optional[list] = None, permissions: Optional[list[str]] = None) -> Callable: """Declare what an app is: the platforms it supports, as a driver names them, and the permissions it needs of the @@ -174,6 +183,67 @@ class Settings(BaseModel): def __init__(self) -> None: values = json.loads(os.environ.get(APP_SETTINGS_ENV) or "{}").get(os.environ.get("ENVIRONMENT_NAME"), {}) self.settings = self.Settings.model_validate(values) + self._computer_app_started_at = time.monotonic() + self._computer_app_launched_at: Optional[float] = None + self._computer_app_launch_failed: Optional[str] = None + + async def _computer_app_start(self) -> None: + """Once the desktop session is up, add the app's menu launcher and launch a boot app, once per boot: a server + that restarts leaves the app it launched be.""" + if self.launch is None: + return + await session.wait_until_up() + session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) + marker = LAUNCHED / Path.cwd().name + if self.launch.at != "boot": + return + if marker.exists(): + self._computer_app_launched_at = time.monotonic() + return + try: + await self._computer_app_launch() + except Exception as e: + self._computer_app_launch_failed = f"{type(e).__name__}: {e}" + return + marker.parent.mkdir(parents=True, exist_ok=True) + marker.touch() + + @extension(COMPUTER_APP, op="launch", description=( + "The app's own server: launch launches the app from its folder, as its launcher and a boot do; ready says " + "whether it's ready, by its launch's ready and its @ready_check methods.")) + async def _computer_app_launch(self) -> dict: + if self.launch is None: + raise InvalidArgument(f"{type(self).__name__} declares no launch") + pid = await session.launch(self.launch.command, Path.cwd(), + {"ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}) + self._computer_app_launched_at, self._computer_app_launch_failed = time.monotonic(), None + return {"pid": pid} + + @extension(COMPUTER_APP, op="ready", method="GET") + async def _computer_app_ready(self) -> dict: + ready = self.launch.ready if self.launch is not None else None + checks: list[tuple[str, Callable[[], Awaitable[Optional[str]]]]] = [] + if ready is not None and ready.window: + checks.append(("window", lambda: _window(self.launch.window))) + if ready is not None and ready.port is not None: + checks.append(("port", lambda: _port(ready.port))) + if ready is not None and ready.command: + checks.append(("command", lambda: _command(ready.command))) + checks += [(f"@ready_check {fn.__name__}", lambda fn=fn: _ready_check(fn, self)) + for fn in _methods(type(self)) if hasattr(fn, _READY_CHECK_ATTR)] + problems = [problem for name, check in checks if (problem := await _bounded(name, check)) is not None] + if self.launch is not None and self._computer_app_launched_at is None: + problems.insert(0, f"its launch failed: {self._computer_app_launch_failed}" + if self._computer_app_launch_failed else "not launched yet") + since = self._computer_app_launched_at or self._computer_app_started_at + return {"ready": not problems, "problems": problems, "launched": self._computer_app_launched_at is not None, + "waited_s": round(time.monotonic() - since, 1), + "timeout_s": ready.timeout_s if ready is not None else Ready.model_fields["timeout_s"].default} + + def extension_params(self, uri: str) -> dict: + if uri == COMPUTER_APP: + return {"launch": self.launch.model_dump(mode="json", exclude_none=True) if self.launch else None} + return super().extension_params(uri) @classmethod def declaration(cls) -> AppDeclaration: @@ -205,3 +275,48 @@ def _methods(cls: type) -> list[Callable]: names = dict.fromkeys(name for klass in reversed(cls.__mro__) for name, member in vars(klass).items() if inspect.isfunction(member)) return [method for name in names if inspect.isfunction(method := getattr(cls, name))] + + +async def _bounded(name: str, check: Callable[[], Awaitable[Optional[str]]]) -> Optional[str]: + """What ``check`` finds wrong, within ``CHECK_TIMEOUT_S``; a check that runs past it or raises says so.""" + try: + return await asyncio.wait_for(check(), CHECK_TIMEOUT_S) + except TimeoutError: + return f"{name} ran past {CHECK_TIMEOUT_S:.0f} s" + except Exception as e: + return f"{name}: {type(e).__name__}: {e}" + + +async def _window(match: dict[str, str]) -> Optional[str]: + if any(all(window.get(key) == value for key, value in match.items()) for window in await session.windows()): + return None + return f"no window with {', '.join(f'{key} {value!r}' for key, value in match.items())}" + + +async def _port(port: int) -> Optional[str]: + try: + _, writer = await asyncio.open_connection("127.0.0.1", port) + except OSError: + return f"nothing answers on port {port}" + writer.close() + return None + + +async def _command(argv: list[str]) -> Optional[str]: + process = await asyncio.create_subprocess_exec(*argv, cwd=Path.cwd(), stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE) + try: + _, err = await process.communicate() + except asyncio.CancelledError: + process.kill() + raise + if process.returncode == 0: + return None + reason = (err.decode(errors="replace").strip().splitlines() or [""])[-1] + return f"{shlex.join(argv)} exited {process.returncode}" + (f": {reason}" if reason else "") + + +async def _ready_check(fn: Callable, app: ComputerAppEnvironment) -> Optional[str]: + """A sync check runs in a thread, so it can't hold up the server.""" + passed = await fn(app) if inspect.iscoroutinefunction(fn) else await asyncio.to_thread(fn, app) + return None if passed else f"@ready_check {fn.__name__} returned {passed!r}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py index 6dc4d4cdf..3a7063362 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py @@ -1,107 +1,47 @@ -"""The apps composed onto a computer, as its driver finds them, and whether each is ready. The composer writes each -app's ``.agentenv/app.json`` into its folder under ``APPS``. An app that starts with the computer counts toward the -computer's readiness: a ``boot`` app, or one with no launch whose ``@ready_check`` methods watch its services.""" +"""The apps composed onto a computer, as its driver finds them, and whether each is ready. Each app's own server +answers on its folder's socket (``serve_app``): its card says how the app launches, and its ``computer-app/v1`` +``ready`` whether it's ready. Every app's server must answer before the computer is ready; an app that starts with the +computer must also be ready, and a ``demand`` app is checked once something launches it.""" from __future__ import annotations -import asyncio -import json -import shlex -import time -from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Optional -from .app import Launch, Ready -from .operations import ReadyCheck +import httpx -if TYPE_CHECKING: - from .runtime import ComputerRuntime +from .. import client +from ..types import RPC_PATH, WELL_KNOWN_PATH +from .app import CHECK_TIMEOUT_S, COMPUTER_APP, SOCKET +from .operations import ReadyCheck #: Where a composed computer installs its apps, a folder each. APPS = Path("/opt/agentenv/apps") -#: An app's description of itself for the driver, in its folder. -APP_FILE = ".agentenv/app.json" -#: How long a readiness command or ``@ready_check`` may run. -CHECK_TIMEOUT_S = 10.0 -#: How long a port gets to answer. -CONNECT_TIMEOUT_S = 1.0 - - -@dataclass(frozen=True) -class InstalledApp: - name: str - folder: Path - module: str - launch: Optional[Launch] - ready_checks: tuple[str, ...] - - @property - def counted(self) -> bool: - """Whether the computer is ready only once this app is, because it starts with the computer.""" - return self.launch.at == "boot" if self.launch is not None else bool(self.ready_checks) - -def installed() -> list[InstalledApp]: - """The apps composed onto this computer, by their folders' names.""" - found = [] - for path in sorted(APPS.glob(f"*/{APP_FILE}")): - data = json.loads(path.read_text()) - found.append(InstalledApp(name=data["name"], folder=path.parent.parent, module=data["module"], - launch=Launch.model_validate(data["launch"]) if data.get("launch") else None, - ready_checks=tuple(data.get("ready_checks", ())))) - return found +def expected() -> list[Path]: + """The folders of the apps composed onto this computer.""" + return sorted(path for path in APPS.iterdir() if path.is_dir()) if APPS.is_dir() else [] -async def check(runtime: ComputerRuntime, app: InstalledApp, since: float) -> ReadyCheck: - """Whether ``app`` is ready, by its launch's ``ready`` and its ``@ready_check`` methods; once past its timeout, the - check says so with what it still waits on.""" - ready = app.launch.ready if app.launch is not None else None - problems = [] - if ready is not None and ready.window: - problems += await _window(runtime, app.launch.window) - if ready is not None and ready.port is not None: - problems += await _port(ready.port) - if ready is not None and ready.command: - problems += await _run(runtime, app, ready.command, shlex.join(ready.command)) - for method in app.ready_checks: - file, _, name = app.module.partition(":") - argv = [str(app.folder / "venv/bin/python"), "-m", "agentenv_protocol.computer._ready", str(app.folder), file, - name, method] - problems += await _run(runtime, app, argv, f"@ready_check {method}") - if not problems: - return ReadyCheck(f"app {app.name}", True, "ready") - waited = time.monotonic() - since - limit = ready.timeout_s if ready is not None else Ready.model_fields["timeout_s"].default - when = f"not ready after {limit:.0f} s" if waited > limit else f"not ready yet, {waited:.0f} s in" - return ReadyCheck(f"app {app.name}", False, f"{when}: {'; '.join(problems)}") - -async def _window(runtime: ComputerRuntime, match: dict[str, str]) -> list[str]: - if not runtime.has("list_windows"): - return ["this driver can't list windows"] - windows = await runtime.call("list_windows") - if any(all(window.get(key) == value for key, value in match.items()) for window in windows): - return [] - return [f"no window with {', '.join(f'{key} {value!r}' for key, value in match.items())}"] - - -async def _port(port: int) -> list[str]: +async def check(folder: Path) -> ReadyCheck: + """Whether the app in ``folder`` is ready, as its own server says; named by its card once the server answers.""" + name = folder.name try: - _, writer = await asyncio.wait_for(asyncio.open_connection("127.0.0.1", port), CONNECT_TIMEOUT_S) - except (OSError, TimeoutError): - return [f"nothing answers on port {port}"] - writer.close() - return [] - - -async def _run(runtime: ComputerRuntime, app: InstalledApp, argv: list[str], what: str) -> list[str]: - if not runtime.has("run_in_session"): - return [f"this driver can't run {what}"] - result = await runtime.call("run_in_session", argv, str(app.folder), {"ENVIRONMENT_NAME": app.name}, - CHECK_TIMEOUT_S) - if result.exit_code == 0: - return [] - if result.timed_out: - return [f"{what} ran past {CHECK_TIMEOUT_S:.0f} s"] - reason = (result.stderr.decode(errors="replace").strip().splitlines() or [""])[-1] - return [f"{what} exited {result.exit_code}" + (f": {reason}" if reason else "")] + async with _client(folder / SOCKET) as http: + card = (await http.get(WELL_KNOWN_PATH)).raise_for_status().json() + name = card["name"] + launch = client.extension_params(card, COMPUTER_APP).get("launch") + answer = (await http.get(f"{RPC_PATH}/ext/computer-app/ready")).raise_for_status().json() + except (httpx.HTTPError, OSError): + return ReadyCheck(f"app {name}", False, "its server isn't answering yet") + if launch is not None and launch["at"] == "demand" and not answer["launched"]: + return ReadyCheck(f"app {name}", True, "its server answers; it launches on demand") + if answer["ready"]: + return ReadyCheck(f"app {name}", True, "ready") + when = (f"not ready after {answer['timeout_s']:.0f} s" if answer["waited_s"] > answer["timeout_s"] + else f"not ready yet, {answer['waited_s']:.0f} s in") + return ReadyCheck(f"app {name}", False, f"{when}: {'; '.join(answer['problems'])}") + + +def _client(socket: Path) -> httpx.AsyncClient: + return httpx.AsyncClient(transport=httpx.AsyncHTTPTransport(uds=str(socket)), base_url="http://app", + timeout=CHECK_TIMEOUT_S * 4) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index f58543545..172b57703 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -150,10 +150,6 @@ async def _computer_linux_run_in_session(self, argv: list, cwd: str | None, env: async def _computer_linux_write_files(self, files: dict) -> None: await asyncio.to_thread(linux.write_files, files) - @default_for("list_windows", os="linux") - async def _computer_linux_list_windows(self) -> list[dict[str, str]]: - return await linux.list_windows() - def computer_driver(name: str | None = None, *, supports: list | None = None) -> Callable: """Name a driver, its card's name unless ENVIRONMENT_NAME gives one, and declare the platforms it supports.""" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/launch.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/launch.py new file mode 100644 index 000000000..776044941 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/launch.py @@ -0,0 +1,23 @@ +"""Launch a computer app through its own server, as its menu launcher does: +``python -m agentenv_protocol.computer.launch `` asks the server on the folder's socket to launch it, so +the server starts every copy of the app.""" +from __future__ import annotations + +import sys +from pathlib import Path + +import httpx + +from ..types import RPC_PATH +from .app import SOCKET + + +def main(argv: list[str]) -> None: + if len(argv) != 1: + raise SystemExit("usage: python -m agentenv_protocol.computer.launch ") + with httpx.Client(transport=httpx.HTTPTransport(uds=str(Path(argv[0]) / SOCKET)), base_url="http://app") as http: + http.post(f"{RPC_PATH}/ext/computer-app/launch", json={}).raise_for_status() + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py index a45d55109..021fbc346 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/linux.py @@ -5,7 +5,6 @@ import asyncio import contextlib import os -import re import signal import tempfile from pathlib import Path @@ -82,28 +81,6 @@ async def read(self, stream: asyncio.StreamReader) -> None: self.kept += chunk[:max(room, 0)] -async def list_windows() -> list[dict[str, str]]: - """The window manager's top-level windows, each as its WM_CLASS ``instance`` and ``class`` and its ``title``, read - with ``xprop``; a window that closes while it's read is left out.""" - listed = await _xprop("-root", "_NET_CLIENT_LIST") - windows = [] - for window in re.findall(r"0x[0-9a-f]+", listed.partition("#")[2]): - props = await _xprop("-id", window, "WM_CLASS", "_NET_WM_NAME") - names = dict(re.findall(r'^(WM_CLASS|_NET_WM_NAME)[^=]*= (.*)$', props, re.MULTILINE)) - parts = re.findall(r'"((?:[^"\\]|\\.)*)"', names.get("WM_CLASS", "")) - if len(parts) == 2: - title = re.findall(r'"((?:[^"\\]|\\.)*)"', names.get("_NET_WM_NAME", "")) - windows.append({"instance": parts[0], "class": parts[1], "title": title[0] if title else ""}) - return windows - - -async def _xprop(*args: str) -> str: - process = await asyncio.create_subprocess_exec("xprop", *args, stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.DEVNULL) - out, _ = await process.communicate() - return out.decode(errors="replace") if process.returncode == 0 else "" - - def write_files(files: dict[str, bytes]) -> None: """Write each file under the home atomically: a temporary file beside it, then a rename.""" home = Path.home().resolve() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py index c742c4430..1e9936ec1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/operations.py @@ -11,7 +11,6 @@ pointer_move(x, y), pointer_down(button), pointer_up(button), pointer_position() -> (x, y) key_down(keys), key_up(keys) run_in_session(argv, cwd, env, timeout_s) -> ExecResult write_files({path: bytes}) - list_windows() -> [{"class": ..., "instance": ..., "title": ...}, ...], the desktop's top-level windows record(hints) -> an async iterator of RecordedFrame or RecordedSegment, until it is closed """ from __future__ import annotations @@ -27,7 +26,7 @@ "pointer": ("pointer_move", "pointer_down", "pointer_up", "pointer_position"), "keyboard": ("key_down", "key_up"), } -OPTIONAL = ("set_display", "run_in_session", "write_files", "record", "list_windows") +OPTIONAL = ("set_display", "run_in_session", "write_files", "record") INPUT = frozenset({"click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", "key_down", "key_up"}) @@ -117,7 +116,6 @@ def deco(f: Callable) -> Callable: run_in_session = _marker("run_in_session") write_files = _marker("write_files") record = _marker("record") -list_windows = _marker("list_windows") def default_for(operation: str, **predicates: Any) -> Callable: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index 8ee333c47..443a6a8cd 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -5,7 +5,6 @@ import asyncio import contextlib -import time from collections import OrderedDict from collections.abc import AsyncIterator from typing import TYPE_CHECKING, Any, Awaitable, Callable @@ -42,8 +41,7 @@ def __init__(self, driver: ComputerDriverEnvironment) -> None: self._commands_running: dict[str, asyncio.Task] = {} self._commands_finished: OrderedDict = OrderedDict() self.recordings = Recordings(self) - self.apps = apps.installed() - self.started = time.monotonic() + self.apps = apps.expected() def has(self, capability: str) -> bool: if capability in GROUPS: @@ -179,7 +177,7 @@ async def ready_checks(self) -> list[ReadyCheck]: checks += list(await self.driver.ready_checks()) except Exception as e: checks.append(ReadyCheck("driver", False, f"{type(e).__name__}: {e}")) - checks += [await apps.check(self, app, self.started) for app in self.apps if app.counted] + checks += [await apps.check(folder) for folder in self.apps] if all(c.ok for c in checks): self.ready = True return checks diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve.py index 1b9ed64a3..81df184c8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve.py @@ -3,10 +3,10 @@ ``MCP_PORT``. The file's folder is importable, so it may import its own modules.""" from __future__ import annotations -import importlib.util import sys from pathlib import Path +from ._load import load_class from .driver import ComputerDriverEnvironment USAGE = "usage: python -m agentenv_protocol.computer.serve :" @@ -21,17 +21,7 @@ def main(argv: list[str]) -> None: def driver_class(file: Path, name: str) -> type[ComputerDriverEnvironment]: """The class ``name`` in ``file``, imported as a module of its own; exits saying why when it isn't a driver.""" - if not file.is_file(): - raise SystemExit(f"{file} isn't a file") - sys.path.insert(0, str(file.resolve().parent)) - spec = importlib.util.spec_from_file_location(file.stem, file) - module = importlib.util.module_from_spec(spec) - sys.modules[file.stem] = module - spec.loader.exec_module(module) - cls = getattr(module, name, None) - if not (isinstance(cls, type) and issubclass(cls, ComputerDriverEnvironment)): - raise SystemExit(f"{file} has no ComputerDriverEnvironment subclass named {name}") - return cls + return load_class(file, name, ComputerDriverEnvironment) if __name__ == "__main__": diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py new file mode 100644 index 000000000..a0eea2c82 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py @@ -0,0 +1,41 @@ +"""Serve a computer app's class from its file, as its computer's program starts it: +``python -m agentenv_protocol.computer.serve_app :``, run in the app's folder, serves the class's card and +``computer-app/v1`` on the Unix socket ``.agentenv/server.sock`` there. Once it serves, the app's server adds the app's +launcher and launches a boot app (``ComputerAppEnvironment._computer_app_start``). Needs the ``app`` extra.""" +from __future__ import annotations + +import asyncio +import contextlib +import sys +from pathlib import Path + +import uvicorn + +from ._load import load_class +from .app import SOCKET, ComputerAppEnvironment + +USAGE = "usage: python -m agentenv_protocol.computer.serve_app :" + + +def main(argv: list[str]) -> None: + if len(argv) != 1 or ":" not in argv[0]: + raise SystemExit(USAGE) + file, _, name = argv[0].partition(":") + app = load_class(Path(file), name, ComputerAppEnvironment)() + asgi = app.create_app().streamable_http_app() + serving = asgi.router.lifespan_context + + @contextlib.asynccontextmanager + async def lifespan(scope): + async with serving(scope): + started = asyncio.ensure_future(app._computer_app_start()) + yield + started.cancel() + + asgi.router.lifespan_context = lifespan + SOCKET.unlink(missing_ok=True) + uvicorn.run(asgi, uds=str(SOCKET), log_level="warning") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py new file mode 100644 index 000000000..71baf11f4 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py @@ -0,0 +1,64 @@ +"""What a computer app's own server does in the desktop session, the Linux way: wait for the session, launch the app +into it, add its menu launcher and list the session's windows. Another OS type brings its own.""" +from __future__ import annotations + +import asyncio +import os +import re +from pathlib import Path + +#: How often a starting session is looked for. +POLL_S = 0.2 +#: The processes launched into the session, kept until they exit so each is reaped. +_running: set[asyncio.Task] = set() + + +async def wait_until_up() -> None: + """Until the X display's socket is there.""" + display = os.environ["DISPLAY"].lstrip(":").split(".")[0] + while not Path(f"/tmp/.X11-unix/X{display}").exists(): + await asyncio.sleep(POLL_S) + + +async def launch(argv: list[str], cwd: Path, env: dict[str, str]) -> int: + """Start ``argv`` in ``cwd``, in a session of its own so it outlives the server, with the server's environment and + ``env``; its pid.""" + process = await asyncio.create_subprocess_exec( + *argv, cwd=cwd, env={**os.environ, **env}, start_new_session=True, stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL) + reaped = asyncio.ensure_future(process.wait()) + _running.add(reaped) + reaped.add_done_callback(_running.discard) + return process.pid + + +def add_launcher(name: str, folder: Path) -> Path: + """The desktop entry that launches the app ``name`` in ``folder`` through its own server, written where the + session user's menu finds it.""" + entry = Path.home() / ".local/share/applications" / f"agentenv-app-{folder.name}.desktop" + entry.parent.mkdir(parents=True, exist_ok=True) + entry.write_text(f"[Desktop Entry]\nType=Application\nName={name}\n" + f"Exec={folder}/venv/bin/python -m agentenv_protocol.computer.launch {folder}\nTerminal=false\n") + return entry + + +async def windows() -> list[dict[str, str]]: + """The window manager's top-level windows, each as its WM_CLASS ``instance`` and ``class`` and its ``title``, read + with ``xprop``; a window that closes while it's read is left out.""" + listed = await _xprop("-root", "_NET_CLIENT_LIST") + found = [] + for window in re.findall(r"0x[0-9a-f]+", listed.partition("#")[2]): + props = await _xprop("-id", window, "WM_CLASS", "_NET_WM_NAME") + names = dict(re.findall(r'^(WM_CLASS|_NET_WM_NAME)[^=]*= (.*)$', props, re.MULTILINE)) + parts = re.findall(r'"((?:[^"\\]|\\.)*)"', names.get("WM_CLASS", "")) + if len(parts) == 2: + title = re.findall(r'"((?:[^"\\]|\\.)*)"', names.get("_NET_WM_NAME", "")) + found.append({"instance": parts[0], "class": parts[1], "title": title[0] if title else ""}) + return found + + +async def _xprop(*args: str) -> str: + process = await asyncio.create_subprocess_exec("xprop", *args, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL) + out, _ = await process.communicate() + return out.decode(errors="replace") if process.returncode == 0 else "" diff --git a/packages/agentenv-protocol/tests/test_computer_app_server.py b/packages/agentenv-protocol/tests/test_computer_app_server.py new file mode 100644 index 000000000..f62c66e98 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_app_server.py @@ -0,0 +1,291 @@ +"""A computer app's own server: it adds the app's launcher, launches a boot app once per boot and says whether the app +is ready (``computer-app/v1``); its computer's driver finds it on the socket in the app's folder and asks it.""" +from __future__ import annotations + +import asyncio +import contextlib +import os +import shlex +import socket +import sys +import tempfile +import time +from pathlib import Path + +import pytest + +pytest.importorskip("mcp.server.fastmcp") +uvicorn = pytest.importorskip("uvicorn") + +import httpx # noqa: E402 + +import agentenv_protocol.computer.launch as launcher # noqa: E402 +from agentenv_protocol import client # noqa: E402 +from agentenv_protocol.computer import ComputerAppEnvironment, Launch, ReadyCheck, Ready, ready_check # noqa: E402 +from agentenv_protocol.computer import app as app_module, apps, serve_app, session # noqa: E402 +from agentenv_protocol.computer.app import COMPUTER_APP, SOCKET # noqa: E402 + +READY = "/agentenv/ext/computer-app/ready" + + +class Mail(ComputerAppEnvironment): + launch = Launch(command=["mail"], window={"class": "Mail"}, ready=Ready(window=True, timeout_s=30)) + synced = False + + @ready_check + def inbox_synced(self): + return self.synced + + +class Music(ComputerAppEnvironment): + launch = Launch(at="demand", command=["music"], window={"class": "Music"}, ready=Ready(window=True)) + + +class Sync(ComputerAppEnvironment): + @ready_check + async def caught_up(self): + return True + + +@pytest.fixture +def desktop(monkeypatch): + """A session with no windows yet, recording what's launched into it.""" + shown, launched = [], [] + + async def windows(): + return shown + + async def launch(argv, cwd, env): + launched.append((argv, cwd, env)) + return 4242 + + async def up(): + pass + + monkeypatch.setattr(session, "windows", windows) + monkeypatch.setattr(session, "launch", launch) + monkeypatch.setattr(session, "wait_until_up", up) + monkeypatch.setenv("ENVIRONMENT_NAME", "mail") + return shown, launched + + +@pytest.mark.asyncio +async def test_an_apps_server_is_ready_once_it_launched_the_app_and_each_check_passes(desktop): + shown, launched = desktop + mail = Mail() + async with _http(mail) as http: + card = (await http.get("/.well-known/agent-env.json")).json() + before = (await http.get(READY)).json() + assert (await http.post("/agentenv/ext/computer-app/launch", json={})).json() == {"pid": 4242} + shown.append({"instance": "mail", "class": "Mail", "title": "Inbox"}) + unsynced = (await http.get(READY)).json() + mail.synced = True + after = (await http.get(READY)).json() + + assert client.extension_params(card, COMPUTER_APP)["launch"] == { + "at": "boot", "command": ["mail"], "window": {"class": "Mail"}, "ready": {"window": True, "timeout_s": 30.0}} + assert {key: value for key, value in before.items() if key != "waited_s"} == { + "ready": False, "launched": False, "timeout_s": 30.0, + "problems": ["not launched yet", "no window with class 'Mail'", "@ready_check inbox_synced returned False"]} + assert launched == [(["mail"], Path.cwd(), {"ENVIRONMENT_NAME": "mail"})] + assert unsynced["problems"] == ["@ready_check inbox_synced returned False"] and unsynced["launched"] is True + assert after["ready"] is True and after["problems"] == [] + + +@pytest.mark.asyncio +async def test_a_check_that_fails_hangs_or_raises_says_so_without_holding_up_the_server(desktop, monkeypatch): + monkeypatch.setattr(app_module, "CHECK_TIMEOUT_S", 1.0) + with socket.socket() as closed: + closed.bind(("127.0.0.1", 0)) + port = closed.getsockname()[1] + exits = [sys.executable, "-c", "import sys; sys.exit('no database yet')"] + + class Slow(ComputerAppEnvironment): + launch = Launch(command=["slow"], ready=Ready(port=port, command=exits)) + + @ready_check + def stuck(self): + time.sleep(3) + return True + + @ready_check + async def broken(self): + raise RuntimeError("mailbox locked") + + started = time.monotonic() + answer = await Slow()._computer_app_ready() + + assert answer["problems"] == ["not launched yet", f"nothing answers on port {port}", + f"{shlex.join(exits)} exited 1: no database yet", "@ready_check stuck ran past 1 s", + "@ready_check broken: RuntimeError: mailbox locked"] + assert time.monotonic() - started < 2.5 + + +@pytest.mark.asyncio +async def test_a_boot_app_is_launched_once_per_boot_with_its_launcher_added_each_start(desktop, tmp_path, monkeypatch): + _, launched = desktop + launchers = [] + monkeypatch.setattr(app_module, "LAUNCHED", tmp_path / "launched") + monkeypatch.setattr(session, "add_launcher", lambda name, folder: launchers.append((name, folder))) + monkeypatch.chdir(tmp_path) + + first, restarted = Mail(), Mail() + await first._computer_app_start() + await restarted._computer_app_start() + + assert len(launched) == 1 and launchers == [("mail", tmp_path)] * 2 + assert (tmp_path / "launched" / tmp_path.name).exists() + assert (await restarted._computer_app_ready())["launched"] is True + + +@pytest.mark.asyncio +async def test_a_boot_app_whose_launch_fails_says_why_and_is_launched_again_by_its_next_server(desktop, tmp_path, + monkeypatch): + async def missing(argv, cwd, env): + raise FileNotFoundError(f"No such file or directory: {argv[0]!r}") + + monkeypatch.setattr(app_module, "LAUNCHED", tmp_path / "launched") + monkeypatch.setattr(session, "add_launcher", lambda name, folder: None) + monkeypatch.setattr(session, "launch", missing) + mail = Mail() + await mail._computer_app_start() + + assert (await mail._computer_app_ready())["problems"][0] == ( + "its launch failed: FileNotFoundError: No such file or directory: 'mail'") + assert not (tmp_path / "launched").exists() + + +@pytest.mark.asyncio +async def test_the_driver_asks_each_apps_server_on_the_socket_in_its_folder(desktop, monkeypatch): + shown, launched = desktop + with tempfile.TemporaryDirectory(dir="/tmp", prefix="apps-") as root: + monkeypatch.setattr(apps, "APPS", Path(root)) + (Path(root) / "gone").mkdir() + async with contextlib.AsyncExitStack() as stack: + served = {name: await stack.enter_async_context(_serving(cls(), Path(root) / name, monkeypatch)) + for name, cls in [("mail", Mail), ("music", Music), ("sync", Sync)]} + first = [await apps.check(folder) for folder in apps.expected()] + served["mail"]._computer_app_started_at -= 120 + late = await apps.check(Path(root) / "mail") + monkeypatch.setenv("ENVIRONMENT_NAME", "music") + await asyncio.to_thread(launcher.main, [str(Path(root) / "music")]) + opened = await apps.check(Path(root) / "music") + shown.append({"instance": "music", "class": "Music", "title": "Library"}) + playing = await apps.check(Path(root) / "music") + + assert first == [ + ReadyCheck("app gone", False, "its server isn't answering yet"), + ReadyCheck("app mail", False, "not ready yet, 0 s in: not launched yet; no window with class 'Mail'; " + "@ready_check inbox_synced returned False"), + ReadyCheck("app music", True, "its server answers; it launches on demand"), + ReadyCheck("app sync", True, "ready"), + ] + assert late.detail.startswith("not ready after 30 s: not launched yet") + assert launched == [(["music"], Path.cwd(), {"ENVIRONMENT_NAME": "music"})] + assert opened == ReadyCheck("app music", False, "not ready yet, 0 s in: no window with class 'Music'") + assert playing == ReadyCheck("app music", True, "ready") + + +@pytest.mark.asyncio +async def test_serving_an_app_starts_it_once_the_server_serves_on_the_socket_in_its_folder(tmp_path, monkeypatch): + (tmp_path / "app.py").write_text("from agentenv_protocol.computer import ComputerAppEnvironment\n\n\n" + "class Notes(ComputerAppEnvironment):\n pass\n") + (tmp_path / SOCKET).parent.mkdir() + (tmp_path / SOCKET).write_text("left by the last server") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "path", list(sys.path)) + monkeypatch.setattr(sys, "modules", dict(sys.modules)) + started, ran = [], [] + monkeypatch.setattr(ComputerAppEnvironment, "_computer_app_start", lambda self: _record(started, type(self))) + monkeypatch.setattr(serve_app.uvicorn, "run", lambda asgi, uds, log_level: ran.append((asgi, uds))) + + serve_app.main(["app.py:Notes"]) + (asgi, uds), = ran + async with asgi.router.lifespan_context(asgi): + await asyncio.sleep(0) + + assert uds == str(SOCKET) and not (tmp_path / SOCKET).exists() + assert [cls.__name__ for cls in started] == ["Notes"] + + +@pytest.mark.parametrize("argv, message", [ + ([], serve_app.USAGE), (["app.py"], serve_app.USAGE), (["missing.py:Notes"], "missing.py isn't a file"), + (["app.py:Helper"], "has no ComputerAppEnvironment subclass named Helper"), +], ids=["nothing", "no-class", "no-file", "not-an-app"]) +def test_what_isnt_an_app_class_exits_saying_why(tmp_path, monkeypatch, argv, message): + (tmp_path / "app.py").write_text("class Helper:\n pass\n") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "path", list(sys.path)) + monkeypatch.setattr(sys, "modules", dict(sys.modules)) + + with pytest.raises(SystemExit) as stopped: + serve_app.main(argv) + + assert message in str(stopped.value) + + +@pytest.mark.asyncio +async def test_the_session_launches_an_app_in_a_session_of_its_own_from_its_folder(tmp_path, monkeypatch): + monkeypatch.setenv("DISPLAY", ":99") + script = "import os; open('seen', 'w').write(f\"{os.getcwd()} {os.environ['ENVIRONMENT_NAME']} " \ + "{os.environ['DISPLAY']} {os.getsid(0) == os.getpid()}\")" + + await session.launch([sys.executable, "-c", script], tmp_path, {"ENVIRONMENT_NAME": "notes"}) + await asyncio.gather(*session._running) + + assert (tmp_path / "seen").read_text() == f"{os.path.realpath(tmp_path)} notes :99 True" + + +def test_the_launcher_runs_the_app_through_its_own_server(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + folder = Path("/opt/agentenv/apps/notes") + + entry = session.add_launcher("Notes", folder) + + assert entry == tmp_path / ".local/share/applications/agentenv-app-notes.desktop" + assert entry.read_text() == ("[Desktop Entry]\nType=Application\nName=Notes\nExec=/opt/agentenv/apps/notes/venv/bin/" + "python -m agentenv_protocol.computer.launch /opt/agentenv/apps/notes\nTerminal=false\n") + + +@pytest.mark.asyncio +async def test_the_sessions_windows_are_listed_by_class_instance_and_title(monkeypatch): + replies = {("-root", "_NET_CLIENT_LIST"): "_NET_CLIENT_LIST(WINDOW): window id # 0x1e00004, 0x2000003\n", + ("-id", "0x1e00004", "WM_CLASS", "_NET_WM_NAME"): + '_NET_WM_NAME(UTF8_STRING) = "Inbox \\"work\\""\nWM_CLASS(STRING) = "mail", "Mail"\n', + ("-id", "0x2000003", "WM_CLASS", "_NET_WM_NAME"): ""} + + async def xprop(*args): + return replies[args] + + monkeypatch.setattr(session, "_xprop", xprop) + assert await session.windows() == [{"instance": "mail", "class": "Mail", "title": 'Inbox \\"work\\"'}] + + +@contextlib.asynccontextmanager +async def _http(app: ComputerAppEnvironment): + asgi = app.create_app().streamable_http_app() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=asgi), base_url="http://t") as http: + yield http + + +@contextlib.asynccontextmanager +async def _serving(app: ComputerAppEnvironment, folder: Path, monkeypatch): + """``app`` served on the socket in ``folder``, named for the folder, as its computer's program serves it.""" + (folder / SOCKET).parent.mkdir(parents=True) + monkeypatch.setenv("ENVIRONMENT_NAME", folder.name) + server = uvicorn.Server(uvicorn.Config(app.create_app().streamable_http_app(), uds=str(folder / SOCKET), + log_level="warning", lifespan="off", ws="none")) + serving = asyncio.create_task(server.serve()) + while not server.started: + if serving.done(): + raise RuntimeError(f"{folder.name}'s server stopped before it served: {serving.exception()!r}") + await asyncio.sleep(0.01) + try: + yield app + finally: + server.should_exit = True + await serving + + +async def _record(started: list, cls: type) -> None: + started.append(cls) diff --git a/packages/agentenv-protocol/tests/test_computer_declare.py b/packages/agentenv-protocol/tests/test_computer_declare.py index f01bbb5bc..9aad8de02 100644 --- a/packages/agentenv-protocol/tests/test_computer_declare.py +++ b/packages/agentenv-protocol/tests/test_computer_declare.py @@ -16,7 +16,6 @@ from agentenv_protocol.computer import AppDeclaration # noqa: E402 from agentenv_protocol.computer._declare import _Stub, _StubModule, declare, main # noqa: E402 -from agentenv_protocol.computer._ready import main as ready # noqa: E402 from agentenv_protocol.computer.platform import Platform # noqa: E402 DRIVER = '''import not_installed_here @@ -196,8 +195,6 @@ def test_an_app_declaring_its_skill_twice_is_refused(folder): ('launch = Launch(command=["mail"], ready=Ready(timeout_s=5))', "Ready waits on nothing: set window, port or command"), ('launch = Launch(command=["mail"], ready=Ready(window=True))', "ready waits on a window: name it with Launch(window="), ('launch = Launch(command=["mail"], window={"role": "main"})', "window matches by class, instance, title, not"), - ('linux = Linux(services=[Service(name="imap", command="bin/imap")])', - "@ready_check inbox_open runs in the app's own venv: name its requirements with Linux(python=...)"), ]) def test_an_app_whose_launch_or_readiness_cant_be_checked_is_refused_saying_why(folder, capsys, change, problem): (folder / "app.py").write_text(APP.format(supports=LINUX) + f""" @@ -211,24 +208,6 @@ class Changed(Mail): assert problem in capsys.readouterr().err.splitlines()[-1] -@pytest.mark.parametrize("body, code", [("return True", 0), ("return False", 1), - ("raise RuntimeError('mailbox locked')", 2)]) -def test_a_ready_check_runs_from_the_apps_folder_and_exits_by_what_it_returns(tmp_path, monkeypatch, capsys, body, - code): - (tmp_path / "app.py").write_text(f"""from agentenv_protocol.computer import ComputerAppEnvironment, ready_check - - -class Mail(ComputerAppEnvironment): - @ready_check - async def inbox_open(self): - {body} -""") - monkeypatch.setenv("ENVIRONMENT_NAME", "mail") - - assert ready([str(tmp_path), "app.py", "Mail", "inbox_open"]) == code - assert capsys.readouterr().err == ("RuntimeError: mailbox locked\n" if code == 2 else "") - - def test_an_app_reads_the_settings_its_computer_gives_it_by_its_name(monkeypatch): from pydantic import BaseModel diff --git a/packages/agentenv-protocol/tests/test_computer_driver.py b/packages/agentenv-protocol/tests/test_computer_driver.py index b326132f3..c6ceec511 100644 --- a/packages/agentenv-protocol/tests/test_computer_driver.py +++ b/packages/agentenv-protocol/tests/test_computer_driver.py @@ -10,20 +10,17 @@ import asyncio # noqa: E402 import base64 # noqa: E402 import io # noqa: E402 -import json # noqa: E402 from computer_fake import LINUX, FakeDriver, FakeScreen, MinimalDriver, http_client, mcp_session # noqa: E402 from mcp.server.fastmcp import FastMCP # noqa: E402 from mcp.shared.memory import create_connected_server_and_client_session # noqa: E402 from PIL import Image as PILImage # noqa: E402 -from agentenv_protocol.computer import apps, linux # noqa: E402 from agentenv_protocol.computer import ( # noqa: E402 SETTINGS_ENV, ComputerDriverEnvironment, ComputerSettings, Display, - ExecResult, Image, InvalidArgument, OutOfBounds, @@ -34,7 +31,6 @@ computer_driver, display, drag, - list_windows, pointer_move, press, run_in_session, @@ -282,68 +278,6 @@ async def ready_checks(self): assert raised["checks"][-1] == {"name": "driver", "ok": False, "detail": "RuntimeError: probe crashed"} -@pytest.mark.asyncio -async def test_each_app_that_starts_with_the_computer_counts_until_it_is_ready(tmp_path, monkeypatch): - listener = await asyncio.start_server(lambda reader, writer: writer.close(), "127.0.0.1", 0) - port = listener.sockets[0].getsockname()[1] - for name, launch, checks in [ - ("mail", {"at": "boot", "command": ["mail"], "window": {"class": "Mail"}, "ready": {"window": True}}, []), - ("notes", {"at": "boot", "command": ["notes"], "ready": {"port": port}}, []), - ("music", {"at": "demand", "command": ["music"], "window": {"class": "Music"}, "ready": {"window": True}}, []), - ("sync", None, ["synced"]), - ]: - (tmp_path / name / ".agentenv").mkdir(parents=True) - (tmp_path / name / ".agentenv/app.json").write_text(json.dumps( - {"name": name, "module": "app.py:App", "launch": launch, "ready_checks": checks})) - monkeypatch.setattr(apps, "APPS", tmp_path) - - class Desktop(FakeDriver): - windows, synced, ran = [], False, [] - - @list_windows - async def listed(self): - return self.windows - - @run_in_session - async def run(self, argv, cwd, env, timeout_s): - self.ran.append((argv, cwd, env)) - return ExecResult(0 if self.synced else 1, stderr=b"no sync yet\n") - - driver = Desktop() - async with http_client(driver) as http: - before = (await http.get("/agentenv/ext/computer-ready/status")).json() - driver._computer.started -= 120 - late = (await http.get("/agentenv/ext/computer-ready/status")).json() - driver.windows, driver.synced = [{"instance": "mail", "class": "Mail", "title": "Inbox"}], True - after = (await http.get("/agentenv/ext/computer-ready/status")).json() - listener.close() - - assert before["ready"] is False and [c for c in before["checks"] if c["name"].startswith("app ")] == [ - {"name": "app mail", "ok": False, "detail": "not ready yet, 0 s in: no window with class 'Mail'"}, - {"name": "app notes", "ok": True, "detail": "ready"}, - {"name": "app sync", "ok": False, "detail": "not ready yet, 0 s in: @ready_check synced exited 1: no sync yet"}, - ] - assert late["checks"][-1]["detail"].startswith("not ready after 60 s: @ready_check synced exited 1") - assert after["ready"] is True - assert driver.ran[0] == ([str(tmp_path / "sync/venv/bin/python"), "-m", "agentenv_protocol.computer._ready", - str(tmp_path / "sync"), "app.py", "App", "synced"], str(tmp_path / "sync"), - {"ENVIRONMENT_NAME": "sync"}) - - -@pytest.mark.asyncio -async def test_the_linux_default_lists_windows_by_their_class_instance_and_title(monkeypatch): - replies = {("-root", "_NET_CLIENT_LIST"): "_NET_CLIENT_LIST(WINDOW): window id # 0x1e00004, 0x2000003\n", - ("-id", "0x1e00004", "WM_CLASS", "_NET_WM_NAME"): - '_NET_WM_NAME(UTF8_STRING) = "Inbox \\"work\\""\nWM_CLASS(STRING) = "mail", "Mail"\n', - ("-id", "0x2000003", "WM_CLASS", "_NET_WM_NAME"): ""} - - async def xprop(*args): - return replies[args] - - monkeypatch.setattr(linux, "_xprop", xprop) - assert await linux.list_windows() == [{"instance": "mail", "class": "Mail", "title": 'Inbox \\"work\\"'}] - - @pytest.mark.asyncio @pytest.mark.parametrize("captured, error", [ (lambda screen: "pixels", "@capture returned str, not an Image"), diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 3808fcfc7..9c260dae0 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -26,7 +26,9 @@ machine's Docker can build, then boots the one this machine runs natively and pr is that env, and `agent-env run desktop` deploys it. A computer app env (`type = "computer_app"`) deployed alone is composed onto it the same way, with the app's files -under `/opt/agentenv/apps//` and its services as programs beside the driver's, run as `user` like everything else. +under `/opt/agentenv/apps//`, a venv there with the protocol's `app` extra, and the app's own server and its +services as programs beside the driver's, run as `user` like everything else. A deploy waits for the computer to be +ready up to 180 s, or 30 s more than the slowest `boot` app's own `timeout_s`. Everything in a computer runs as that one user, so any code in it, an app's, a driver's, a `run_code` script or a command run through `computer-exec`, can read the environment of any other process there, settings included. A secret diff --git a/src/agent_env/drivers/envs/reference/driver.py b/src/agent_env/drivers/envs/reference/driver.py index 2b2cfd628..a8de1b3b0 100644 --- a/src/agent_env/drivers/envs/reference/driver.py +++ b/src/agent_env/drivers/envs/reference/driver.py @@ -18,7 +18,7 @@ import mss from Xlib import X, XK from Xlib import display as xdisplay -from Xlib.error import BadWindow, ConnectionClosedError, DisplayConnectionError +from Xlib.error import ConnectionClosedError, DisplayConnectionError from Xlib.ext import xtest from agentenv_protocol.computer import ( @@ -40,7 +40,6 @@ frames_in_thread, key_down, key_up, - list_windows, pointer_down, pointer_move, pointer_position, @@ -186,10 +185,6 @@ async def down(self, button) -> None: async def up(self, button) -> None: await self._send(lambda c: c.button(button, False)) - @list_windows - async def windows(self) -> list[dict[str, str]]: - return await self._read(_windows) - @pointer_position async def where(self) -> tuple[int, int]: def read(c): @@ -272,24 +267,6 @@ def _connected(self) -> _Connection: raise DeviceUnavailable(f"the X server is unavailable: {type(e).__name__}") from None -def _windows(c: _Connection) -> list[dict[str, str]]: - """The window manager's top-level windows by WM_CLASS and title; a window that closes while it's read is left - out.""" - listed = c.d.screen().root.get_full_property(c.d.intern_atom("_NET_CLIENT_LIST"), X.AnyPropertyType) - windows = [] - for wid in listed.value if listed is not None else (): - window = c.d.create_resource_object("window", wid) - try: - names = window.get_wm_class() - title = window.get_full_property(c.d.intern_atom("_NET_WM_NAME"), c.d.intern_atom("UTF8_STRING")) - except BadWindow: - continue - if names: - windows.append({"instance": names[0], "class": names[1], - "title": title.value.decode(errors="replace") if title is not None else ""}) - return windows - - class _Connection: """One connection to the X server, with XTEST input on it. A character the keymap lacks is typed through a spare keycode mapped to it, kept for that character and reused; once all are taken, the least recently used one that diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index 6d622c5fc..ed3e392c8 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -83,7 +83,7 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact app = cls(id=id, version=None, environment_name=environment_name, module=module, files=files, declaration=declaration, computer=computer, driver=driver) _, problems = _installs(declaration) - problems += _ready_checks_beyond_reach(module, declaration) + problems += _class_beyond_reach(module, declaration) if (computer is None) != (driver is None): problems.append("a computer app deploys alone on a computer and a driver; name both or neither") elif computer is not None and not problems: @@ -149,9 +149,9 @@ def _installs(declaration: AppDeclaration) -> tuple[dict[str, AppInstall], list[ return installs({os_type: linux.model_dump() for os_type, linux in declaration.install.items()}, AppInstall) -def _ready_checks_beyond_reach(module: str, declaration: AppDeclaration) -> list[str]: - """An install that leaves out the class's file, from which the computer runs its ``@ready_check`` methods.""" +def _class_beyond_reach(module: str, declaration: AppDeclaration) -> list[str]: + """An install that leaves out the class's file, which the app's own server serves from the app's folder.""" file = PurePosixPath(module.partition(":")[0]) - return [f"install.{os_type}.files: @ready_check runs from the app's folder, which needs {str(file)!r} in " - f"{linux.files!r}" for os_type, linux in declaration.install.items() - if declaration.ready_checks and linux.files != "." and not file.is_relative_to(linux.files)] + return [f"install.{os_type}.files: the app's server serves its class from the app's folder, which needs " + f"{str(file)!r} in {linux.files!r}" for os_type, linux in declaration.install.items() + if linux.files != "." and not file.is_relative_to(linux.files)] diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index e4312c63a..bfdf61fda 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -29,6 +29,7 @@ from typing import TYPE_CHECKING, Any from agentenv_protocol.computer import apps as installed_apps +from agentenv_protocol.computer.app import SOCKET from agentenv_protocol.computer.conformance import run_probe from agent_env.artifact.artifacts.computer import ComputerArtifact @@ -63,11 +64,6 @@ #: Where a driver's files go in a composed computer, in a folder named for the driver; an app's, under APPS. DRIVERS = "/opt/agentenv/drivers" APPS = str(installed_apps.APPS) -#: Where a desktop session finds its launchers, and the entries it starts with it. -LAUNCHERS = "/usr/share/applications" -AUTOSTART = "/etc/xdg/autostart" -#: The script in an app's folder that its launcher and autostart entry run. -LAUNCH_SCRIPT = ".agentenv/launch" KEY_LABEL = "dev.agentenv.computer-key" # What a build installed and whether it was emulated, kept on its image for an env that reuses it. INSTALLED_LABEL, EMULATED_LABEL = "dev.agentenv.installed", "dev.agentenv.emulated" @@ -277,14 +273,14 @@ def _ready_base(base: ComputerArtifact, arch: str) -> None: @dataclass(frozen=True) class _Layer: """A driver's or an app's part of a build: its folder in the build context, its prefix in the image, what it - installs, the file of its supervisord programs, and files of the build context it places elsewhere in the image, - as (source, target).""" + installs, and the file of its supervisord programs. An app's venv also has the protocol's ``app`` extra, for the + app's own server, which listens in a folder of the app's the session user owns.""" folder: str prefix: str install: Install conf: str - placed: tuple[tuple[str, str], ...] = () + app: bool = False def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifact) -> _Layer: @@ -292,53 +288,37 @@ def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifa its port, its folder under ``DRIVERS`` named for its id's last part.""" install, name = driver.install[base.os.type], _name(driver.id, "driver") layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-{name}.conf") - _stage(context, layer, f"driver {driver.id}", driver.files, name, {"MCP_PORT": driver.port}) + _stage(context, layer, f"driver {driver.id}", driver.files, _services(name, install), {"MCP_PORT": driver.port}) return layer def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact) -> _Layer: - """Write the app's install files under ``context/apps//``, its supervisord programs beside them, run under - its name, and the description of it its computer's driver reads; with a launch, its launcher, which a ``boot`` app - also places among the desktop session's autostart entries. Its folder under ``APPS`` is named for its id's last - part.""" - install, name, launch = app.install[base.os.type], _name(app.id, "app"), app.declaration.launch - prefix, placed = f"{APPS}/{name}", () - if launch is not None: - entry = f"desktop/agentenv-app-{name}.desktop" - (context / "desktop").mkdir(exist_ok=True) - (context / entry).write_text(f"[Desktop Entry]\nType=Application\nName={app.environment_name}\n" - f"Exec={prefix}/{LAUNCH_SCRIPT}\nTerminal=false\n") - placed = ((entry, f"{LAUNCHERS}/agentenv-app-{name}.desktop"), - *([(entry, f"{AUTOSTART}/agentenv-app-{name}.desktop")] if launch.at == "boot" else [])) - layer = _Layer(f"apps/{name}", prefix, install, f"agentenv-app-{name}.conf", placed) - _stage(context, layer, f"app {app.id}", app.files, f"app-{name}", {"ENVIRONMENT_NAME": app.environment_name}) - described = context / layer.folder / installed_apps.APP_FILE - described.parent.mkdir(parents=True, exist_ok=True) - if launch is not None: - script = context / layer.folder / LAUNCH_SCRIPT - script.write_text(_launcher(app.environment_name, prefix, launch.command)) - script.chmod(_EXECUTABLE) - described.write_text(json.dumps({ - "name": app.environment_name, "module": app.module, - "launch": launch.model_dump(mode="json", exclude_none=True) if launch is not None else None, - "ready_checks": app.declaration.ready_checks}, indent=1)) + """Write the app's install files under ``context/apps//`` and its supervisord programs beside them, run under + its name: its services and its own server, which serves its class from its venv and launches the app. Its folder + under ``APPS`` is named for its id's last part.""" + install, name = app.install[base.os.type], _name(app.id, "app") + layer = _Layer(f"apps/{name}", f"{APPS}/{name}", install, f"agentenv-app-{name}.conf", app=True) + file, _, cls = app.module.partition(":") + served = shlex.quote(f"{PurePosixPath(file).relative_to(install.files)}:{cls}") + programs = {f"app-{name}": f"venv/bin/python -m agentenv_protocol.computer.serve_app {served}", + **_services(f"app-{name}", install)} + _stage(context, layer, f"app {app.id}", app.files, programs, {"ENVIRONMENT_NAME": app.environment_name}) return layer -def _launcher(name: str, prefix: str, command: list[str]) -> str: - """The script a desktop entry runs: ``command`` from the app's folder, as its services run, with - ``ENVIRONMENT_NAME`` set. A desktop session may ignore an entry's own working directory.""" - return f"#!/bin/sh\ncd {prefix} && ENVIRONMENT_NAME={shlex.quote(name)} exec {shlex.join(command)}\n" - - def _name(env_id: str, fallback: str) -> str: return re.sub(r"[^a-z0-9-]+", "-", env_id.rsplit("/", 1)[-1].lower()).strip("-") or fallback -def _stage(context: Path, layer: _Layer, what: str, files: FileArtifactUniverse, programs: str, +def _services(name: str, install: Install) -> dict[str, str]: + """The commands of ``install``'s services by their programs' names, each named for ``name``.""" + return {f"{name}-{service.name}": service.command for service in install.services} + + +def _stage(context: Path, layer: _Layer, what: str, files: FileArtifactUniverse, programs: dict[str, str], environment: dict[str, Any]) -> None: """Write the ``files`` of ``what``'s folder its install names to ``layer``'s folder, a script or binary executable, - and its services as supervisord programs named for ``programs``, with ``environment``.""" + and ``programs``, commands by name, as its supervisord programs, with ``environment``.""" folder = PurePosixPath(layer.install.files) for name, artifact in files.get_file_artifacts().items(): try: @@ -352,19 +332,19 @@ def _stage(context: Path, layer: _Layer, what: str, files: FileArtifactUniverse, if data.startswith(_RUN_ONLY): target.chmod(_EXECUTABLE) (context / layer.folder).mkdir(parents=True, exist_ok=True) - (context / layer.conf).write_text(_programs(programs, layer.prefix, layer.install, environment)) + (context / layer.conf).write_text(_programs(programs, layer.prefix, environment)) -def _programs(name: str, prefix: str, install: Install, environment: dict[str, Any]) -> str: +def _programs(commands: dict[str, str], prefix: str, environment: dict[str, Any]) -> str: variables = ",".join(f'{key}="{value}"' for key, value in environment.items()) programs = [] - for service in install.services: - program, _, rest = service.command.partition(" ") + for name, command in commands.items(): + program, _, rest = command.partition(" ") if "/" in program and not program.startswith("/"): program = f"{prefix}/{program}" - programs.append(f"[program:{name}-{service.name}]\ncommand={program} {rest}".rstrip() + "\n" + programs.append(f"[program:{name}]\ncommand={program} {rest}".rstrip() + "\n" f"directory={prefix}\nenvironment={variables}\nautorestart=true\nredirect_stderr=true\n" - f"stdout_logfile=/var/log/agentenv/{name}-{service.name}.log\n") + f"stdout_logfile=/var/log/agentenv/{name}.log\n") return "\n".join(programs) @@ -403,21 +383,22 @@ def _values(env: ComputerEnv) -> dict[str, Any]: def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build_settings: dict, protocol: bool) -> str: """The composed image's Dockerfile but its key's label, which the key is a digest of: each layer's files under its - prefix, its venv beside them, with this checkout's protocol when ``protocol``, and its programs.""" - venvs = [layer for layer in layers if layer.install.python is not None] + prefix, its venv beside them, with this checkout's protocol when ``protocol``, and its programs; an app's server's + folder owned by the session user.""" + venvs = [layer for layer in layers if layer.install.python is not None or layer.app] lines = [f"FROM {source}", "USER root", *(["COPY protocol /tmp/agentenv-protocol"] if protocol and venvs else [])] for layer in layers: - prefix, install = layer.prefix, layer.install + prefix = layer.prefix lines.append(f"COPY {layer.folder}/ {prefix}/") - if install.python is not None: - packages = f"/tmp/agentenv-protocol -r {prefix}/{install.python}" if protocol else f"-r {prefix}/{install.python}" - lines.append(f"RUN python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip install --no-cache-dir {packages} " - f"&& {prefix}/venv/bin/pip list --format=json > {prefix}/installed.json") + if layer in venvs: + lines.append(f"RUN python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip install --no-cache-dir " + f"{_packages(layer, protocol)} && {prefix}/venv/bin/pip list --format=json > " + f"{prefix}/installed.json") + if layer.app: + lines.append(f"RUN mkdir -p {prefix}/{SOCKET.parent} && chown {base.session.user} {prefix}/{SOCKET.parent}") lines.append(f"COPY {layer.conf} /etc/supervisor/conf.d/{layer.conf}") - lines += [f"COPY {source} {target}" for source, target in layer.placed] - checks = [f"test -d {prefix}", f"test -f /etc/supervisor/conf.d/{layer.conf}", - *(f"test -f {target}" for _, target in layer.placed)] - if install.python is not None: + checks = [f"test -d {prefix}", f"test -f /etc/supervisor/conf.d/{layer.conf}"] + if layer in venvs: checks.append(f"test -x {prefix}/venv/bin/python") lines.append(f"RUN {' && '.join(checks)}") if protocol and venvs: @@ -427,6 +408,16 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build return "\n".join(lines) + "\n" +def _packages(layer: _Layer, protocol: bool) -> str: + """What ``layer``'s venv installs: its requirements and, for an app, the protocol's ``app`` extra at this release, + which a driver's requirements name themselves; the protocol from this checkout's source when ``protocol``.""" + extra = "[app]" if layer.app else "" + protocols = ([f"/tmp/agentenv-protocol{extra}"] if protocol else + [f"{_PROTOCOL}{extra}=={importlib.metadata.version(_PROTOCOL)}"] if layer.app else []) + requirements = ["-r", f"{layer.prefix}/{layer.install.python}"] if layer.install.python is not None else [] + return shlex.join([*protocols, *requirements]) + + def _installed(local: str, prefix: str) -> dict[str, str]: """The packages ``local``'s build installed into the venv of the driver at ``prefix``, by name, read from the image without running it, so on any arch; none for a driver with no venv.""" diff --git a/tst/integration/computer/app/artifacts/check_notes/run.py b/tst/integration/computer/app/artifacts/check_notes/run.py index 9190cd4c3..533c5a6d0 100644 --- a/tst/integration/computer/app/artifacts/check_notes/run.py +++ b/tst/integration/computer/app/artifacts/check_notes/run.py @@ -1,5 +1,6 @@ """What the notes app's service wrote, which it can only once its settings reached it, the name the computer's card -gives and what its readiness says of the app, asked on loopback from inside the computer, and where its launcher is.""" +gives and what its readiness says of the app, asked on loopback from inside the computer, the launcher its server added +to the session user's menu and the apps launched since the computer booted.""" import json import time import urllib.request @@ -18,5 +19,5 @@ def run(input): ready = json.load(response) return {**json.loads(written.read_text()), "card": card["name"], "app_ready": [check for check in ready["checks"] if check["name"].startswith("app ")], - "launchers": sorted(p.name for p in Path("/usr/share/applications").glob("agentenv-app-*.desktop")), - "autostart": sorted(p.name for p in Path("/etc/xdg/autostart").glob("agentenv-app-*.desktop"))} + "launchers": sorted(p.name for p in (Path.home() / ".local/share/applications").glob("agentenv-app-*")), + "launched": sorted(p.name for p in Path("/run/agentenv/apps").iterdir())} diff --git a/tst/integration/computer/app/envs/notes/app.py b/tst/integration/computer/app/envs/notes/app.py index 285e71105..8e5f75b03 100644 --- a/tst/integration/computer/app/envs/notes/app.py +++ b/tst/integration/computer/app/envs/notes/app.py @@ -1,5 +1,6 @@ """A notes app for the computer app tests: its service writes the settings a deploy gave it to ~/.notes.json, and its -window opens with the desktop session; it's ready once both have happened.""" +own server launches its window once the desktop session is up; it's ready once both have happened. It requires nothing +of its own, so its service and window run in the venv every app gets for its server.""" from pathlib import Path from pydantic import BaseModel, SecretStr @@ -9,7 +10,7 @@ @computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) class Notes(ComputerAppEnvironment): - linux = Linux(python="requirements.txt", services=[Service(name="notes", command="venv/bin/python serve.py")]) + linux = Linux(services=[Service(name="notes", command="venv/bin/python serve.py")]) launch = Launch(at="boot", command=["venv/bin/python", "window.py"], window={"class": "Notes"}, ready=Ready(window=True)) skill = "skill.md" diff --git a/tst/integration/computer/app/envs/notes/requirements.txt b/tst/integration/computer/app/envs/notes/requirements.txt deleted file mode 100644 index f671e2e13..000000000 --- a/tst/integration/computer/app/envs/notes/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -agentenv-framework-protocol diff --git a/tst/integration/computer/computer_app_test.py b/tst/integration/computer/computer_app_test.py index b699feb2a..3d1a1dfd1 100644 --- a/tst/integration/computer/computer_app_test.py +++ b/tst/integration/computer/computer_app_test.py @@ -1,7 +1,8 @@ """A computer app deployed alone through ``agent-env run``: a one-app computer of the computer and driver its env.toml -names, its card named after it, its service running from its folder in a venv of the plain protocol package, on the -settings the deploy gave it, a secret resolved at boot, its window opened with the desktop session and the computer ready -only once its window and its @ready_check say so. Needs Docker and the local registry on localhost:5000.""" +names, its card named after it, its service running from its folder in the venv its own server runs in, on the +settings the deploy gave it, a secret resolved at boot, its window launched once by its server as the desktop session +comes up, and the computer ready only once its server says its window and its @ready_check are. Needs Docker and the +local registry on localhost:5000.""" from pathlib import Path @@ -28,4 +29,4 @@ def test_an_app_deployed_alone_runs_its_service_on_its_settings_on_a_computer_na assert context["metadata"]["script_results"]["service"] == { "signature": "-- Ada", "token": "t0ken", "card": "notes", "app_ready": [{"name": "app notes", "ok": True, "detail": "ready"}], - "launchers": ["agentenv-app-notes.desktop"], "autostart": ["agentenv-app-notes.desktop"]} + "launchers": ["agentenv-app-notes.desktop"], "launched": ["notes"]} diff --git a/tst/unit/drivers/reference_driver_test.py b/tst/unit/drivers/reference_driver_test.py index 42db3d6fe..cbed28526 100644 --- a/tst/unit/drivers/reference_driver_test.py +++ b/tst/unit/drivers/reference_driver_test.py @@ -32,10 +32,6 @@ class ConnectionClosedError(Exception): pass -class BadWindow(Exception): - pass - - class FakeX: """An X display: its screen and pointer, a US-like keymap whose capital letters are the second level of their keys, the keys down, the XTEST events sent, ``drops`` syncs and the ``fail_at``-th event that fail, and the @@ -51,25 +47,11 @@ def __init__(self) -> None: self.fail_at = None self.spare = {8} self.display = types.SimpleNamespace(info=types.SimpleNamespace(min_keycode=8, max_keycode=255)) - #: The window manager's windows by id, each its WM_CLASS instance and class and its title, or None once closed. - self.windows: dict[int, tuple[str, str, str] | None] = {} def screen(self): query = lambda: types.SimpleNamespace(root_x=self.pointer[0], root_y=self.pointer[1]) # noqa: E731 - listed = lambda atom, kind: types.SimpleNamespace(value=list(self.windows)) # noqa: E731 return types.SimpleNamespace(width_in_pixels=WIDTH, height_in_pixels=HEIGHT, - root=types.SimpleNamespace(query_pointer=query, get_full_property=listed)) - - def intern_atom(self, name: str) -> str: - return name - - def create_resource_object(self, kind: str, wid: int): - def named(): - if self.windows[wid] is None: - raise BadWindow(wid) - return self.windows[wid][:2] - title = lambda atom, kind: types.SimpleNamespace(value=self.windows[wid][2].encode()) # noqa: E731 - return types.SimpleNamespace(get_wm_class=named, get_full_property=title) + root=types.SimpleNamespace(query_pointer=query)) def sync(self) -> None: if self.drops: @@ -174,11 +156,10 @@ def module(name: str, **attrs) -> types.ModuleType: return m X = module("Xlib.X", MotionNotify="move", ButtonPress="button-down", ButtonRelease="button-up", - KeyPress="key-down", KeyRelease="key-up", AnyPropertyType=0) + KeyPress="key-down", KeyRelease="key-up") XK = module("Xlib.XK", string_to_keysym=lambda name: keysyms.setdefault(name, 0xff00 + len(keysyms))) display = module("Xlib.display", Display=open_display) - module("Xlib.error", ConnectionClosedError=ConnectionClosedError, DisplayConnectionError=ConnectionClosedError, - BadWindow=BadWindow) + module("Xlib.error", ConnectionClosedError=ConnectionClosedError, DisplayConnectionError=ConnectionClosedError) xtest = module("Xlib.ext.xtest", fake_input=fake_input) module("Xlib.ext", xtest=xtest) module("Xlib", X=X, XK=XK, display=display) @@ -416,12 +397,3 @@ async def test_the_reference_driver_records_from_its_stream_at_up_to_30_fps(x11) card = (await http.get("/.well-known/agent-env.json")).json() recording = next(e for e in card["capabilities"]["extensions"] if e["uri"] == COMPUTER_RECORDING) assert recording["params"]["source"] == "stream" and recording["params"]["fps"] == {"default": 10, "max": 30} - - -@pytest.mark.asyncio -async def test_the_desktops_windows_are_listed_by_class_and_title_leaving_out_one_that_closed(x11): - driver, fake, _ = x11 - fake.windows = {0x1e00004: ("notes", "Notes", "Notes"), 0x2000003: None, 0x2200001: ("xfdesktop", "Xfdesktop", "")} - - assert await driver.windows() == [{"instance": "notes", "class": "Notes", "title": "Notes"}, - {"instance": "xfdesktop", "class": "Xfdesktop", "title": ""}] diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 07f556281..c30b077e2 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -4,6 +4,7 @@ and the registry are faked.""" import asyncio +import importlib.metadata import io import json import logging @@ -84,12 +85,7 @@ def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_ self.builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), "files": sorted(p.relative_to(context).as_posix() for p in context.rglob("*") if p.is_file()), "conf": next(context.glob("agentenv-*.conf")).read_text(), - "confs": {conf.name: conf.read_text() for conf in context.glob("agentenv-*.conf")}, - "desktop": {entry.name: entry.read_text() for entry in context.glob("desktop/*")}, - "described": {path.parent.parent.name: json.loads(path.read_text()) - for path in context.glob("apps/*/.agentenv/app.json")}, - "launchers": {path.parent.parent.name: (path.read_text(), bool(path.stat().st_mode & 0o111)) - for path in context.glob("apps/*/.agentenv/launch")}}) + "confs": {conf.name: conf.read_text() for conf in context.glob("agentenv-*.conf")}}) def label(self, local, platform, extra): self.labels[local] = extra @@ -327,11 +323,7 @@ def test_a_base_built_from_its_context_is_built_here_for_each_arch_and_keyed_by_ def test_a_checkout_composes_its_own_protocol_into_the_drivers_venv(base, driver, docker, tmp_path, monkeypatch): - source = tmp_path / "protocol" - (source / "src" / "agentenv_protocol").mkdir(parents=True) - (source / "src" / "agentenv_protocol" / "__init__.py").write_text("") - (source / "pyproject.toml").write_text("[project]\nname = 'agentenv-framework-protocol'\n") - monkeypatch.setattr(composer, "_protocol_source", lambda: source) + monkeypatch.setattr(composer, "_protocol_source", lambda: _protocol_checkout(tmp_path)) _put(base, driver) @@ -341,6 +333,14 @@ def test_a_checkout_composes_its_own_protocol_into_the_drivers_venv(base, driver assert "protocol/pyproject.toml" in docker.builds[0]["files"] +def _protocol_checkout(folder): + source = folder / "protocol" + (source / "src" / "agentenv_protocol").mkdir(parents=True) + (source / "src" / "agentenv_protocol" / "__init__.py").write_text("") + (source / "pyproject.toml").write_text("[project]\nname = 'agentenv-framework-protocol'\n") + return source + + @pytest.mark.parametrize("left", [{"status": "building"}, {"status": "unverified", "image": "built", "key": "k"}], ids=["interrupted", "unprobed"]) def test_the_pairs_left_to_compose_are_those_failed_interrupted_or_unprobed_on_a_native_arch(base, driver, docker, @@ -939,8 +939,16 @@ async def test_a_load_outside_the_home_is_refused_before_anything_is_sent(base, "theme": {"enum": ["light", "dark"], "type": "string", "default": "light"}}, "type": "object"} +#: The program of the mail app's own server. +MAIL_SERVER = ("[program:app-mail]\ncommand=/opt/agentenv/apps/mail/venv/bin/python -m " + "agentenv_protocol.computer.serve_app app.py:Mail\ndirectory=/opt/agentenv/apps/mail\n" + "environment=ENVIRONMENT_NAME=\"mail\"\nautorestart=true\nredirect_stderr=true\n" + "stdout_logfile=/var/log/agentenv/app-mail.log\n") +PROTOCOL_VERSION = importlib.metadata.version("agentenv-framework-protocol") + + def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, launch=None, - ready_checks=(), **changes): + **changes): """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements.""" folder.mkdir(parents=True, exist_ok=True) (folder / "app.py").write_text("class Mail: ...\n") @@ -950,7 +958,7 @@ def _app(base, driver, folder, compose=True, supports=None, install=None, settin declaration = AppDeclaration.model_validate({ "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], "permissions": [], "launch": launch, "settings": settings, "skill": None, - "data": [], "install_steps": [], "ready_checks": list(ready_checks), + "data": [], "install_steps": [], "ready_checks": [], "install": {"linux": install or {"python": "requirements.txt", "services": [ {"name": "imap", "command": "venv/bin/python imap.py"}]}}}) return ComputerAppEnv.put(**{"id": "@local/~/kit/mail", "environment_name": "mail", "module": "app.py:Mail", @@ -968,11 +976,13 @@ def test_an_app_deployed_alone_is_composed_with_its_files_venv_and_programs_unde dockerfile = build["dockerfile"] for line in ("COPY apps/mail/ /opt/agentenv/apps/mail/", "RUN python3 -m venv /opt/agentenv/apps/mail/venv && /opt/agentenv/apps/mail/venv/bin/pip install " - "--no-cache-dir -r /opt/agentenv/apps/mail/requirements.txt", + f"--no-cache-dir 'agentenv-framework-protocol[app]=={PROTOCOL_VERSION}' -r " + "/opt/agentenv/apps/mail/requirements.txt", + "RUN mkdir -p /opt/agentenv/apps/mail/.agentenv && chown user /opt/agentenv/apps/mail/.agentenv", "COPY agentenv-app-mail.conf /etc/supervisor/conf.d/agentenv-app-mail.conf"): assert line in dockerfile assert dockerfile.index("COPY driver/") < dockerfile.index("COPY apps/mail/") < dockerfile.index("USER user") - assert build["confs"]["agentenv-app-mail.conf"] == ( + assert build["confs"]["agentenv-app-mail.conf"] == MAIL_SERVER + "\n" + ( "[program:app-mail-imap]\ncommand=/opt/agentenv/apps/mail/venv/bin/python imap.py\n" "directory=/opt/agentenv/apps/mail\nenvironment=ENVIRONMENT_NAME=\"mail\"\nautorestart=true\n" "redirect_stderr=true\nstdout_logfile=/var/log/agentenv/app-mail-imap.log\n") @@ -981,23 +991,19 @@ def test_an_app_deployed_alone_is_composed_with_its_files_venv_and_programs_unde "container/arm64": "ready"} -@pytest.mark.parametrize("at", ["boot", "demand"]) -def test_an_apps_launch_reaches_the_desktop_session_and_its_driver(base, driver, docker, tmp_path, at): - launch = {"at": at, "command": ["bin/mail", "--folder", "Work mail"], "window": {"class": "Mail"}, - "ready": {"window": True, "timeout_s": 240.0}} - app = _app(base, driver, tmp_path / "mail", launch=launch, ready_checks=["inbox_open"]) +@pytest.mark.parametrize("at, checkout", [("boot", False), ("demand", True)], ids=["boot-release", "demand-checkout"]) +def test_an_app_that_requires_nothing_still_gets_a_venv_for_its_own_server(base, driver, docker, tmp_path, monkeypatch, + at, checkout): + if checkout: + monkeypatch.setattr(composer, "_protocol_source", lambda: _protocol_checkout(tmp_path)) + launch = {"at": at, "command": ["bin/mail"], "window": {"class": "Mail"}, "ready": {"window": True, "timeout_s": 240.0}} + app = _app(base, driver, tmp_path / "mail", install={"services": []}, launch=launch) build = docker.builds[0] - assert build["desktop"] == {"agentenv-app-mail.desktop": ( - "[Desktop Entry]\nType=Application\nName=mail\nExec=/opt/agentenv/apps/mail/.agentenv/launch\nTerminal=false\n")} - assert build["launchers"] == {"mail": ( - "#!/bin/sh\ncd /opt/agentenv/apps/mail && ENVIRONMENT_NAME=mail exec bin/mail --folder 'Work mail'\n", True)} - copies = [line for line in build["dockerfile"].splitlines() if line.startswith("COPY desktop/")] - assert copies == ["COPY desktop/agentenv-app-mail.desktop /usr/share/applications/agentenv-app-mail.desktop", - *(["COPY desktop/agentenv-app-mail.desktop /etc/xdg/autostart/agentenv-app-mail.desktop"] - if at == "boot" else [])] - assert build["described"] == {"mail": {"name": "mail", "module": "app.py:Mail", "launch": launch, - "ready_checks": ["inbox_open"]}} + protocol = "/tmp/agentenv-protocol[app]" if checkout else f"agentenv-framework-protocol[app]=={PROTOCOL_VERSION}" + assert (f"RUN python3 -m venv /opt/agentenv/apps/mail/venv && /opt/agentenv/apps/mail/venv/bin/pip install " + f"--no-cache-dir '{protocol}' && ") in build["dockerfile"] + assert build["confs"]["agentenv-app-mail.conf"] == MAIL_SERVER assert boot.ready_timeout([app]) == (270.0 if at == "boot" else boot.READY_TIMEOUT_S) @@ -1014,9 +1020,9 @@ def test_an_app_is_composed_only_for_the_arches_it_supports(base, driver, docker ({"install": {"services": [{"name": "imap", "command": "a"}, {"name": "imap", "command": "b"}]}}, "install.linux.services: Value error, names a service twice"), ({"driver": None}, "a computer app deploys alone on a computer and a driver; name both or neither"), - ({"install": {"files": "linux", "python": "requirements.txt", "services": []}, "ready_checks": ["inbox_open"]}, - "install.linux.files: @ready_check runs from the app's folder, which needs 'app.py' in 'linux'"), -], ids=["platform", "arch", "install", "no-driver", "ready-check-file"]) + ({"install": {"files": "linux", "services": []}}, + "install.linux.files: the app's server serves its class from the app's folder, which needs 'app.py' in 'linux'"), +], ids=["platform", "arch", "install", "no-driver", "class-file"]) def test_an_app_that_doesnt_fit_its_computer_and_driver_is_refused_before_anything_is_stored(base, driver, docker, tmp_path, changes, problem): with pytest.raises(ValueError, match=problem.replace(".", "[.]")): diff --git a/uv.lock b/uv.lock index cf3b6e711..c45813706 100644 --- a/uv.lock +++ b/uv.lock @@ -231,6 +231,10 @@ agent = [ { name = "regex" }, { name = "uvicorn" }, ] +app = [ + { name = "mcp" }, + { name = "uvicorn" }, +] computer = [ { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "av", version = "19.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, @@ -260,6 +264,7 @@ requires-dist = [ { name = "fastmcp", marker = "extra == 'dev'", specifier = ">=3.4,<4" }, { name = "httpx", specifier = ">=0.27" }, { name = "mcp", marker = "extra == 'agent'", specifier = ">=1.25,<2" }, + { name = "mcp", marker = "extra == 'app'", specifier = ">=1.25,<2" }, { name = "mcp", marker = "extra == 'computer'", specifier = ">=1.25,<2" }, { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.25,<2" }, { name = "pillow", marker = "extra == 'computer'", specifier = ">=10.4" }, @@ -273,9 +278,10 @@ requires-dist = [ { name = "regex", marker = "extra == 'dev'", specifier = ">=2023.12.25" }, { name = "starlette", specifier = ">=0.37" }, { name = "uvicorn", marker = "extra == 'agent'", specifier = ">=0.30" }, + { name = "uvicorn", marker = "extra == 'app'", specifier = ">=0.30" }, { name = "uvicorn", marker = "extra == 'computer'", specifier = ">=0.30" }, ] -provides-extras = ["agent", "computer", "dev"] +provides-extras = ["agent", "app", "computer", "dev"] [[package]] name = "aiofile" From 203d4957c1848141816cccf7645f23f5d211fb84 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 10:12:54 -0700 Subject: [PATCH 3/6] fix(computer): check apps at once and say why an app's server didn't start An app's server ran its checks one after another and the driver asked each app in turn, while a deploy gave the whole computer-ready status 10 s, so one slow check timed the status out and the deploy's error stopped naming the app. Checks now run at once in the server, each still within 10 s, the driver asks every app at once and counts a server as not answering after 15 s (ANSWER_TIMEOUT_S), and the deploy's status call allows 30 s. A server that can't start writes why to .agentenv/server.failure in the app's folder, cleared at each start, and the driver reports it instead of "its server isn't answering yet". Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 15 ++-- .../src/agentenv_protocol/computer/app.py | 8 +- .../src/agentenv_protocol/computer/apps.py | 18 ++++- .../src/agentenv_protocol/computer/runtime.py | 2 +- .../agentenv_protocol/computer/serve_app.py | 15 +++- .../tests/test_computer_app_server.py | 75 +++++++++++++++---- src/agent_env/env/envs/computer/utils/boot.py | 6 +- 7 files changed, 108 insertions(+), 31 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index adcc94760..6340355af 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -586,13 +586,14 @@ has the `app` extra (`agentenv-framework-protocol[app]`: mcp and uvicorn). It se card and tools on the Unix socket `.agentenv/server.sock` in that folder, with `computer-app/v1`: the app's `launch` as its params, and the methods `launch` (POST), which starts the app in the desktop session, and `ready` (GET: `ready`, `problems`, `launched`, `waited_s` and `timeout_s`), -which runs the app's readiness checks then, each within 10 s. Once the session is up the server adds -the app's launcher to the session user's menu, which runs `python -m -agentenv_protocol.computer.launch ` to ask the server, so the server starts every copy of -the app, and launches a `boot` app, once per boot. The driver finds each app by the socket in its -folder: the computer is ready once every app's server answers and each `boot` app, and each app -with no launch, says it's ready; a `demand` app counts once something launches it. What the server -does in the session (`agentenv_protocol.computer.session`) is Linux's. +which runs the app's readiness checks then, at once, each within 10 s. A server that can't start +writes why to `.agentenv/server.failure` in the folder, and the driver reports that in its place. +Once the session is up the server adds the app's launcher to the session user's menu, which runs +`python -m agentenv_protocol.computer.launch ` to ask the server, so the server starts every +copy of the app, and launches a `boot` app, once per boot. The driver finds each app by the socket +in its folder: the computer is ready once every app's server answers and each `boot` app, and each +app with no launch, says it's ready; a `demand` app counts once something launches it. What the +server does in the session (`agentenv_protocol.computer.session`) is Linux's. `MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app` env stores, and `python -m agentenv_protocol.computer._declare ` prints it as diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index de3a1e0e5..e72f4db62 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -32,11 +32,12 @@ WINDOW_KEYS = frozenset({"class", "instance", "title"}) #: The framework extension an app's own server serves: its launch as params, and its ``launch`` and ``ready`` methods. COMPUTER_APP = "urn:agentenv:computer-app/v1" -#: Where an app's own server listens, in the app's folder. +#: Where an app's own server listens, in the app's folder, and where it says why it last failed to start. SOCKET = Path(".agentenv/server.sock") +FAILURE = Path(".agentenv/server.failure") #: Where an app's server marks that it launched the app since the computer booted. LAUNCHED = Path("/run/agentenv/apps") -#: How long one readiness check may take. +#: How long one readiness check may take; an app's checks run at once. CHECK_TIMEOUT_S = 10.0 @@ -231,7 +232,8 @@ async def _computer_app_ready(self) -> dict: checks.append(("command", lambda: _command(ready.command))) checks += [(f"@ready_check {fn.__name__}", lambda fn=fn: _ready_check(fn, self)) for fn in _methods(type(self)) if hasattr(fn, _READY_CHECK_ATTR)] - problems = [problem for name, check in checks if (problem := await _bounded(name, check)) is not None] + found = await asyncio.gather(*(_bounded(name, check) for name, check in checks)) + problems = [problem for problem in found if problem is not None] if self.launch is not None and self._computer_app_launched_at is None: problems.insert(0, f"its launch failed: {self._computer_app_launch_failed}" if self._computer_app_launch_failed else "not launched yet") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py index 3a7063362..5b8e1ffce 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py @@ -10,11 +10,13 @@ from .. import client from ..types import RPC_PATH, WELL_KNOWN_PATH -from .app import CHECK_TIMEOUT_S, COMPUTER_APP, SOCKET +from .app import CHECK_TIMEOUT_S, COMPUTER_APP, FAILURE, SOCKET from .operations import ReadyCheck #: Where a composed computer installs its apps, a folder each. APPS = Path("/opt/agentenv/apps") +#: How long an app's server may take to answer, its checks run at once, before it counts as not answering. +ANSWER_TIMEOUT_S = CHECK_TIMEOUT_S + 5 def expected() -> list[Path]: @@ -23,7 +25,8 @@ def expected() -> list[Path]: async def check(folder: Path) -> ReadyCheck: - """Whether the app in ``folder`` is ready, as its own server says; named by its card once the server answers.""" + """Whether the app in ``folder`` is ready, as its own server says, or why its server last failed to start; named by + its card once the server answers.""" name = folder.name try: async with _client(folder / SOCKET) as http: @@ -32,7 +35,7 @@ async def check(folder: Path) -> ReadyCheck: launch = client.extension_params(card, COMPUTER_APP).get("launch") answer = (await http.get(f"{RPC_PATH}/ext/computer-app/ready")).raise_for_status().json() except (httpx.HTTPError, OSError): - return ReadyCheck(f"app {name}", False, "its server isn't answering yet") + return ReadyCheck(f"app {name}", False, _unanswered(folder)) if launch is not None and launch["at"] == "demand" and not answer["launched"]: return ReadyCheck(f"app {name}", True, "its server answers; it launches on demand") if answer["ready"]: @@ -42,6 +45,13 @@ async def check(folder: Path) -> ReadyCheck: return ReadyCheck(f"app {name}", False, f"{when}: {'; '.join(answer['problems'])}") +def _unanswered(folder: Path) -> str: + try: + return f"its server failed to start: {(folder / FAILURE).read_text()}" + except FileNotFoundError: + return "its server isn't answering yet" + + def _client(socket: Path) -> httpx.AsyncClient: return httpx.AsyncClient(transport=httpx.AsyncHTTPTransport(uds=str(socket)), base_url="http://app", - timeout=CHECK_TIMEOUT_S * 4) + timeout=ANSWER_TIMEOUT_S) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py index 443a6a8cd..2546679f0 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/runtime.py @@ -177,7 +177,7 @@ async def ready_checks(self) -> list[ReadyCheck]: checks += list(await self.driver.ready_checks()) except Exception as e: checks.append(ReadyCheck("driver", False, f"{type(e).__name__}: {e}")) - checks += [await apps.check(folder) for folder in self.apps] + checks += await asyncio.gather(*(apps.check(folder) for folder in self.apps)) if all(c.ok for c in checks): self.ready = True return checks diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py index a0eea2c82..e7b2e51af 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py @@ -1,23 +1,34 @@ """Serve a computer app's class from its file, as its computer's program starts it: ``python -m agentenv_protocol.computer.serve_app :``, run in the app's folder, serves the class's card and ``computer-app/v1`` on the Unix socket ``.agentenv/server.sock`` there. Once it serves, the app's server adds the app's -launcher and launches a boot app (``ComputerAppEnvironment._computer_app_start``). Needs the ``app`` extra.""" +launcher and launches a boot app (``ComputerAppEnvironment._computer_app_start``). A server that can't start writes why +to ``.agentenv/server.failure``, which its computer's driver reports. Needs the ``app`` extra.""" from __future__ import annotations import asyncio import contextlib import sys +import traceback from pathlib import Path import uvicorn from ._load import load_class -from .app import SOCKET, ComputerAppEnvironment +from .app import FAILURE, SOCKET, ComputerAppEnvironment USAGE = "usage: python -m agentenv_protocol.computer.serve_app :" def main(argv: list[str]) -> None: + FAILURE.unlink(missing_ok=True) + try: + _serve(argv) + except (Exception, SystemExit) as e: + FAILURE.write_text(traceback.format_exception_only(e)[-1].strip()) + raise + + +def _serve(argv: list[str]) -> None: if len(argv) != 1 or ":" not in argv[0]: raise SystemExit(USAGE) file, _, name = argv[0].partition(":") diff --git a/packages/agentenv-protocol/tests/test_computer_app_server.py b/packages/agentenv-protocol/tests/test_computer_app_server.py index f62c66e98..76a844776 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_server.py +++ b/packages/agentenv-protocol/tests/test_computer_app_server.py @@ -23,9 +23,20 @@ from agentenv_protocol import client # noqa: E402 from agentenv_protocol.computer import ComputerAppEnvironment, Launch, ReadyCheck, Ready, ready_check # noqa: E402 from agentenv_protocol.computer import app as app_module, apps, serve_app, session # noqa: E402 -from agentenv_protocol.computer.app import COMPUTER_APP, SOCKET # noqa: E402 +from agentenv_protocol.computer.app import COMPUTER_APP, FAILURE, SOCKET # noqa: E402 READY = "/agentenv/ext/computer-app/ready" +APP = '''from agentenv_protocol.computer import ComputerAppEnvironment + + +class Helper: + pass + + +class Broken(ComputerAppEnvironment): + def __init__(self): + raise RuntimeError("no mailbox") +''' class Mail(ComputerAppEnvironment): @@ -93,7 +104,7 @@ async def test_an_apps_server_is_ready_once_it_launched_the_app_and_each_check_p @pytest.mark.asyncio -async def test_a_check_that_fails_hangs_or_raises_says_so_without_holding_up_the_server(desktop, monkeypatch): +async def test_checks_that_fail_hang_or_raise_say_so_at_once_without_holding_up_the_server(desktop, monkeypatch): monkeypatch.setattr(app_module, "CHECK_TIMEOUT_S", 1.0) with socket.socket() as closed: closed.bind(("127.0.0.1", 0)) @@ -108,6 +119,11 @@ def stuck(self): time.sleep(3) return True + @ready_check + async def waiting(self): + await asyncio.sleep(3) + return True + @ready_check async def broken(self): raise RuntimeError("mailbox locked") @@ -117,8 +133,8 @@ async def broken(self): assert answer["problems"] == ["not launched yet", f"nothing answers on port {port}", f"{shlex.join(exits)} exited 1: no database yet", "@ready_check stuck ran past 1 s", - "@ready_check broken: RuntimeError: mailbox locked"] - assert time.monotonic() - started < 2.5 + "@ready_check waiting ran past 1 s", "@ready_check broken: RuntimeError: mailbox locked"] + assert time.monotonic() - started < 1.8 @pytest.mark.asyncio @@ -161,6 +177,8 @@ async def test_the_driver_asks_each_apps_server_on_the_socket_in_its_folder(desk with tempfile.TemporaryDirectory(dir="/tmp", prefix="apps-") as root: monkeypatch.setattr(apps, "APPS", Path(root)) (Path(root) / "gone").mkdir() + (Path(root) / "broken" / SOCKET).parent.mkdir(parents=True) + (Path(root) / "broken" / FAILURE).write_text("RuntimeError: no mailbox") async with contextlib.AsyncExitStack() as stack: served = {name: await stack.enter_async_context(_serving(cls(), Path(root) / name, monkeypatch)) for name, cls in [("mail", Mail), ("music", Music), ("sync", Sync)]} @@ -174,6 +192,7 @@ async def test_the_driver_asks_each_apps_server_on_the_socket_in_its_folder(desk playing = await apps.check(Path(root) / "music") assert first == [ + ReadyCheck("app broken", False, "its server failed to start: RuntimeError: no mailbox"), ReadyCheck("app gone", False, "its server isn't answering yet"), ReadyCheck("app mail", False, "not ready yet, 0 s in: not launched yet; no window with class 'Mail'; " "@ready_check inbox_synced returned False"), @@ -192,6 +211,7 @@ async def test_serving_an_app_starts_it_once_the_server_serves_on_the_socket_in_ "class Notes(ComputerAppEnvironment):\n pass\n") (tmp_path / SOCKET).parent.mkdir() (tmp_path / SOCKET).write_text("left by the last server") + (tmp_path / FAILURE).write_text("RuntimeError: the last start failed") monkeypatch.chdir(tmp_path) monkeypatch.setattr(sys, "path", list(sys.path)) monkeypatch.setattr(sys, "modules", dict(sys.modules)) @@ -204,24 +224,53 @@ async def test_serving_an_app_starts_it_once_the_server_serves_on_the_socket_in_ async with asgi.router.lifespan_context(asgi): await asyncio.sleep(0) - assert uds == str(SOCKET) and not (tmp_path / SOCKET).exists() + assert uds == str(SOCKET) and not (tmp_path / SOCKET).exists() and not (tmp_path / FAILURE).exists() assert [cls.__name__ for cls in started] == ["Notes"] -@pytest.mark.parametrize("argv, message", [ - ([], serve_app.USAGE), (["app.py"], serve_app.USAGE), (["missing.py:Notes"], "missing.py isn't a file"), - (["app.py:Helper"], "has no ComputerAppEnvironment subclass named Helper"), -], ids=["nothing", "no-class", "no-file", "not-an-app"]) -def test_what_isnt_an_app_class_exits_saying_why(tmp_path, monkeypatch, argv, message): - (tmp_path / "app.py").write_text("class Helper:\n pass\n") +@pytest.mark.parametrize("argv, failure", [ + ([], f"SystemExit: {serve_app.USAGE}"), (["app.py"], f"SystemExit: {serve_app.USAGE}"), + (["missing.py:Notes"], "SystemExit: missing.py isn't a file"), + (["app.py:Helper"], "SystemExit: app.py has no ComputerAppEnvironment subclass named Helper"), + (["app.py:Broken"], "RuntimeError: no mailbox"), +], ids=["nothing", "no-class", "no-file", "not-an-app", "fails-to-construct"]) +def test_a_server_that_cant_start_says_why_in_the_apps_folder(tmp_path, monkeypatch, argv, failure): + (tmp_path / "app.py").write_text(APP) + (tmp_path / SOCKET).parent.mkdir() monkeypatch.chdir(tmp_path) monkeypatch.setattr(sys, "path", list(sys.path)) monkeypatch.setattr(sys, "modules", dict(sys.modules)) - with pytest.raises(SystemExit) as stopped: + with pytest.raises((SystemExit, RuntimeError)) as stopped: serve_app.main(argv) - assert message in str(stopped.value) + assert (tmp_path / FAILURE).read_text() == failure == f"{type(stopped.value).__name__}: {stopped.value}" + + +@pytest.mark.asyncio +async def test_the_drivers_readiness_asks_every_apps_server_at_once(desktop, monkeypatch): + pytest.importorskip("PIL") + from computer_fake import FakeDriver, http_client + + class Settling(ComputerAppEnvironment): + @ready_check + async def settled(self): + await asyncio.sleep(1) + return True + + with tempfile.TemporaryDirectory(dir="/tmp", prefix="apps-") as root: + monkeypatch.setattr(apps, "APPS", Path(root)) + async with contextlib.AsyncExitStack() as stack: + for name in ("mail", "music", "sync"): + await stack.enter_async_context(_serving(Settling(), Path(root) / name, monkeypatch)) + started = time.monotonic() + async with http_client(FakeDriver()) as http: + status = (await http.get("/agentenv/ext/computer-ready/status")).json() + took = time.monotonic() - started + + assert [check for check in status["checks"] if check["name"].startswith("app ")] == [ + {"name": f"app {name}", "ok": True, "detail": "ready"} for name in ("mail", "music", "sync")] + assert status["ready"] is True and took < 2.5 @pytest.mark.asyncio diff --git a/src/agent_env/env/envs/computer/utils/boot.py b/src/agent_env/env/envs/computer/utils/boot.py index 66d309ea3..f348702cc 100644 --- a/src/agent_env/env/envs/computer/utils/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -13,6 +13,7 @@ from agentenv_protocol import client as protocol_v1 from agentenv_protocol.computer import APP_SETTINGS_ENV, COMPUTER_READY, SETTINGS_ENV +from agentenv_protocol.computer.apps import ANSWER_TIMEOUT_S from agent_env.config import get_config from agent_env.providers.env_providers.env_provider import _size_kwargs @@ -29,6 +30,8 @@ READY_TIMEOUT_S = 180.0 #: How much longer than the slowest starting app's own readiness timeout a computer may take, for the rest of it first. APP_READY_MARGIN_S = 30.0 +#: How long one computer-ready status may take: the driver's own checks, then every app's server at once. +_STATUS_TIMEOUT_S = int(ANSWER_TIMEOUT_S) + 15 _POLL_S = 0.5 #: How a setting a record keeps shows a secret's or a generated one's value, which it never holds. WITHHELD = "" @@ -128,7 +131,8 @@ async def ready(url: str, timeout: float) -> dict: while True: try: card = await protocol_v1.get_card(url, timeout=5) - status = await protocol_v1.invoke_extension(url, card, COMPUTER_READY, None, 10, method="status") + status = await protocol_v1.invoke_extension(url, card, COMPUTER_READY, None, _STATUS_TIMEOUT_S, + method="status") if status.get("ready"): return card last = "not ready: " + "; ".join(f"{check['name']}: {check.get('detail')}" From 8d43829353394ba6497f7d953011cbf1b87d0efc Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 10:21:07 -0700 Subject: [PATCH 4/6] fix(computer): keep a demand app's launch, one call per stuck check, and xprop's titles A restarted app server now finds a demand app it launched as well as a boot one: every launch marks /run/agentenv/apps/, so the driver keeps counting the open app. A sync @ready_check past its bound keeps running in its thread, which can't be stopped, so a later poll waits on that call instead of starting another, and polls can't fill the worker threads. The session reads window titles and classes without xprop's escapes, so a title with quotes matches, and kills xprop when a window check is cut short. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/computer/app.py | 44 ++++++++----- .../src/agentenv_protocol/computer/session.py | 15 ++++- .../tests/test_computer_app_server.py | 62 +++++++++++++++++-- 3 files changed, 96 insertions(+), 25 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index e72f4db62..e4c74f38e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -35,7 +35,7 @@ #: Where an app's own server listens, in the app's folder, and where it says why it last failed to start. SOCKET = Path(".agentenv/server.sock") FAILURE = Path(".agentenv/server.failure") -#: Where an app's server marks that it launched the app since the computer booted. +#: Where an app's server marks that it launched the app since the computer booted, at boot or on demand. LAUNCHED = Path("/run/agentenv/apps") #: How long one readiness check may take; an app's checks run at once. CHECK_TIMEOUT_S = 10.0 @@ -187,27 +187,23 @@ def __init__(self) -> None: self._computer_app_started_at = time.monotonic() self._computer_app_launched_at: Optional[float] = None self._computer_app_launch_failed: Optional[str] = None + #: Each sync ``@ready_check`` still running in its thread, by name, which a later poll waits on. + self._computer_app_checking: dict[str, asyncio.Future] = {} async def _computer_app_start(self) -> None: """Once the desktop session is up, add the app's menu launcher and launch a boot app, once per boot: a server - that restarts leaves the app it launched be.""" + that restarts finds the app launched, at boot or on demand, and leaves it be.""" if self.launch is None: return await session.wait_until_up() session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) - marker = LAUNCHED / Path.cwd().name - if self.launch.at != "boot": - return - if marker.exists(): + if _launched().exists(): self._computer_app_launched_at = time.monotonic() - return - try: - await self._computer_app_launch() - except Exception as e: - self._computer_app_launch_failed = f"{type(e).__name__}: {e}" - return - marker.parent.mkdir(parents=True, exist_ok=True) - marker.touch() + elif self.launch.at == "boot": + try: + await self._computer_app_launch() + except Exception as e: + self._computer_app_launch_failed = f"{type(e).__name__}: {e}" @extension(COMPUTER_APP, op="launch", description=( "The app's own server: launch launches the app from its folder, as its launcher and a boot do; ready says " @@ -218,6 +214,8 @@ async def _computer_app_launch(self) -> dict: pid = await session.launch(self.launch.command, Path.cwd(), {"ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}) self._computer_app_launched_at, self._computer_app_launch_failed = time.monotonic(), None + _launched().parent.mkdir(parents=True, exist_ok=True) + _launched().touch() return {"pid": pid} @extension(COMPUTER_APP, op="ready", method="GET") @@ -272,6 +270,10 @@ def declaration(cls) -> AppDeclaration: ) +def _launched() -> Path: + return LAUNCHED / Path.cwd().name + + def _methods(cls: type) -> list[Callable]: """The class's methods, each as the class resolves its name, in the order they were first defined.""" names = dict.fromkeys(name for klass in reversed(cls.__mro__) for name, member in vars(klass).items() @@ -284,7 +286,7 @@ async def _bounded(name: str, check: Callable[[], Awaitable[Optional[str]]]) -> try: return await asyncio.wait_for(check(), CHECK_TIMEOUT_S) except TimeoutError: - return f"{name} ran past {CHECK_TIMEOUT_S:.0f} s" + return f"{name} ran past {CHECK_TIMEOUT_S:g} s" except Exception as e: return f"{name}: {type(e).__name__}: {e}" @@ -319,6 +321,14 @@ async def _command(argv: list[str]) -> Optional[str]: async def _ready_check(fn: Callable, app: ComputerAppEnvironment) -> Optional[str]: - """A sync check runs in a thread, so it can't hold up the server.""" - passed = await fn(app) if inspect.iscoroutinefunction(fn) else await asyncio.to_thread(fn, app) + """A sync check runs in a thread, so it can't hold up the server, and once at a time: a thread can't be stopped, so + a poll while one runs past its bound waits on that same call.""" + if inspect.iscoroutinefunction(fn): + passed = await fn(app) + else: + running = app._computer_app_checking.get(fn.__name__) + if running is None or running.done(): + running = app._computer_app_checking[fn.__name__] = asyncio.ensure_future(asyncio.to_thread(fn, app)) + running.add_done_callback(lambda done: done.cancelled() or done.exception()) + passed = await asyncio.shield(running) return None if passed else f"@ready_check {fn.__name__} returned {passed!r}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py index 71baf11f4..18da07594 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py @@ -50,15 +50,24 @@ async def windows() -> list[dict[str, str]]: for window in re.findall(r"0x[0-9a-f]+", listed.partition("#")[2]): props = await _xprop("-id", window, "WM_CLASS", "_NET_WM_NAME") names = dict(re.findall(r'^(WM_CLASS|_NET_WM_NAME)[^=]*= (.*)$', props, re.MULTILINE)) - parts = re.findall(r'"((?:[^"\\]|\\.)*)"', names.get("WM_CLASS", "")) + parts = _strings(names.get("WM_CLASS", "")) if len(parts) == 2: - title = re.findall(r'"((?:[^"\\]|\\.)*)"', names.get("_NET_WM_NAME", "")) + title = _strings(names.get("_NET_WM_NAME", "")) found.append({"instance": parts[0], "class": parts[1], "title": title[0] if title else ""}) return found +def _strings(value: str) -> list[str]: + """The quoted strings of an ``xprop`` value, as the window holds them, without the escapes ``xprop`` prints.""" + return [re.sub(r"\\(.)", r"\1", text) for text in re.findall(r'"((?:[^"\\]|\\.)*)"', value)] + + async def _xprop(*args: str) -> str: process = await asyncio.create_subprocess_exec("xprop", *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL) - out, _ = await process.communicate() + try: + out, _ = await process.communicate() + except asyncio.CancelledError: + process.kill() + raise return out.decode(errors="replace") if process.returncode == 0 else "" diff --git a/packages/agentenv-protocol/tests/test_computer_app_server.py b/packages/agentenv-protocol/tests/test_computer_app_server.py index 76a844776..7d1cd0db5 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_server.py +++ b/packages/agentenv-protocol/tests/test_computer_app_server.py @@ -6,9 +6,11 @@ import contextlib import os import shlex +import signal import socket import sys import tempfile +import threading import time from pathlib import Path @@ -59,8 +61,8 @@ async def caught_up(self): @pytest.fixture -def desktop(monkeypatch): - """A session with no windows yet, recording what's launched into it.""" +def desktop(monkeypatch, tmp_path): + """A session with no windows yet, recording what's launched into it, its launches marked under ``tmp_path``.""" shown, launched = [], [] async def windows(): @@ -76,6 +78,7 @@ async def up(): monkeypatch.setattr(session, "windows", windows) monkeypatch.setattr(session, "launch", launch) monkeypatch.setattr(session, "wait_until_up", up) + monkeypatch.setattr(app_module, "LAUNCHED", tmp_path / "launched") monkeypatch.setenv("ENVIRONMENT_NAME", "mail") return shown, launched @@ -141,7 +144,6 @@ async def broken(self): async def test_a_boot_app_is_launched_once_per_boot_with_its_launcher_added_each_start(desktop, tmp_path, monkeypatch): _, launched = desktop launchers = [] - monkeypatch.setattr(app_module, "LAUNCHED", tmp_path / "launched") monkeypatch.setattr(session, "add_launcher", lambda name, folder: launchers.append((name, folder))) monkeypatch.chdir(tmp_path) @@ -154,13 +156,48 @@ async def test_a_boot_app_is_launched_once_per_boot_with_its_launcher_added_each assert (await restarted._computer_app_ready())["launched"] is True +@pytest.mark.asyncio +async def test_a_restarted_server_finds_the_demand_app_it_launched_and_leaves_it_be(desktop, tmp_path, monkeypatch): + _, launched = desktop + monkeypatch.setattr(session, "add_launcher", lambda name, folder: None) + monkeypatch.setenv("ENVIRONMENT_NAME", "music") + + opened, restarted = Music(), Music() + await opened._computer_app_start() + await opened._computer_app_launch() + await restarted._computer_app_start() + + assert len(launched) == 1 and (await restarted._computer_app_ready())["launched"] is True + + +@pytest.mark.asyncio +async def test_a_sync_check_past_its_bound_runs_once_however_often_readiness_is_asked(desktop, monkeypatch): + monkeypatch.setattr(app_module, "CHECK_TIMEOUT_S", 0.2) + released, calls = threading.Event(), [] + + class Syncing(ComputerAppEnvironment): + @ready_check + def synced(self): + calls.append(len(calls)) + return released.wait(5) + + syncing = Syncing() + stuck = [(await syncing._computer_app_ready())["problems"] for _ in range(3)] + released.set() + while not syncing._computer_app_checking["synced"].done(): + await asyncio.sleep(0.01) + after = await syncing._computer_app_ready() + + assert stuck == [["@ready_check synced ran past 0.2 s"]] * 3 + assert after["ready"] is True and calls == [0, 1] + + @pytest.mark.asyncio async def test_a_boot_app_whose_launch_fails_says_why_and_is_launched_again_by_its_next_server(desktop, tmp_path, monkeypatch): async def missing(argv, cwd, env): raise FileNotFoundError(f"No such file or directory: {argv[0]!r}") - monkeypatch.setattr(app_module, "LAUNCHED", tmp_path / "launched") monkeypatch.setattr(session, "add_launcher", lambda name, folder: None) monkeypatch.setattr(session, "launch", missing) mail = Mail() @@ -307,7 +344,22 @@ async def xprop(*args): return replies[args] monkeypatch.setattr(session, "_xprop", xprop) - assert await session.windows() == [{"instance": "mail", "class": "Mail", "title": 'Inbox \\"work\\"'}] + assert await session.windows() == [{"instance": "mail", "class": "Mail", "title": 'Inbox "work"'}] + + +@pytest.mark.asyncio +async def test_a_window_listing_cut_short_kills_its_xprop(monkeypatch): + started, spawn = [], asyncio.create_subprocess_exec + + async def hanging(*args, **kwargs): + started.append(await spawn("sleep", "30", **kwargs)) + return started[-1] + + monkeypatch.setattr(asyncio, "create_subprocess_exec", hanging) + with pytest.raises(TimeoutError): + await asyncio.wait_for(session.windows(), 0.2) + + assert await asyncio.wait_for(started[0].wait(), 5) == -signal.SIGKILL @contextlib.asynccontextmanager From 8a89ed770ae2707794e6fd328de422107b7212fe Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 11:22:53 -0700 Subject: [PATCH 5/6] refactor(computer): an app server's runtime of its own, with a typed readiness What an app's server does moves out of app.py into app_runtime.py, as a driver's does into runtime.py: AppRuntime holds its state, launches the app and runs its checks, and the class keeps thin launch and ready methods and the declaration. A subclass whose __init__ skips super().__init__() now fails at start, saying so in .agentenv/server.failure. The ready answer is an AppReadiness the server dumps and the driver validates. Whether the app was launched comes from its boot mark alone, and its clock from the mark's time, so a restarted server no longer restarts the app's clock. The driver and the launcher take the computer-app/v1 endpoints from the app's card instead of building them, and checks run as coroutines rather than lambdas. The serve_app startup tests join the serve tests, sharing one import-isolation fixture. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 18 +- .../src/agentenv_protocol/computer/app.py | 128 ++------------- .../agentenv_protocol/computer/app_runtime.py | 155 ++++++++++++++++++ .../src/agentenv_protocol/computer/apps.py | 22 ++- .../src/agentenv_protocol/computer/launch.py | 6 +- .../agentenv_protocol/computer/serve_app.py | 5 +- .../tests/test_computer_app_server.py | 95 +++-------- .../tests/test_computer_serve.py | 87 +++++++++- 8 files changed, 303 insertions(+), 213 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 6340355af..c183c11bb 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -585,15 +585,15 @@ On a computer each app is its own server: `python -m agentenv_protocol.computer. has the `app` extra (`agentenv-framework-protocol[app]`: mcp and uvicorn). It serves the class's card and tools on the Unix socket `.agentenv/server.sock` in that folder, with `computer-app/v1`: the app's `launch` as its params, and the methods `launch` (POST), which starts the app in the -desktop session, and `ready` (GET: `ready`, `problems`, `launched`, `waited_s` and `timeout_s`), -which runs the app's readiness checks then, at once, each within 10 s. A server that can't start -writes why to `.agentenv/server.failure` in the folder, and the driver reports that in its place. -Once the session is up the server adds the app's launcher to the session user's menu, which runs -`python -m agentenv_protocol.computer.launch ` to ask the server, so the server starts every -copy of the app, and launches a `boot` app, once per boot. The driver finds each app by the socket -in its folder: the computer is ready once every app's server answers and each `boot` app, and each -app with no launch, says it's ready; a `demand` app counts once something launches it. What the -server does in the session (`agentenv_protocol.computer.session`) is Linux's. +desktop session, and `ready` (GET: an `AppReadiness` of `ready`, `problems`, `launched`, `waited_s` +and `timeout_s`), which runs the app's readiness checks then, at once, each within 10 s. A server +that can't start writes why to `.agentenv/server.failure` in the folder, and the driver reports that +in its place. Once the session is up the server adds the app's launcher to the session user's menu, +which runs `python -m agentenv_protocol.computer.launch ` to ask the server, so the server +starts every copy of the app, and launches a `boot` app, once per boot. The driver finds each app by +the socket in its folder: the computer is ready once every app's server answers and each `boot` app, +and each app with no launch, says it's ready; a `demand` app counts once something launches it. What +the server does in the session (`agentenv_protocol.computer.session`) is Linux's. `MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app` env stores, and `python -m agentenv_protocol.computer._declare ` prints it as diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py index e4c74f38e..488ffcc84 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -6,13 +6,9 @@ which launches the app, adds its launcher and says whether it's ready (``computer-app/v1``).""" from __future__ import annotations -import asyncio import inspect import json import os -import shlex -import time -from collections.abc import Awaitable from pathlib import Path from typing import Any, Callable, ClassVar, Literal, Optional @@ -20,7 +16,7 @@ from ..agent_env_environment import _EXT_ATTR, _OP_ATTR, AgentEnvEnvironment, extension from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, SKILL_EXTENSION_URI -from . import session +from .app_runtime import AppRuntime from .errors import InvalidArgument #: The environment variable a computer delivers its apps' settings in: JSON of each app's values by its name. @@ -35,10 +31,8 @@ #: Where an app's own server listens, in the app's folder, and where it says why it last failed to start. SOCKET = Path(".agentenv/server.sock") FAILURE = Path(".agentenv/server.failure") -#: Where an app's server marks that it launched the app since the computer booted, at boot or on demand. -LAUNCHED = Path("/run/agentenv/apps") -#: How long one readiness check may take; an app's checks run at once. -CHECK_TIMEOUT_S = 10.0 +#: How long an app may take to be ready when its launch doesn't say. +DEFAULT_TIMEOUT_S = 60.0 class Service(BaseModel): @@ -70,7 +64,7 @@ class Ready(BaseModel): window: bool = False port: Optional[int] = None command: Optional[list[str]] = None - timeout_s: float = 60 + timeout_s: float = DEFAULT_TIMEOUT_S @model_validator(mode="after") def _waits_on_something(self) -> Ready: @@ -173,6 +167,7 @@ class ComputerAppEnvironment(AgentEnvEnvironment): supports: ClassVar[list] = [] permissions: ClassVar[list[str]] = [] + _computer_app: Optional[AppRuntime] = None linux: ClassVar[Optional[Linux]] = None launch: ClassVar[Optional[Launch]] = None #: A file of the app's folder holding its skill for agents, when no ``@agent_skill`` method writes it. @@ -184,26 +179,11 @@ class Settings(BaseModel): def __init__(self) -> None: values = json.loads(os.environ.get(APP_SETTINGS_ENV) or "{}").get(os.environ.get("ENVIRONMENT_NAME"), {}) self.settings = self.Settings.model_validate(values) - self._computer_app_started_at = time.monotonic() - self._computer_app_launched_at: Optional[float] = None - self._computer_app_launch_failed: Optional[str] = None - #: Each sync ``@ready_check`` still running in its thread, by name, which a later poll waits on. - self._computer_app_checking: dict[str, asyncio.Future] = {} - - async def _computer_app_start(self) -> None: - """Once the desktop session is up, add the app's menu launcher and launch a boot app, once per boot: a server - that restarts finds the app launched, at boot or on demand, and leaves it be.""" - if self.launch is None: - return - await session.wait_until_up() - session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) - if _launched().exists(): - self._computer_app_launched_at = time.monotonic() - elif self.launch.at == "boot": - try: - await self._computer_app_launch() - except Exception as e: - self._computer_app_launch_failed = f"{type(e).__name__}: {e}" + ready = self.launch.ready if self.launch is not None else None + self._computer_app = AppRuntime( + self.launch, {fn.__name__: getattr(self, fn.__name__) for fn in _methods(type(self)) + if hasattr(fn, _READY_CHECK_ATTR)}, + ready.timeout_s if ready is not None else DEFAULT_TIMEOUT_S) @extension(COMPUTER_APP, op="launch", description=( "The app's own server: launch launches the app from its folder, as its launcher and a boot do; ready says " @@ -211,34 +191,16 @@ async def _computer_app_start(self) -> None: async def _computer_app_launch(self) -> dict: if self.launch is None: raise InvalidArgument(f"{type(self).__name__} declares no launch") - pid = await session.launch(self.launch.command, Path.cwd(), - {"ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}) - self._computer_app_launched_at, self._computer_app_launch_failed = time.monotonic(), None - _launched().parent.mkdir(parents=True, exist_ok=True) - _launched().touch() - return {"pid": pid} + return {"pid": await self._computer_app_runtime().launch_app()} @extension(COMPUTER_APP, op="ready", method="GET") async def _computer_app_ready(self) -> dict: - ready = self.launch.ready if self.launch is not None else None - checks: list[tuple[str, Callable[[], Awaitable[Optional[str]]]]] = [] - if ready is not None and ready.window: - checks.append(("window", lambda: _window(self.launch.window))) - if ready is not None and ready.port is not None: - checks.append(("port", lambda: _port(ready.port))) - if ready is not None and ready.command: - checks.append(("command", lambda: _command(ready.command))) - checks += [(f"@ready_check {fn.__name__}", lambda fn=fn: _ready_check(fn, self)) - for fn in _methods(type(self)) if hasattr(fn, _READY_CHECK_ATTR)] - found = await asyncio.gather(*(_bounded(name, check) for name, check in checks)) - problems = [problem for problem in found if problem is not None] - if self.launch is not None and self._computer_app_launched_at is None: - problems.insert(0, f"its launch failed: {self._computer_app_launch_failed}" - if self._computer_app_launch_failed else "not launched yet") - since = self._computer_app_launched_at or self._computer_app_started_at - return {"ready": not problems, "problems": problems, "launched": self._computer_app_launched_at is not None, - "waited_s": round(time.monotonic() - since, 1), - "timeout_s": ready.timeout_s if ready is not None else Ready.model_fields["timeout_s"].default} + return (await self._computer_app_runtime().readiness()).model_dump() + + def _computer_app_runtime(self) -> AppRuntime: + if self._computer_app is None: + raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") + return self._computer_app def extension_params(self, uri: str) -> dict: if uri == COMPUTER_APP: @@ -270,65 +232,9 @@ def declaration(cls) -> AppDeclaration: ) -def _launched() -> Path: - return LAUNCHED / Path.cwd().name - - def _methods(cls: type) -> list[Callable]: """The class's methods, each as the class resolves its name, in the order they were first defined.""" names = dict.fromkeys(name for klass in reversed(cls.__mro__) for name, member in vars(klass).items() if inspect.isfunction(member)) return [method for name in names if inspect.isfunction(method := getattr(cls, name))] - -async def _bounded(name: str, check: Callable[[], Awaitable[Optional[str]]]) -> Optional[str]: - """What ``check`` finds wrong, within ``CHECK_TIMEOUT_S``; a check that runs past it or raises says so.""" - try: - return await asyncio.wait_for(check(), CHECK_TIMEOUT_S) - except TimeoutError: - return f"{name} ran past {CHECK_TIMEOUT_S:g} s" - except Exception as e: - return f"{name}: {type(e).__name__}: {e}" - - -async def _window(match: dict[str, str]) -> Optional[str]: - if any(all(window.get(key) == value for key, value in match.items()) for window in await session.windows()): - return None - return f"no window with {', '.join(f'{key} {value!r}' for key, value in match.items())}" - - -async def _port(port: int) -> Optional[str]: - try: - _, writer = await asyncio.open_connection("127.0.0.1", port) - except OSError: - return f"nothing answers on port {port}" - writer.close() - return None - - -async def _command(argv: list[str]) -> Optional[str]: - process = await asyncio.create_subprocess_exec(*argv, cwd=Path.cwd(), stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.PIPE) - try: - _, err = await process.communicate() - except asyncio.CancelledError: - process.kill() - raise - if process.returncode == 0: - return None - reason = (err.decode(errors="replace").strip().splitlines() or [""])[-1] - return f"{shlex.join(argv)} exited {process.returncode}" + (f": {reason}" if reason else "") - - -async def _ready_check(fn: Callable, app: ComputerAppEnvironment) -> Optional[str]: - """A sync check runs in a thread, so it can't hold up the server, and once at a time: a thread can't be stopped, so - a poll while one runs past its bound waits on that same call.""" - if inspect.iscoroutinefunction(fn): - passed = await fn(app) - else: - running = app._computer_app_checking.get(fn.__name__) - if running is None or running.done(): - running = app._computer_app_checking[fn.__name__] = asyncio.ensure_future(asyncio.to_thread(fn, app)) - running.add_done_callback(lambda done: done.cancelled() or done.exception()) - passed = await asyncio.shield(running) - return None if passed else f"@ready_check {fn.__name__} returned {passed!r}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py new file mode 100644 index 000000000..d5f27370f --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app_runtime.py @@ -0,0 +1,155 @@ +"""A computer app's own server at work: it launches the app into the desktop session, marking each launch for the rest +of the boot, and says whether the app is ready, its checks run at once, each within ``CHECK_TIMEOUT_S`` +(``AppReadiness``). ``ComputerAppEnvironment`` keeps one as ``self._computer_app``.""" +from __future__ import annotations + +import asyncio +import inspect +import os +import shlex +import time +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional + +from pydantic import BaseModel, ConfigDict + +from . import session + +if TYPE_CHECKING: + from .app import Launch + +#: Where an app's server marks that it launched the app since the computer booted, at boot or on demand. +LAUNCHED = Path("/run/agentenv/apps") +#: How long one readiness check may take; an app's checks run at once. +CHECK_TIMEOUT_S = 10.0 + + +class AppReadiness(BaseModel): + """What an app's server says of its readiness, ``computer-app/v1``'s ``ready``: the ``problems`` that keep it from + being ``ready``, whether it was ``launched`` since the computer booted, and ``waited_s`` since then, or for an app + with no launch since its server started, against its ``timeout_s``.""" + + model_config = ConfigDict(extra="forbid") + + ready: bool + problems: list[str] + launched: bool + waited_s: float + timeout_s: float + + +class AppRuntime: + """An app's server's state: its ``launch``, its ``@ready_check`` methods by name, bound to the app, how long it may + take to be ready, why its last launch failed, and each sync check still running in its thread.""" + + def __init__(self, launch: Optional[Launch], ready_checks: dict[str, Callable[[], Any]], timeout_s: float) -> None: + self.launch = launch + self.ready_checks = ready_checks + self.timeout_s = timeout_s + self.started = time.time() + self.launch_failed: Optional[str] = None + self.checking: dict[str, asyncio.Future] = {} + + async def start(self) -> None: + """Once the desktop session is up, add the app's menu launcher and launch a boot app, unless the computer has + launched it since it booted, at boot or on demand: a server that restarts leaves the app be.""" + if self.launch is None: + return + await session.wait_until_up() + session.add_launcher(os.environ["ENVIRONMENT_NAME"], Path.cwd()) + if self.launch.at == "boot" and _launched_at() is None: + try: + await self.launch_app() + except Exception as e: + self.launch_failed = f"{type(e).__name__}: {e}" + + async def launch_app(self) -> int: + """Start the app from its folder, marked as launched; its pid.""" + pid = await session.launch(self.launch.command, Path.cwd(), {"ENVIRONMENT_NAME": os.environ["ENVIRONMENT_NAME"]}) + self.launch_failed = None + _mark().parent.mkdir(parents=True, exist_ok=True) + _mark().touch() + return pid + + async def readiness(self) -> AppReadiness: + ready = self.launch.ready if self.launch is not None else None + checks: list[tuple[str, Awaitable[Optional[str]]]] = [] + if ready is not None and ready.window: + checks.append(("window", _window(self.launch.window))) + if ready is not None and ready.port is not None: + checks.append(("port", _port(ready.port))) + if ready is not None and ready.command: + checks.append(("command", _command(ready.command))) + checks += [(f"@ready_check {name}", self._ready_check(name, fn)) for name, fn in self.ready_checks.items()] + found = await asyncio.gather(*(_bounded(name, check) for name, check in checks)) + problems = [problem for problem in found if problem is not None] + launched = _launched_at() + if self.launch is not None and launched is None: + problems.insert(0, f"its launch failed: {self.launch_failed}" if self.launch_failed else "not launched yet") + return AppReadiness(ready=not problems, problems=problems, launched=launched is not None, + waited_s=round(time.time() - (launched or self.started), 1), timeout_s=self.timeout_s) + + async def _ready_check(self, name: str, fn: Callable[[], Any]) -> Optional[str]: + """A sync check runs in a thread, so it can't hold up the server, and once at a time: a thread can't be + stopped, so a poll while one runs past its bound waits on that same call.""" + if inspect.iscoroutinefunction(fn): + passed = await fn() + else: + running = self.checking.get(name) + if running is None or running.done(): + running = self.checking[name] = asyncio.ensure_future(asyncio.to_thread(fn)) + running.add_done_callback(lambda done: done.cancelled() or done.exception()) + passed = await asyncio.shield(running) + return None if passed else f"@ready_check {name} returned {passed!r}" + + +def _mark() -> Path: + return LAUNCHED / Path.cwd().name + + +def _launched_at() -> Optional[float]: + """When the app was last launched since the computer booted, by its mark.""" + try: + return _mark().stat().st_mtime + except FileNotFoundError: + return None + + +async def _bounded(name: str, check: Awaitable[Optional[str]]) -> Optional[str]: + """What ``check`` finds wrong, within ``CHECK_TIMEOUT_S``; a check that runs past it or raises says so.""" + try: + return await asyncio.wait_for(check, CHECK_TIMEOUT_S) + except TimeoutError: + return f"{name} ran past {CHECK_TIMEOUT_S:g} s" + except Exception as e: + return f"{name}: {type(e).__name__}: {e}" + + +async def _window(match: dict[str, str]) -> Optional[str]: + if any(all(window.get(key) == value for key, value in match.items()) for window in await session.windows()): + return None + return f"no window with {', '.join(f'{key} {value!r}' for key, value in match.items())}" + + +async def _port(port: int) -> Optional[str]: + try: + _, writer = await asyncio.open_connection("127.0.0.1", port) + except OSError: + return f"nothing answers on port {port}" + writer.close() + return None + + +async def _command(argv: list[str]) -> Optional[str]: + process = await asyncio.create_subprocess_exec(*argv, cwd=Path.cwd(), stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE) + try: + _, err = await process.communicate() + except asyncio.CancelledError: + process.kill() + raise + if process.returncode == 0: + return None + reason = (err.decode(errors="replace").strip().splitlines() or [""])[-1] + return f"{shlex.join(argv)} exited {process.returncode}" + (f": {reason}" if reason else "") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py index 5b8e1ffce..56c41b567 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py @@ -9,8 +9,9 @@ import httpx from .. import client -from ..types import RPC_PATH, WELL_KNOWN_PATH -from .app import CHECK_TIMEOUT_S, COMPUTER_APP, FAILURE, SOCKET +from ..types import WELL_KNOWN_PATH +from .app import COMPUTER_APP, FAILURE, SOCKET +from .app_runtime import CHECK_TIMEOUT_S, AppReadiness from .operations import ReadyCheck #: Where a composed computer installs its apps, a folder each. @@ -33,16 +34,21 @@ async def check(folder: Path) -> ReadyCheck: card = (await http.get(WELL_KNOWN_PATH)).raise_for_status().json() name = card["name"] launch = client.extension_params(card, COMPUTER_APP).get("launch") - answer = (await http.get(f"{RPC_PATH}/ext/computer-app/ready")).raise_for_status().json() + answer = AppReadiness.model_validate((await http.get(endpoint(card, "ready"))).raise_for_status().json()) except (httpx.HTTPError, OSError): return ReadyCheck(f"app {name}", False, _unanswered(folder)) - if launch is not None and launch["at"] == "demand" and not answer["launched"]: + if launch is not None and launch["at"] == "demand" and not answer.launched: return ReadyCheck(f"app {name}", True, "its server answers; it launches on demand") - if answer["ready"]: + if answer.ready: return ReadyCheck(f"app {name}", True, "ready") - when = (f"not ready after {answer['timeout_s']:.0f} s" if answer["waited_s"] > answer["timeout_s"] - else f"not ready yet, {answer['waited_s']:.0f} s in") - return ReadyCheck(f"app {name}", False, f"{when}: {'; '.join(answer['problems'])}") + when = (f"not ready after {answer.timeout_s:.0f} s" if answer.waited_s > answer.timeout_s + else f"not ready yet, {answer.waited_s:.0f} s in") + return ReadyCheck(f"app {name}", False, f"{when}: {'; '.join(answer.problems)}") + + +def endpoint(card: dict, method: str) -> str: + """The path of a ``computer-app/v1`` method, as the app's card advertises it.""" + return client.find_extension_method(card, COMPUTER_APP, method)["endpoint"] def _unanswered(folder: Path) -> str: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/launch.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/launch.py index 776044941..0232cf6c7 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/launch.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/launch.py @@ -8,15 +8,17 @@ import httpx -from ..types import RPC_PATH +from ..types import WELL_KNOWN_PATH from .app import SOCKET +from .apps import endpoint def main(argv: list[str]) -> None: if len(argv) != 1: raise SystemExit("usage: python -m agentenv_protocol.computer.launch ") with httpx.Client(transport=httpx.HTTPTransport(uds=str(Path(argv[0]) / SOCKET)), base_url="http://app") as http: - http.post(f"{RPC_PATH}/ext/computer-app/launch", json={}).raise_for_status() + card = http.get(WELL_KNOWN_PATH).raise_for_status().json() + http.post(endpoint(card, "launch"), json={}).raise_for_status() if __name__ == "__main__": diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py index e7b2e51af..963471e89 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py @@ -1,7 +1,7 @@ """Serve a computer app's class from its file, as its computer's program starts it: ``python -m agentenv_protocol.computer.serve_app :``, run in the app's folder, serves the class's card and ``computer-app/v1`` on the Unix socket ``.agentenv/server.sock`` there. Once it serves, the app's server adds the app's -launcher and launches a boot app (``ComputerAppEnvironment._computer_app_start``). A server that can't start writes why +launcher and launches a boot app (``AppRuntime.start``). A server that can't start writes why to ``.agentenv/server.failure``, which its computer's driver reports. Needs the ``app`` extra.""" from __future__ import annotations @@ -33,13 +33,14 @@ def _serve(argv: list[str]) -> None: raise SystemExit(USAGE) file, _, name = argv[0].partition(":") app = load_class(Path(file), name, ComputerAppEnvironment)() + runtime = app._computer_app_runtime() asgi = app.create_app().streamable_http_app() serving = asgi.router.lifespan_context @contextlib.asynccontextmanager async def lifespan(scope): async with serving(scope): - started = asyncio.ensure_future(app._computer_app_start()) + started = asyncio.ensure_future(runtime.start()) yield started.cancel() diff --git a/packages/agentenv-protocol/tests/test_computer_app_server.py b/packages/agentenv-protocol/tests/test_computer_app_server.py index 7d1cd0db5..9b87d832d 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_server.py +++ b/packages/agentenv-protocol/tests/test_computer_app_server.py @@ -24,21 +24,9 @@ import agentenv_protocol.computer.launch as launcher # noqa: E402 from agentenv_protocol import client # noqa: E402 from agentenv_protocol.computer import ComputerAppEnvironment, Launch, ReadyCheck, Ready, ready_check # noqa: E402 -from agentenv_protocol.computer import app as app_module, apps, serve_app, session # noqa: E402 +from agentenv_protocol.computer import app_runtime, apps, session # noqa: E402 from agentenv_protocol.computer.app import COMPUTER_APP, FAILURE, SOCKET # noqa: E402 -READY = "/agentenv/ext/computer-app/ready" -APP = '''from agentenv_protocol.computer import ComputerAppEnvironment - - -class Helper: - pass - - -class Broken(ComputerAppEnvironment): - def __init__(self): - raise RuntimeError("no mailbox") -''' class Mail(ComputerAppEnvironment): @@ -78,7 +66,7 @@ async def up(): monkeypatch.setattr(session, "windows", windows) monkeypatch.setattr(session, "launch", launch) monkeypatch.setattr(session, "wait_until_up", up) - monkeypatch.setattr(app_module, "LAUNCHED", tmp_path / "launched") + monkeypatch.setattr(app_runtime, "LAUNCHED", tmp_path / "launched") monkeypatch.setenv("ENVIRONMENT_NAME", "mail") return shown, launched @@ -89,12 +77,13 @@ async def test_an_apps_server_is_ready_once_it_launched_the_app_and_each_check_p mail = Mail() async with _http(mail) as http: card = (await http.get("/.well-known/agent-env.json")).json() - before = (await http.get(READY)).json() - assert (await http.post("/agentenv/ext/computer-app/launch", json={})).json() == {"pid": 4242} + ready = apps.endpoint(card, "ready") + before = (await http.get(ready)).json() + assert (await http.post(apps.endpoint(card, "launch"), json={})).json() == {"pid": 4242} shown.append({"instance": "mail", "class": "Mail", "title": "Inbox"}) - unsynced = (await http.get(READY)).json() + unsynced = (await http.get(ready)).json() mail.synced = True - after = (await http.get(READY)).json() + after = (await http.get(ready)).json() assert client.extension_params(card, COMPUTER_APP)["launch"] == { "at": "boot", "command": ["mail"], "window": {"class": "Mail"}, "ready": {"window": True, "timeout_s": 30.0}} @@ -108,7 +97,7 @@ async def test_an_apps_server_is_ready_once_it_launched_the_app_and_each_check_p @pytest.mark.asyncio async def test_checks_that_fail_hang_or_raise_say_so_at_once_without_holding_up_the_server(desktop, monkeypatch): - monkeypatch.setattr(app_module, "CHECK_TIMEOUT_S", 1.0) + monkeypatch.setattr(app_runtime, "CHECK_TIMEOUT_S", 1.0) with socket.socket() as closed: closed.bind(("127.0.0.1", 0)) port = closed.getsockname()[1] @@ -141,19 +130,21 @@ async def broken(self): @pytest.mark.asyncio -async def test_a_boot_app_is_launched_once_per_boot_with_its_launcher_added_each_start(desktop, tmp_path, monkeypatch): +async def test_a_boot_app_is_launched_once_per_boot_its_clock_from_that_launch_across_restarts(desktop, tmp_path, monkeypatch): _, launched = desktop launchers = [] monkeypatch.setattr(session, "add_launcher", lambda name, folder: launchers.append((name, folder))) monkeypatch.chdir(tmp_path) first, restarted = Mail(), Mail() - await first._computer_app_start() - await restarted._computer_app_start() + await first._computer_app.start() + os.utime(tmp_path / "launched" / tmp_path.name, (time.time() - 100,) * 2) + await restarted._computer_app.start() assert len(launched) == 1 and launchers == [("mail", tmp_path)] * 2 assert (tmp_path / "launched" / tmp_path.name).exists() - assert (await restarted._computer_app_ready())["launched"] is True + answer = await restarted._computer_app_ready() + assert answer["launched"] is True and answer["waited_s"] >= 100 @pytest.mark.asyncio @@ -163,16 +154,16 @@ async def test_a_restarted_server_finds_the_demand_app_it_launched_and_leaves_it monkeypatch.setenv("ENVIRONMENT_NAME", "music") opened, restarted = Music(), Music() - await opened._computer_app_start() + await opened._computer_app.start() await opened._computer_app_launch() - await restarted._computer_app_start() + await restarted._computer_app.start() assert len(launched) == 1 and (await restarted._computer_app_ready())["launched"] is True @pytest.mark.asyncio async def test_a_sync_check_past_its_bound_runs_once_however_often_readiness_is_asked(desktop, monkeypatch): - monkeypatch.setattr(app_module, "CHECK_TIMEOUT_S", 0.2) + monkeypatch.setattr(app_runtime, "CHECK_TIMEOUT_S", 0.2) released, calls = threading.Event(), [] class Syncing(ComputerAppEnvironment): @@ -184,7 +175,7 @@ def synced(self): syncing = Syncing() stuck = [(await syncing._computer_app_ready())["problems"] for _ in range(3)] released.set() - while not syncing._computer_app_checking["synced"].done(): + while not syncing._computer_app.checking["synced"].done(): await asyncio.sleep(0.01) after = await syncing._computer_app_ready() @@ -201,7 +192,7 @@ async def missing(argv, cwd, env): monkeypatch.setattr(session, "add_launcher", lambda name, folder: None) monkeypatch.setattr(session, "launch", missing) mail = Mail() - await mail._computer_app_start() + await mail._computer_app.start() assert (await mail._computer_app_ready())["problems"][0] == ( "its launch failed: FileNotFoundError: No such file or directory: 'mail'") @@ -220,7 +211,7 @@ async def test_the_driver_asks_each_apps_server_on_the_socket_in_its_folder(desk served = {name: await stack.enter_async_context(_serving(cls(), Path(root) / name, monkeypatch)) for name, cls in [("mail", Mail), ("music", Music), ("sync", Sync)]} first = [await apps.check(folder) for folder in apps.expected()] - served["mail"]._computer_app_started_at -= 120 + served["mail"]._computer_app.started -= 120 late = await apps.check(Path(root) / "mail") monkeypatch.setenv("ENVIRONMENT_NAME", "music") await asyncio.to_thread(launcher.main, [str(Path(root) / "music")]) @@ -242,48 +233,6 @@ async def test_the_driver_asks_each_apps_server_on_the_socket_in_its_folder(desk assert playing == ReadyCheck("app music", True, "ready") -@pytest.mark.asyncio -async def test_serving_an_app_starts_it_once_the_server_serves_on_the_socket_in_its_folder(tmp_path, monkeypatch): - (tmp_path / "app.py").write_text("from agentenv_protocol.computer import ComputerAppEnvironment\n\n\n" - "class Notes(ComputerAppEnvironment):\n pass\n") - (tmp_path / SOCKET).parent.mkdir() - (tmp_path / SOCKET).write_text("left by the last server") - (tmp_path / FAILURE).write_text("RuntimeError: the last start failed") - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(sys, "path", list(sys.path)) - monkeypatch.setattr(sys, "modules", dict(sys.modules)) - started, ran = [], [] - monkeypatch.setattr(ComputerAppEnvironment, "_computer_app_start", lambda self: _record(started, type(self))) - monkeypatch.setattr(serve_app.uvicorn, "run", lambda asgi, uds, log_level: ran.append((asgi, uds))) - - serve_app.main(["app.py:Notes"]) - (asgi, uds), = ran - async with asgi.router.lifespan_context(asgi): - await asyncio.sleep(0) - - assert uds == str(SOCKET) and not (tmp_path / SOCKET).exists() and not (tmp_path / FAILURE).exists() - assert [cls.__name__ for cls in started] == ["Notes"] - - -@pytest.mark.parametrize("argv, failure", [ - ([], f"SystemExit: {serve_app.USAGE}"), (["app.py"], f"SystemExit: {serve_app.USAGE}"), - (["missing.py:Notes"], "SystemExit: missing.py isn't a file"), - (["app.py:Helper"], "SystemExit: app.py has no ComputerAppEnvironment subclass named Helper"), - (["app.py:Broken"], "RuntimeError: no mailbox"), -], ids=["nothing", "no-class", "no-file", "not-an-app", "fails-to-construct"]) -def test_a_server_that_cant_start_says_why_in_the_apps_folder(tmp_path, monkeypatch, argv, failure): - (tmp_path / "app.py").write_text(APP) - (tmp_path / SOCKET).parent.mkdir() - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(sys, "path", list(sys.path)) - monkeypatch.setattr(sys, "modules", dict(sys.modules)) - - with pytest.raises((SystemExit, RuntimeError)) as stopped: - serve_app.main(argv) - - assert (tmp_path / FAILURE).read_text() == failure == f"{type(stopped.value).__name__}: {stopped.value}" - - @pytest.mark.asyncio async def test_the_drivers_readiness_asks_every_apps_server_at_once(desktop, monkeypatch): pytest.importorskip("PIL") @@ -386,7 +335,3 @@ async def _serving(app: ComputerAppEnvironment, folder: Path, monkeypatch): finally: server.should_exit = True await serving - - -async def _record(started: list, cls: type) -> None: - started.append(cls) diff --git a/packages/agentenv-protocol/tests/test_computer_serve.py b/packages/agentenv-protocol/tests/test_computer_serve.py index bcf5e0b23..e31106c1b 100644 --- a/packages/agentenv-protocol/tests/test_computer_serve.py +++ b/packages/agentenv-protocol/tests/test_computer_serve.py @@ -1,15 +1,19 @@ -"""Serving a driver from its file: the class is imported with its folder importable and served; anything that isn't -a driver class exits saying why.""" +"""Serving a driver or an app from its file: the class is imported with its folder importable and served; anything +that isn't such a class exits saying why, and an app's server that can't start says why in the app's folder.""" from __future__ import annotations +import asyncio import sys import pytest pytest.importorskip("mcp.server.fastmcp") pytest.importorskip("PIL") +pytest.importorskip("uvicorn") -from agentenv_protocol.computer import ComputerDriverEnvironment # noqa: E402 +from agentenv_protocol.computer import ComputerDriverEnvironment, serve_app # noqa: E402 +from agentenv_protocol.computer.app import FAILURE, SOCKET # noqa: E402 +from agentenv_protocol.computer.app_runtime import AppRuntime # noqa: E402 from agentenv_protocol.computer.serve import USAGE, main # noqa: E402 DRIVER = '''from desk_helpers import SIZE @@ -43,12 +47,39 @@ def press(self, keys): ... ''' +APP = '''from agentenv_protocol.computer import ComputerAppEnvironment + + +class Helper: + pass + + +class Notes(ComputerAppEnvironment): + pass + + +class Broken(ComputerAppEnvironment): + def __init__(self): + raise RuntimeError("no mailbox") + + +class Skipping(ComputerAppEnvironment): + def __init__(self): + pass +''' + + @pytest.fixture -def served(monkeypatch): - calls = [] - monkeypatch.setattr(ComputerDriverEnvironment, "serve", lambda self: calls.append(type(self).__name__)) +def isolated(monkeypatch): + """Imports that a class's file makes, undone after the test.""" monkeypatch.setattr(sys, "path", list(sys.path)) monkeypatch.setattr(sys, "modules", dict(sys.modules)) + + +@pytest.fixture +def served(monkeypatch, isolated): + calls = [] + monkeypatch.setattr(ComputerDriverEnvironment, "serve", lambda self: calls.append(type(self).__name__)) return calls @@ -73,3 +104,47 @@ def test_what_isnt_a_driver_class_exits_saying_why(tmp_path, served, monkeypatch main(argv) assert message in str(stopped.value) and served == [] + + +@pytest.mark.asyncio +async def test_an_apps_server_serves_on_the_socket_in_its_folder_and_starts_the_app_once_it_serves(tmp_path, + monkeypatch, + isolated): + (tmp_path / "app.py").write_text(APP) + (tmp_path / SOCKET).parent.mkdir() + (tmp_path / SOCKET).write_text("left by the last server") + (tmp_path / FAILURE).write_text("RuntimeError: the last start failed") + monkeypatch.chdir(tmp_path) + started, ran = [], [] + + async def start(runtime): + started.append(runtime) + + monkeypatch.setattr(AppRuntime, "start", start) + monkeypatch.setattr(serve_app.uvicorn, "run", lambda asgi, uds, log_level: ran.append((asgi, uds))) + + serve_app.main(["app.py:Notes"]) + (asgi, uds), = ran + async with asgi.router.lifespan_context(asgi): + await asyncio.sleep(0) + + assert uds == str(SOCKET) and not (tmp_path / SOCKET).exists() and not (tmp_path / FAILURE).exists() + assert len(started) == 1 and isinstance(started[0], AppRuntime) + + +@pytest.mark.parametrize("argv, failure", [ + ([], f"SystemExit: {serve_app.USAGE}"), (["app.py"], f"SystemExit: {serve_app.USAGE}"), + (["missing.py:Notes"], "SystemExit: missing.py isn't a file"), + (["app.py:Helper"], "SystemExit: app.py has no ComputerAppEnvironment subclass named Helper"), + (["app.py:Broken"], "RuntimeError: no mailbox"), + (["app.py:Skipping"], "TypeError: Skipping.__init__ must call super().__init__()"), +], ids=["nothing", "no-class", "no-file", "not-an-app", "fails-to-construct", "skips-super-init"]) +def test_an_apps_server_that_cant_start_says_why_in_the_apps_folder(tmp_path, monkeypatch, isolated, argv, failure): + (tmp_path / "app.py").write_text(APP) + (tmp_path / SOCKET).parent.mkdir() + monkeypatch.chdir(tmp_path) + + with pytest.raises((SystemExit, RuntimeError, TypeError)) as stopped: + serve_app.main(argv) + + assert (tmp_path / FAILURE).read_text() == failure == f"{type(stopped.value).__name__}: {stopped.value}" From 6ecb0f45b4af91e5e99c49c6931e4def068e2911 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 12:18:11 -0700 Subject: [PATCH 6/6] fix(computer): a failed app start stops its server, and WM_NAME titles match The app server's start in the desktop session ran as a background task whose error nobody read, so a launcher it couldn't add left readiness at "not launched yet" until the deploy timed out. A start that fails now stops the server, which writes the error to .agentenv/server.failure and exits, and the driver reports it as for any server that can't start, a demand app's too. A failed launch is still reported by readiness. The session reads a window's title from WM_NAME when it sets no _NET_WM_NAME, as older X11 apps do. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 15 +++--- .../agentenv_protocol/computer/serve_app.py | 19 +++++-- .../src/agentenv_protocol/computer/session.py | 9 ++-- .../tests/test_computer_app_server.py | 16 +++--- .../tests/test_computer_serve.py | 50 +++++++++++++++---- 5 files changed, 78 insertions(+), 31 deletions(-) diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index c183c11bb..c7be5820a 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -587,13 +587,14 @@ card and tools on the Unix socket `.agentenv/server.sock` in that folder, with ` the app's `launch` as its params, and the methods `launch` (POST), which starts the app in the desktop session, and `ready` (GET: an `AppReadiness` of `ready`, `problems`, `launched`, `waited_s` and `timeout_s`), which runs the app's readiness checks then, at once, each within 10 s. A server -that can't start writes why to `.agentenv/server.failure` in the folder, and the driver reports that -in its place. Once the session is up the server adds the app's launcher to the session user's menu, -which runs `python -m agentenv_protocol.computer.launch ` to ask the server, so the server -starts every copy of the app, and launches a `boot` app, once per boot. The driver finds each app by -the socket in its folder: the computer is ready once every app's server answers and each `boot` app, -and each app with no launch, says it's ready; a `demand` app counts once something launches it. What -the server does in the session (`agentenv_protocol.computer.session`) is Linux's. +that can't start, or whose start in the session fails (adding the launcher, say), writes why to +`.agentenv/server.failure` in the folder and exits, and the driver reports that in its place. Once +the session is up the server adds the app's launcher to the session user's menu, which runs `python +-m agentenv_protocol.computer.launch ` to ask the server, so the server starts every copy of +the app, and launches a `boot` app, once per boot. The driver finds each app by the socket in its +folder: the computer is ready once every app's server answers and each `boot` app, and each app with +no launch, says it's ready; a `demand` app counts once something launches it. What the server does +in the session (`agentenv_protocol.computer.session`) is Linux's. `MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app` env stores, and `python -m agentenv_protocol.computer._declare ` prints it as diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py index 963471e89..e7367c42c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/serve_app.py @@ -1,8 +1,8 @@ """Serve a computer app's class from its file, as its computer's program starts it: ``python -m agentenv_protocol.computer.serve_app :``, run in the app's folder, serves the class's card and ``computer-app/v1`` on the Unix socket ``.agentenv/server.sock`` there. Once it serves, the app's server adds the app's -launcher and launches a boot app (``AppRuntime.start``). A server that can't start writes why -to ``.agentenv/server.failure``, which its computer's driver reports. Needs the ``app`` extra.""" +launcher and launches a boot app (``AppRuntime.start``). A server that can't start, or whose start fails once it serves, +writes why to ``.agentenv/server.failure``, which its computer's driver reports, and exits. Needs the ``app`` extra.""" from __future__ import annotations import asyncio @@ -29,24 +29,35 @@ def main(argv: list[str]) -> None: def _serve(argv: list[str]) -> None: + """Serve the class ``argv`` names, starting the app once it serves; a start that fails stops the server and is + raised once it has.""" if len(argv) != 1 or ":" not in argv[0]: raise SystemExit(USAGE) file, _, name = argv[0].partition(":") app = load_class(Path(file), name, ComputerAppEnvironment)() runtime = app._computer_app_runtime() asgi = app.create_app().streamable_http_app() - serving = asgi.router.lifespan_context + server = uvicorn.Server(uvicorn.Config(asgi, uds=str(SOCKET), log_level="warning")) + serving, failed = asgi.router.lifespan_context, [] + + def stop(task: asyncio.Task) -> None: + if not task.cancelled() and task.exception() is not None: + failed.append(task.exception()) + server.should_exit = True @contextlib.asynccontextmanager async def lifespan(scope): async with serving(scope): started = asyncio.ensure_future(runtime.start()) + started.add_done_callback(stop) yield started.cancel() asgi.router.lifespan_context = lifespan SOCKET.unlink(missing_ok=True) - uvicorn.run(asgi, uds=str(SOCKET), log_level="warning") + server.run() + if failed: + raise failed[0] if __name__ == "__main__": diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py index 18da07594..3102ff8a5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/session.py @@ -44,15 +44,16 @@ def add_launcher(name: str, folder: Path) -> Path: async def windows() -> list[dict[str, str]]: """The window manager's top-level windows, each as its WM_CLASS ``instance`` and ``class`` and its ``title``, read - with ``xprop``; a window that closes while it's read is left out.""" + with ``xprop``: ``_NET_WM_NAME``, or ``WM_NAME`` for a window that sets only that; a window that closes while it's + read is left out.""" listed = await _xprop("-root", "_NET_CLIENT_LIST") found = [] for window in re.findall(r"0x[0-9a-f]+", listed.partition("#")[2]): - props = await _xprop("-id", window, "WM_CLASS", "_NET_WM_NAME") - names = dict(re.findall(r'^(WM_CLASS|_NET_WM_NAME)[^=]*= (.*)$', props, re.MULTILINE)) + props = await _xprop("-id", window, "WM_CLASS", "_NET_WM_NAME", "WM_NAME") + names = dict(re.findall(r'^(WM_CLASS|_NET_WM_NAME|WM_NAME)[^=]*= (.*)$', props, re.MULTILINE)) parts = _strings(names.get("WM_CLASS", "")) if len(parts) == 2: - title = _strings(names.get("_NET_WM_NAME", "")) + title = _strings(names.get("_NET_WM_NAME") or names.get("WM_NAME", "")) found.append({"instance": parts[0], "class": parts[1], "title": title[0] if title else ""}) return found diff --git a/packages/agentenv-protocol/tests/test_computer_app_server.py b/packages/agentenv-protocol/tests/test_computer_app_server.py index 9b87d832d..aaf9c0fec 100644 --- a/packages/agentenv-protocol/tests/test_computer_app_server.py +++ b/packages/agentenv-protocol/tests/test_computer_app_server.py @@ -283,17 +283,21 @@ def test_the_launcher_runs_the_app_through_its_own_server(tmp_path, monkeypatch) @pytest.mark.asyncio -async def test_the_sessions_windows_are_listed_by_class_instance_and_title(monkeypatch): - replies = {("-root", "_NET_CLIENT_LIST"): "_NET_CLIENT_LIST(WINDOW): window id # 0x1e00004, 0x2000003\n", - ("-id", "0x1e00004", "WM_CLASS", "_NET_WM_NAME"): - '_NET_WM_NAME(UTF8_STRING) = "Inbox \\"work\\""\nWM_CLASS(STRING) = "mail", "Mail"\n', - ("-id", "0x2000003", "WM_CLASS", "_NET_WM_NAME"): ""} +async def test_the_sessions_windows_are_listed_by_class_instance_and_title_from_either_name(monkeypatch): + replies = {("-root", "_NET_CLIENT_LIST"): "_NET_CLIENT_LIST(WINDOW): window id # 0x1e00004, 0x2000003, 0x2200001\n", + ("-id", "0x1e00004", "WM_CLASS", "_NET_WM_NAME", "WM_NAME"): + '_NET_WM_NAME(UTF8_STRING) = "Inbox \\"work\\""\nWM_CLASS(STRING) = "mail", "Mail"\n' + 'WM_NAME(STRING) = "Inbox"\n', + ("-id", "0x2000003", "WM_CLASS", "_NET_WM_NAME", "WM_NAME"): "", + ("-id", "0x2200001", "WM_CLASS", "_NET_WM_NAME", "WM_NAME"): + 'WM_CLASS(STRING) = "xcalc", "XCalc"\n_NET_WM_NAME: not found.\nWM_NAME(STRING) = "Calculator"\n'} async def xprop(*args): return replies[args] monkeypatch.setattr(session, "_xprop", xprop) - assert await session.windows() == [{"instance": "mail", "class": "Mail", "title": 'Inbox "work"'}] + assert await session.windows() == [{"instance": "mail", "class": "Mail", "title": 'Inbox "work"'}, + {"instance": "xcalc", "class": "XCalc", "title": "Calculator"}] @pytest.mark.asyncio diff --git a/packages/agentenv-protocol/tests/test_computer_serve.py b/packages/agentenv-protocol/tests/test_computer_serve.py index e31106c1b..8c7662cc5 100644 --- a/packages/agentenv-protocol/tests/test_computer_serve.py +++ b/packages/agentenv-protocol/tests/test_computer_serve.py @@ -106,32 +106,62 @@ def test_what_isnt_a_driver_class_exits_saying_why(tmp_path, served, monkeypatch assert message in str(stopped.value) and served == [] -@pytest.mark.asyncio -async def test_an_apps_server_serves_on_the_socket_in_its_folder_and_starts_the_app_once_it_serves(tmp_path, - monkeypatch, - isolated): +@pytest.fixture +def app_folder(tmp_path, monkeypatch, isolated): + """An app's folder, holding ``APP`` and what its last server left, whose server only runs its app's lifespan, as + uvicorn runs it around serving; each server is recorded.""" (tmp_path / "app.py").write_text(APP) (tmp_path / SOCKET).parent.mkdir() (tmp_path / SOCKET).write_text("left by the last server") (tmp_path / FAILURE).write_text("RuntimeError: the last start failed") monkeypatch.chdir(tmp_path) - started, ran = [], [] + ran = [] + + def run(server): + async def lifespan(): + async with server.config.app.router.lifespan_context(server.config.app): + await asyncio.sleep(0.05) + + asyncio.run(lifespan()) + ran.append(server) + + monkeypatch.setattr(serve_app.uvicorn.Server, "run", run) + return tmp_path, ran + + +def test_an_apps_server_serves_on_the_socket_in_its_folder_and_starts_the_app_once_it_serves(app_folder, + monkeypatch): + folder, ran = app_folder + started = [] async def start(runtime): started.append(runtime) monkeypatch.setattr(AppRuntime, "start", start) - monkeypatch.setattr(serve_app.uvicorn, "run", lambda asgi, uds, log_level: ran.append((asgi, uds))) serve_app.main(["app.py:Notes"]) - (asgi, uds), = ran - async with asgi.router.lifespan_context(asgi): - await asyncio.sleep(0) - assert uds == str(SOCKET) and not (tmp_path / SOCKET).exists() and not (tmp_path / FAILURE).exists() + (server,) = ran + assert server.config.uds == str(SOCKET) and not server.should_exit + assert not (folder / SOCKET).exists() and not (folder / FAILURE).exists() assert len(started) == 1 and isinstance(started[0], AppRuntime) +def test_an_apps_server_whose_start_fails_stops_saying_why_in_the_apps_folder(app_folder, monkeypatch): + folder, ran = app_folder + + async def start(runtime): + raise PermissionError("~/.local/share/applications isn't writable") + + monkeypatch.setattr(AppRuntime, "start", start) + + with pytest.raises(PermissionError): + serve_app.main(["app.py:Notes"]) + + assert ran[0].should_exit is True + assert (folder / FAILURE).read_text() == "PermissionError: ~/.local/share/applications isn't writable" + + @pytest.mark.parametrize("argv, failure", [ ([], f"SystemExit: {serve_app.USAGE}"), (["app.py"], f"SystemExit: {serve_app.USAGE}"), (["missing.py:Notes"], "SystemExit: missing.py isn't a file"),