diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee2e8dd62..419c8c1d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: run: uv python install ${{ matrix.python-version }} - name: Run tests - run: uv run --python ${{ matrix.python-version }} --all-extras pytest --cov --cov-report='' + run: uv run --python ${{ matrix.python-version }} --all-extras pytest -n auto --cov --cov-report='' lint-ruff: runs-on: ubuntu-latest @@ -41,6 +41,9 @@ jobs: uv run --with=".[dev]" ruff format . --check uv run --with=".[dev]" ruff check . + - name: Check tests stay black-box + run: uv run --no-project python scripts/check_tests.py + lint-ty: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..c8073021d --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,156 @@ +name: E2E + +# Tests marked `e2e` need something outside the test process. Each job selects +# the ones its runner can provide; they skip themselves when it cannot. + +on: + pull_request: + paths: + - "hud/**" + - "tests/**" + - "environments/**" + - "pyproject.toml" + - ".github/workflows/e2e.yml" + push: + branches: [main] + schedule: + - cron: "17 7 * * *" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + docker: + name: Docker + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + # Ubuntu 24.04 runners block unprivileged user namespaces through AppArmor; + # environments' bubblewrap sandboxes need them, as they have on HUD's hosts. + - name: Allow unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Run Docker scenarios + run: >- + uv run --python 3.12 --all-extras + pytest -m "e2e and docker" -n 4 --timeout 1800 --durations 20 + + sandbox: + name: Sandbox + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + # Ubuntu 24.04 runners block unprivileged user namespaces through AppArmor; + # environments' bubblewrap sandboxes need them, as they have on HUD's hosts. + - name: Allow unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + # Environments run in containers with HUD's seccomp profile, so the + # sandbox scenarios run there too, as root, with bubblewrap installed. + - name: Run sandbox scenarios in a container + run: >- + docker run --rm + --security-opt seccomp=hud/eval/docker-seccomp.json + --security-opt systempaths=unconfined + --security-opt apparmor=unconfined + --volume "$PWD:/src" --workdir /src + --env UV_PROJECT_ENVIRONMENT=/opt/venv + python:3.12-bookworm + sh -c 'apt-get update -qq && apt-get install -y -qq bubblewrap openssh-client >/dev/null + && pip install -q "uv>=0.12.2" + && uv run --all-extras pytest -m "e2e and sandbox" -n 4 --timeout 600' + + live: + name: Live + # Secrets are not available to pull requests from forks or Dependabot. + if: >- + github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login != 'dependabot[bot]') + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + HUD_API_KEY: ${{ secrets.HUD_API_KEY }} + HUD_CI_DATA_FILE_ID: ${{ vars.HUD_CI_DATA_FILE_ID }} + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + # The CLI agents' live scenarios run the real Claude Code and Codex CLIs. + - name: Install the agent CLIs + run: npm install --global @anthropic-ai/claude-code @openai/codex + + - name: Run live scenarios + run: >- + uv run --python 3.12 --all-extras + pytest -m "e2e and live and not hosted" -n 4 --timeout 1200 --durations 20 + + modal: + name: Modal + # Runs real Modal sandboxes on the account in MODAL_TOKEN_ID / MODAL_TOKEN_SECRET; + # without them the scenarios skip themselves. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + - name: Run Modal scenarios + run: >- + uv run --python 3.12 --all-extras + pytest -m "e2e and modal" -n 4 --timeout 1200 --durations 20 + + lerobot: + name: LeRobot + # Records robot rollouts with the real lerobot package (and torch), which no + # extra installs, so it runs on its own schedule. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + - name: Run LeRobot scenarios + run: >- + uv run --python 3.12 --all-extras --with 'lerobot[dataset]' + pytest -m "e2e and lerobot" -n 4 --timeout 600 --durations 20 + + cookbooks: + name: Cookbooks + # Cookbooks live outside the package, so SDK changes can break them silently. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + - name: Check cookbooks + run: >- + uv run --python 3.12 --all-extras + pytest tests/e2e/test_cookbooks.py -m e2e -n 4 --timeout 1200 --durations 20 diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 26bb57909..8cdcaa509 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -1,5 +1,8 @@ name: Pre-release +# The release gate (release.yml) looks up a successful run of this workflow on the +# tagged commit, so keep its file name and trigger. + on: workflow_dispatch: @@ -15,8 +18,6 @@ jobs: concurrency: group: ci-example-${{ matrix.template }} cancel-in-progress: false - env: - EXAMPLE_NAME: ci-${{ matrix.template }} strategy: fail-fast: false matrix: @@ -27,53 +28,30 @@ jobs: uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Create example environment - run: uv run hud init "$RUNNER_TEMP/$EXAMPLE_NAME" --template "${{ matrix.template }}" - - - name: Install example with the SDK from this checkout - working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }} - run: | - uv build --wheel --no-create-gitignore --out-dir .hud-sdk "$GITHUB_WORKSPACE" - uv add --no-sync .hud-sdk/hud-*.whl - uv sync --all-extras - sed -i '/^RUN .*uv sync/i COPY .hud-sdk .hud-sdk' Dockerfile.hud - echo '!.hud-sdk/' >> .dockerignore - echo '!.hud-sdk/**' >> .dockerignore - - - name: Run example checks - working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }} - run: | - uv run --no-sync hud task list --source . - if [ -d tests ]; then - uv run --no-sync ruff format . --check - uv run --no-sync ruff check . - uv run --no-sync pytest -q - fi + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - - name: Deploy example environment - working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }} + # The hosted test skips without credentials, which must not pass the gate. + - name: Require credentials env: HUD_API_KEY: ${{ secrets.HUD_API_KEY }} - run: uv run --no-sync hud deploy --no-env + run: test -n "$HUD_API_KEY" - - name: Run hosted task lifecycle - working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }} + # hud init, install this checkout's SDK, deploy, run one rollout on the HUD + # runtime with a live model, check its lifecycle, and read the trace back. + - name: Deploy and run the template on HUD env: HUD_API_KEY: ${{ secrets.HUD_API_KEY }} HUD_CI_DATA_FILE_ID: ${{ vars.HUD_CI_DATA_FILE_ID }} - HUD_TELEMETRY_LOCAL_DIR: ${{ runner.temp }}/traces-${{ matrix.template }} - run: | - uv run --no-sync "$GITHUB_WORKSPACE/scripts/example_lifecycle.py" prepare "${{ matrix.template }}" - timeout --signal=INT --kill-after=30s 15m uv run --no-sync hud eval .hud/ci-tasks.json claude \ - --model claude-sonnet-4-6 --gateway --runtime hud --max-steps 10 --config max_tokens=2048 -y - uv run --no-sync "$GITHUB_WORKSPACE/scripts/example_lifecycle.py" check + HUD_E2E_TRACE_DIR: ${{ runner.temp }}/traces + run: >- + uv run --python 3.12 --all-extras + pytest -m "e2e and hosted" -p no:cacheprovider -rs + "tests/e2e/test_templates.py::test_a_deployed_template_runs_on_hud_and_its_trace_reads_back[${{ matrix.template }}]" - name: Save traces if: always() uses: actions/upload-artifact@v4 with: name: example-traces-${{ matrix.template }} - path: ${{ runner.temp }}/traces-${{ matrix.template }} + path: ${{ runner.temp }}/traces/${{ matrix.template }} retention-days: 7 diff --git a/.gitignore b/.gitignore index dcd1827d4..216ccde19 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ build/ !docs/v6/build/ *.egg-info/ uv.lock +# Harbor case fixtures carry a project's .env as test data. +!tests/integrations/harbor/cases/**/.env # Media files *.jpg diff --git a/AGENTS.md b/AGENTS.md index 3625e317b..c62056963 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ adding local workarounds. - `CONTRIBUTING.md` for setup, test, lint, and type-check commands. - `pyproject.toml` for supported Python versions, dependencies, optional extras, ruff, ty, pytest, and coverage configuration. -- Source files and colocated tests for exact behavior. Trust code and tests over +- Source files and their tests for exact behavior. Trust code and tests over stale prose. - `cookbooks/` for runnable end-to-end examples (each is its own uv project). @@ -31,6 +31,8 @@ inventory of current incidents. - `hud/cli/` is the Typer surface over the same modules. - `hud/integrations/` contains packaged adapters for external task formats. - `hud/_legacy.py` and `hud/patches/` quarantine v5 compatibility. +- `tests/` follows the SDK's subsystems; `tests/support` holds stand-ins for the + outside world and `tests/fixtures` the SDK code tests run. - `cookbooks/` contains standalone runnable examples outside the `hud` package. ## Working Style @@ -53,14 +55,15 @@ Use the commands in `CONTRIBUTING.md` as the source of truth. Common commands: ```bash uv sync --extra dev -uv run pytest -q uv run ruff format . --check uv run ruff check . uv run --extra dev --extra train ty check +uv run python scripts/check_tests.py +uv run pytest -n auto ``` -The shared pre-push hook lives in `.githooks/pre-push`, but agents should not -change local git config unless explicitly asked. +Locally, ruff and ty are the loop; CI runs the test suite. Run the tests you +touched, and leave the full suite and the `e2e` lanes to CI. Tests run on Python 3.11 and 3.12 in CI. `pyproject.toml` currently supports Python `>=3.11, <3.13`. @@ -119,27 +122,38 @@ Python `>=3.11, <3.13`. - Use `TYPE_CHECKING` imports for type-only imports that would otherwise add runtime dependency cost or cycles. -## Testing Expectations - -- Add or update focused tests for behavior changes. Put tests near the module - they cover, following the existing `*/tests/` layout. -- Test behavior and contracts, not private implementation details. -- Regression tests should fail on the old behavior through the normal lifecycle - or public boundary. Do not manually seed private state such as internal maps, - caches, cursors, or prepared containers just to prove a changed line. -- If a bug involves internal state, reach it through real setup and execution: - construction, configuration, preparation, run loop, provider response, tool - execution, or public API call. -- Do not add hooks, helper methods, or abstraction layers only to make tests - easier. If a test needs that, reconsider the behavior boundary instead. -- Test names should describe the observable behavior or contract, not the - private mechanism. -- Mock external services, provider APIs, network, Docker, browser, and filesystem - boundaries as needed. Do not mock core logic just to make a test easy. -- Mark tests that require `HUD_API_KEY`, network access, or deployed services as - integration tests. -- Run the narrowest relevant tests first, then broader checks when the blast - radius is shared or user-facing. +## Testing + +Tests are black-box: they drive the SDK through a public boundary and fake only +what lies outside it, using the stand-ins in `tests/support`. `CONTRIBUTING.md` +shows how. + +- Enter through a public boundary: the `hud` CLI, an exported name, the control + channel or another wire protocol, a file format, or a process effect. +- Fake only the outside world: HUD services, model providers, Docker, cloud SDKs, + the clock. Never patch a `hud` name, import a private one, or touch a private + attribute; `scripts/check_tests.py` enforces this in CI. Configure the SDK the + way a user does, through environment variables (`hud_env.set(...)`). +- Never hand-build what a producer emits. Run the producer, so the test covers + the seam between it and its consumer. +- Assert what a user or a peer observes: the reward, run and trace fields, + output and exit codes, files, what the platform recorded, wire frames. Prefer + fixture environments whose reward proves the path ran. +- Write each contract once, as a table, at the outermost boundary where it is + observable. A contract's implementations (runtimes, agent families, commands) + are rows, drawn from the product's own list, and a completeness test fails when + one has no row. +- No unit tests: drive a scenario and check what it left behind, never what a + function returns. +- Assert the fields the case is about, never a recorded copy of a whole output: + no snapshots of argv, request bodies or rendered text. +- Do not add tests to check your own change. Check it with a temporary script, + and extend a scenario only when the change adds or fixes a contract; a bug fix + lands with the scenario row that failed before it. +- Rows needing something outside the process are marked `e2e` plus `docker`, + `modal`, `sandbox`, `live` (a real HUD account) or `hosted`, and run only when + selected. +- Test names describe the observable behavior, not the mechanism. ## Operational Debugging diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 92ea35ed1..f5c8d255c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,31 +14,76 @@ We welcome contributions to the HUD SDK! This guide covers how to get started. ## Development Workflow -### Git Hooks +### Running Tests -Enable the shared pre-push hook (runs ruff, ty, pytest before each push): +CI runs the full suite; locally, run what you touched: ```bash -git config core.hooksPath .githooks +uv run pytest tests/cli -n auto # one area, in parallel +uv run pytest -n auto # everything not marked e2e ``` -### Running Tests +Tests run on Python 3.11 and 3.12 in CI. The template scenarios in +`tests/e2e/test_templates.py` take each `hud init` template to a graded rollout +and install its packages, so they need network access to PyPI. + +Tests that need something outside the process are marked `e2e` and run only +when selected. They skip themselves when what they need is missing: ```bash -uv run pytest -q +uv run pytest -m "e2e and docker" # a running Docker daemon +uv run pytest -m "e2e and sandbox" # Linux, root and bubblewrap +uv run pytest -m "e2e and live and not hosted" # HUD_API_KEY; spends credits +uv run pytest -m "e2e and hosted" # deploys and runs on the HUD platform +uv run pytest -m "e2e and modal" # a Modal account +uv run --with "lerobot[dataset]" pytest -m "e2e and lerobot" # real LeRobot datasets +uv run pytest -m e2e tests/e2e/test_cookbooks.py # the cookbooks; CI checks them nightly ``` -Tests run on Python 3.11 and 3.12 in CI. +### Writing Tests -Integration tests are excluded by default because they may use external -services, network access, or container builds. Run them explicitly: +Tests are black-box: they drive the SDK through a public boundary and fake only +what lies outside it. -```bash -uv run pytest -m integration +- `tests/support`: stand-ins for the outside world (HUD services, model + providers, a browser, a VNC screen, the network, the user's machine). +- `tests/fixtures`: SDK code the tests run: fixture environments and an agent with + fixed answers. +- `tests/conftest.py`: the fixtures that wire them in: `hud_env` (configuration), + `platform` and `services` (HUD services), `models` (scripted providers) and + `hud` (the CLI in a subprocess). + +```python +async def test_a_graded_rollout_reports_its_reward(reporting): + job = await Task(env="echo", id="echo", args={"word": "hi"}).run( + ScriptedAgent("hi"), runtime=LocalRuntime(source("echo")) + ) + + (trace,) = reporting.traces.values() + assert (job.reward, trace.exit["reward"]) == (1.0, 1.0) ``` -The Harbor integration tests require Docker and may take several minutes with -a cold build cache. +Test directories follow the SDK's subsystems (`environment`, `clients`, `eval`, +`agents`, `telemetry`, `cli`, ...), and each file holds the contracts of one owner. +When a contract has several implementations (runtimes, agent families, commands, +capability clients), they are rows of one table, and a completeness test fails +when an implementation has no row. + +Rules, enforced by `scripts/check_tests.py` and in review: + +- Never patch a `hud` name, import a private one, or touch a private attribute. +- Never hand-build what a producer emits; run the producer. +- Write one scenario per contract as a table, and add a row for a new case or a + new implementation. Test the contract once, at the outermost boundary where it + is observable. +- Keep a test for one implementation only for behavior no other implementation + has, such as a provider's wire translation. +- Write no unit tests. A test drives a scenario (a command, a rollout, a served + environment, a wire peer) and checks what it left behind, never what a + function returns. +- Assert the fields a case is about, not a recorded copy of a whole output: no + snapshots of argv, request bodies or rendered text. +- Do not commit a test to check your own change; use a temporary script. ### Code Quality @@ -46,6 +91,7 @@ a cold build cache. uv run ruff format . --check # Formatting uv run ruff check . # Linting uv run --extra dev --extra train ty check # Type checking +uv run python scripts/check_tests.py # Tests stay black-box ``` ## Code Style @@ -59,7 +105,7 @@ uv run --extra dev --extra train ty check # Type checking 1. **Branch naming**: `feature/description` or `fix/issue-number` 2. **Commits**: Use clear, descriptive messages -3. **Tests**: All CI checks must pass (ruff, ty, pytest) +3. **Tests**: All CI checks must pass (ruff, ty, the test policy check, pytest) 4. **Review**: Address feedback promptly ## Releases diff --git a/REVIEW.md b/REVIEW.md new file mode 100644 index 000000000..d43ba6c17 --- /dev/null +++ b/REVIEW.md @@ -0,0 +1,33 @@ +# Reviewing hud-python + +Review adversarially: assume the change breaks something and find where. The +tests cover contracts at public boundaries, so the review is where a broken +seam, a missing scenario, or an unintended behavior change gets caught. + +## Behavior + +- Trace each changed contract from a real producer to a real consumer. Ask who + calls the code, with what inputs, and what they do with its output. +- Look for inputs a real producer emits that the change mishandles, and for + outputs the change emits that a real consumer cannot read. +- When you find a break, give the scenario row that fails because of it. + +## Tests + +The rules are under "Testing" in `AGENTS.md`. Treat each of these as a finding: + +- A test that patches a `hud` name, reads a private attribute, hand-builds what + a producer emits, or asserts that a mock was called the way it was set up. +- A bug fix without a scenario row that fails before the fix. +- A changed inline snapshot that the change does not explain. Every snapshot + change is a behavior change; one updated just to make a test pass hides one. +- A new test that only re-checks the author's change without adding a contract. + That belongs in a temporary script. +- A scenario that could pass with the code under test deleted. + +## Isolation and credentials + +For changes to `hud/environment`, the runtimes, egress, or Harbor, ask what an +agent inside the sandbox could now do: read the host's credentials, reach the +control channel, run code outside the sandbox, or keep a process alive past +teardown. Anything it newly could is a finding. diff --git a/cookbooks/a2a-chat/chat_env.py b/cookbooks/a2a-chat/chat_env.py index de5e6b277..b473a9ad1 100644 --- a/cookbooks/a2a-chat/chat_env.py +++ b/cookbooks/a2a-chat/chat_env.py @@ -17,8 +17,8 @@ from mcp.types import PromptMessage, TextContent -from hud.agents.types import EvaluationResult from hud.environment import Environment +from hud.graders import EvaluationResult env = Environment(name="chat") diff --git a/cookbooks/browser-use/README.md b/cookbooks/browser-use/README.md new file mode 100644 index 000000000..e58262a7c --- /dev/null +++ b/cookbooks/browser-use/README.md @@ -0,0 +1,19 @@ +# browser-use + +Run [browser-use](https://github.com/browser-use/browser-use) as a HUD agent. It +attaches to the browser an environment publishes as a `cdp` capability and runs its +own loop; the run records browser-use's final answer. + +`agent.py` is the whole adapter: copy it as a template for wrapping another agent +framework over the capability it needs (`ssh`, `mcp`, `rfb`, `robot`). + +## Run + +From this directory, with `ANTHROPIC_API_KEY` set: + +```bash +uv run run.py path/to/tasks.py +``` + +The tasks' environment must publish a browser, e.g. +`Capability.cdp(url="ws://127.0.0.1:9222/devtools/browser/...")`. diff --git a/cookbooks/browser-use/agent.py b/cookbooks/browser-use/agent.py new file mode 100644 index 000000000..a13ce5ffb --- /dev/null +++ b/cookbooks/browser-use/agent.py @@ -0,0 +1,51 @@ +"""browser-use as a HUD agent: it drives the browser an environment publishes as ``cdp``. + +The agent reads the ``cdp/1.3`` binding's URL off the run and hands it to browser-use, +which drives the browser over its own CDP client and runs its own loop. The run +records one coarse agent step with browser-use's final answer. +""" + +from __future__ import annotations + +import contextlib +import os +from urllib.parse import urlsplit, urlunsplit + +from browser_use import Agent as BrowserUse +from browser_use import Browser, ChatAnthropic + +from hud.agents.base import Agent +from hud.agents.types import AgentConfig, AgentStep +from hud.eval import Run + + +class BrowserUseConfig(AgentConfig): + model: str = "claude-sonnet-4-5" + max_steps: int = 25 + + +class BrowserUseAgent(Agent[BrowserUseConfig]): + config_cls = BrowserUseConfig + + async def __call__(self, run: Run) -> None: + cdp = urlsplit(run.client.binding("cdp/1.3").url) + scheme = {"ws": "http", "wss": "https"}.get(cdp.scheme, cdp.scheme) + browser = Browser(cdp_url=urlunsplit((scheme, *cdp[1:]))) + llm = ChatAnthropic(model=self.config.model, api_key=os.environ["ANTHROPIC_API_KEY"]) + try: + history = await BrowserUse(task=run.prompt_text, llm=llm, browser=browser).run( + max_steps=self.config.max_steps + ) + finally: + with contextlib.suppress(Exception): + await browser.stop() + + run.trace.content = history.final_result() or "" + run.trace.extra.update( + { + "is_successful": history.is_successful(), + "steps": history.number_of_steps(), + "urls": history.urls(), + } + ) + run.record(AgentStep(content=run.trace.content, done=history.is_done())) diff --git a/cookbooks/browser-use/pyproject.toml b/cookbooks/browser-use/pyproject.toml new file mode 100644 index 000000000..715873f94 --- /dev/null +++ b/cookbooks/browser-use/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "browser-use-agent" +version = "0.1.0" +description = "Run browser-use as a HUD agent over an environment's cdp capability (cookbook)" +requires-python = ">=3.11,<3.13" +dependencies = [ + "hud", + "browser-use>=0.11.13", +] + +[tool.uv] +package = false + +# Track the SDK from this repo. If you copied this folder out, delete this +# block to use the released hud from PyPI. +[tool.uv.sources] +hud = { path = "../..", editable = true } diff --git a/cookbooks/browser-use/run.py b/cookbooks/browser-use/run.py new file mode 100644 index 000000000..f49c3ad80 --- /dev/null +++ b/cookbooks/browser-use/run.py @@ -0,0 +1,21 @@ +"""Run browser-use on a taskset whose environment publishes a browser as ``cdp``: +``uv run run.py path/to/tasks.py``.""" + +from __future__ import annotations + +import asyncio +import sys + +from agent import BrowserUseAgent, BrowserUseConfig + +from hud.eval import Taskset + + +async def main(source: str) -> None: + job = await Taskset.from_file(source).run(BrowserUseAgent(BrowserUseConfig())) + for run in job.runs: + print(f"{run.slug}: reward {run.reward} ({run.trace.content!r})") + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1])) diff --git a/docs/skill.md b/docs/skill.md index 07dd98907..bf8cdd9dd 100644 --- a/docs/skill.md +++ b/docs/skill.md @@ -162,8 +162,8 @@ resources (ports, file handles) are not released otherwise. Once `hud eval env.py model` passes locally, two commands push it to the platform: ```bash -hud deploy . # package and deploy the environment (gives it a platform id) -hud sync tasks my-taskset env.py # upload tasks from env.py to the "my-taskset" taskset (name first, source second) +hud deploy . --yes # package and deploy the environment (gives it a platform id) +hud sync tasks my-taskset env.py --yes # upload tasks from env.py to the "my-taskset" taskset (name first, source second) ``` Then run at scale across models with `group=` for reward spread: @@ -184,7 +184,7 @@ Cite [Deploy](/v6/reference/runtime), [Models](/v6/reference/agents), [Training] ## Training scenarios -Training drives a **trainable model** (fork one: `hud models fork --name `); a `TrainingClient` targets that slug and advances its weights in place. Mark rollouts for training with `return_token_ids` so the gateway records tokens + logprobs. Pick the loop by scenario — all cite [Training](/v6/run/training) and [reference: Training](/v6/reference/training). +Training drives a **trainable model** (fork one: `hud models fork --name --yes`); a `TrainingClient` targets that slug and advances its weights in place. Mark rollouts for training with `return_token_ids` so the gateway records tokens + logprobs. Pick the loop by scenario — all cite [Training](/v6/run/training) and [reference: Training](/v6/reference/training). **Standard managed loop.** Roll out a batch with `group=` for within-group spread, hand the `Run`s to `step` (one `forward_backward` + one `optim_step`), repeat. @@ -224,7 +224,7 @@ Get the opponent's tokens from the gateway call with `extra_body={"return_token_ **Reset when the objective changes.** If you edit the reward or the environment mid-run, the current head encodes the *old* objective — continuing from it trains a contaminated policy, and the next steps mostly undo the old shaping. Roll the head back to a checkpoint from before the change (or fork a fresh model): ```python -await trainer.set_head(checkpoint_id) # or: hud models head --set +await trainer.set_head(checkpoint_id) # or: hud models head --set --yes ``` **Custom loss / primitives.** Author the loss yourself (e.g. double-sided IS) with `forward_backward_custom` — the service returns per-token tensors, your torch function makes the gradients (needs `pip install 'hud[train]'`). Drop to `forward_backward` + `optim_step` directly when you want the `forward_backward` metrics or gradient accumulation (`num_substeps`) in between; `step` is just the two chained. diff --git a/docs/v6/advanced/extending.mdx b/docs/v6/advanced/extending.mdx index 14b10be43..59a846ac6 100644 --- a/docs/v6/advanced/extending.mdx +++ b/docs/v6/advanced/extending.mdx @@ -79,18 +79,9 @@ this for you. ### Wrap an existing framework: browser-use on `cdp` -The bundled `BrowserUseAgent` is exactly this adapter - `browser-use` driving the `cdp` (browser) -capability: - -```python run.py -from hud.agents.browser_use import BrowserUseAgent -from hud.agents.types import BrowserUseConfig - -agent = BrowserUseAgent(BrowserUseConfig(model="claude-sonnet-4-5", max_steps=25)) -job = await my_browser_task().run(agent) -``` - -Use it as a template for wrapping other frameworks over whichever capability they need (`ssh`, `mcp`, +The [browser-use cookbook](https://github.com/hud-evals/hud-python/tree/main/cookbooks/browser-use) +is exactly this adapter - `browser-use` driving the `cdp` (browser) capability. Use it as a +template for wrapping other frameworks over whichever capability they need (`ssh`, `mcp`, `rfb`, `robot`). ### Any OpenAI-compatible endpoint diff --git a/docs/v6/experimental/harbor.mdx b/docs/v6/experimental/harbor.mdx index a87d5e8de..10ac15b25 100644 --- a/docs/v6/experimental/harbor.mdx +++ b/docs/v6/experimental/harbor.mdx @@ -123,7 +123,7 @@ The adapter handles: that agent service; - the Compose variables Harbor defines (`CONTEXT_DIR`, `MAIN_IMAGE_NAME`, `CPUS`, `MEMORY`, `PREBUILT_IMAGE_NAME`) and literal - `[environment.env]` values, alongside the project `.env`; + `[environment.env]` values, which take precedence over the project `.env`; - Compose sidecars, including ones that expose no TCP ports; - HTTP MCP servers (`sse` and `streamable-http`), healthchecks, network modes, allowlists, phase users, environment variables, and placement requirements; @@ -139,10 +139,13 @@ it can provision those requirements. stdio MCP servers and skills directories configure Harbor's own agents; the adapter accepts them and does not expose them to HUD agents. -Unsupported declarations fail during adaptation. These include multi-step -tasks, Compose variables only the host environment could supply, `include`, -and `extends`. A project also fails if its main service has neither an image -nor a valid build recipe. +Unsupported declarations fail during adaptation with a finding of kind +`contract`: multi-step tasks (`harbor.unsupported.multi_step`), Compose +variables only the host environment could supply +(`harbor.unsupported.host_compose_variable`), and Compose `include` and +`extends` (`harbor.unsupported.compose_include_extends`). Malformed tasks fail +with findings of kind `invalid`, such as a project whose main service has +neither an image nor a valid build recipe. ## Import into HUD diff --git a/docs/v6/experimental/verifier-environments.mdx b/docs/v6/experimental/verifier-environments.mdx index 9fb02d568..393ba2480 100644 --- a/docs/v6/experimental/verifier-environments.mdx +++ b/docs/v6/experimental/verifier-environments.mdx @@ -137,7 +137,10 @@ runtime, the phases remain isolated: artifact paths and `/logs`; - `/tests/test.sh` runs from the verifier image with its own user, workdir, environment, network mode, allowlist, and credentials directory; and -- verifier output is read from `/logs/verifier/reward.json` or `/logs/verifier/reward.txt`. +- verifier output is read from `/logs/verifier/reward.json` (a number, or an object with a + `reward` or `score` number) or, when that file is absent, `/logs/verifier/reward.txt`. A + boolean, non-finite, or missing reward, or a verifier past its timeout, is an error result + rather than a score. `RuntimeConfig.compose.service_access=True` mounts a Docker socket in the generated `main` service. diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx index 0b15fb170..bf4e3b327 100644 --- a/docs/v6/reference/cli.mdx +++ b/docs/v6/reference/cli.mdx @@ -26,7 +26,7 @@ List, get, create, and status commands accept `--json`. JSON is written to **std | `1` | Failure | | `2` | Usage error (bad arguments, or a prompt needed in a non-interactive terminal) | -Use `--dry-run` to preview supported commands without performing the action or prompting. Pass `--yes` / `-y` to skip confirmation where supported; a required prompt exits `2` in a non-interactive terminal. For deployment, select local secrets with `--env-file` or skip them with `--no-env`. +Every command that changes platform state takes `--dry-run`, which previews the action without sending or writing anything, and asks for confirmation before it acts. Pass `--yes` / `-y` to skip the confirmation; without it, the command exits `2` in a non-interactive terminal. For deployment, select local secrets with `--env-file` or skip them with `--no-env`. If the HUD API deprecates a route that `hud` calls, `hud` prints a warning to stderr with the route's sunset date and replacement. Upgrade `hud` when you see one. SDK code receives it as `hud.utils.exceptions.HudDeprecationWarning`, a `FutureWarning`. @@ -101,6 +101,7 @@ hud deploy --project | `--runtime-config` | JSON `RuntimeConfig` to persist as the registry default (compose, resources, limits). Must set at least one field. | | `--json` | Structured result on stdout. | | `--dry-run` | Print the deploy plan without uploading or prompting. A first deploy with an unresolved `.env` reports `dotenv_pending`. | +| `--yes`, `-y` | Skip the deploy confirmation (required when stdin is not a TTY). | `--env` / `--env-file` upsert onto the registry's existing secrets. Omitting them on a rebuild keeps whatever is already stored. A first deploy that finds a local `.env` prompts once for that invocation (or needs `--env-file` / `--no-env` when stdin is not a TTY). @@ -149,6 +150,14 @@ own environment must include an image or Compose configuration. **Single-task runs** show step-by-step progress (step number + tool calls). Multi-task batches are silent unless `--verbose` is passed. +`hud eval` exits `0` when every run was graded, whatever the rewards: a low +reward is a result, not a failure. It exits `1` when any run ended without a +valid grade (the environment reported a grading error, the grader raised, or +the rollout failed before grading); `--json` still prints the result document +first. In that document, `error_count` counts those runs and each run's +`is_error` says whether it is one of them; errored runs are left out of +`mean_reward`. + | Option | Description | |--------|-------------| | `--full` | Run the whole dataset (`--all --auto-respond --max-steps 100`). | @@ -235,7 +244,7 @@ hud project use browser-evals -C ./my-env |-------------------|-------------| | `hud project [--directory PATH]` | Show where new resources from a directory will be created. `-C` is an alias. | | `hud project list` | List visible Projects, marking the team default and read-only entries. | -| `hud project create ` | Create and pin a Project. Accepts `--description`, `--directory`/`-C`, and `--no-use`. | +| `hud project create ` | Create and pin a Project. Accepts `--description`, `--directory`/`-C`, `--no-use`, `--dry-run`, and `--yes`. | | `hud project use ` | Pin a directory to an existing writable Project. Names are normalized (`My Project` = `my-project`). Accepts `--directory`/`-C`. | #### Project selection @@ -299,7 +308,7 @@ hud harbor import ./jobs/2026-10-02__09-00 # into the dir hud harbor import bundle.zip --taskset tb2 --no-wait --json # a prepared zip; return at once hud harbor imports --taskset tb2 # past imports hud harbor report --wait # follow one, then show its report -hud harbor cancel +hud harbor cancel --yes ``` `hud harbor import` zips the folders, uploads the zip, imports it into the taskset, and prints what became of each task, job, and trial. `--taskset` takes a name or ID; a name @@ -338,8 +347,8 @@ every one. The checks decide what the IDs are: trace-level checks take completed trace IDs, task-level checks take task IDs. Trace-level and task-level checks can't be mixed in one run. Where a check's current result is completed or still running, it is kept rather than run again; pass -`--overwrite` to run it anyway. Pass `--no-wait` to return after launch, or -`--json` for machine-readable output. Exit code `1` means a check did not pass +`--overwrite` to run it anyway. Pass `--no-wait` to return after launch, `--yes` +to skip the confirmation, or `--json` for machine-readable output. Exit code `1` means a check did not pass or the SDK could not complete the command. `hud qa results` prints the result in effect for every check on the given @@ -392,9 +401,9 @@ without an API call. |---------|-------------| | `hud set KEY=VALUE` | Persist credentials/vars to `~/.hud/.env`. | | `hud models list` | List gateway models (`--json`, `--quiet`). | -| `hud models fork --name ` | Fork a trainable model (`--json`, `--dry-run`, `--if-not-exists`). | +| `hud models fork --name ` | Fork a trainable model (`--json`, `--dry-run`, `--yes`, `--if-not-exists`). | | `hud models checkpoints ` | List a model's checkpoint tree. | -| `hud models head [--set ]` | Show - or set - a model's active checkpoint. | +| `hud models head [--set ]` | Show - or set - a model's active checkpoint (`--dry-run`, `--yes` with `--set`). | | `hud jobs cancel` | Cancel a running job (`--yes`, `--dry-run`, `--json`). | | `hud version` | Show the CLI version (`--json`). | diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index 2345f7d05..b2656fd84 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -34,7 +34,7 @@ HostedRuntime, Runtime`); `ModalRuntime` and `DaytonaRuntime` import from `hud.e **You can usually omit `runtime=`.** A run without one uses its known placement: -- a platform taskset (`Taskset.from_api`) runs on the platform +- a platform taskset (`Taskset.from_api`) runs against its deployed environment over the HUD tunnel (`HUDRuntime`), with the agent loop in this process - a row whose `runtime_config` names an image or Compose project starts it under `DockerRuntime` - a task created by `@env.template` runs against that environment, in this process @@ -83,7 +83,7 @@ is best effort: `DockerRuntime` admits against available disk and `DaytonaRuntim enough whole GiB, while providers without disk sizing proceed with their default capacity. `HUDRuntime` proceeds on the deployed environment's defaults and warns when it ignores CPU, memory, or storage requirements; it rejects GPU, OS, TPU, and `limits`. Daytona accepts a list of GPU -alternatives; Modal accepts one GPU type. Daytona rejects `run_timeout_s` and resource overrides +alternatives; Modal accepts one GPU type. Daytona rejects Compose, `run_timeout_s`, and resource overrides when booting from an already-built snapshot. `LocalRuntime` rejects a per-task `runtime_config`. @@ -219,10 +219,8 @@ DaytonaRuntime(snapshot_name=None, *, image=None, command=None, workdir="/app", Resources (cpu/memory/gpu) are fixed on the snapshot at build time. With `image`, a task's `runtime_config.resources` builds a sized variant under a suffixed name (`my-env-4cpu`); without `image`, an already-built snapshot cannot be resized. -For Compose input, Daytona runs `main` as the rollout sandbox and each sidecar -as a linked sandbox. Sidecar images resolve to reusable Daytona snapshots. -Compose authoring conventions are documented in -[Compose environments](/v6/experimental/compose). +`DaytonaRuntime` does not run Compose projects: a `runtime_config.compose` is rejected. Use +`DockerRuntime` or `ModalRuntime` for [Compose environments](/v6/experimental/compose). Measured behavior at scale, up to 256 concurrent sandboxes with warm pools, is written up in the [Daytona cookbook](/v6/cookbooks/daytona-rl) and the [full guide on Daytona's docs](https://www.daytona.io/docs/guides/reinforcement-learning/hud-rl-cookbook). diff --git a/docs/v6/reference/tasks.mdx b/docs/v6/reference/tasks.mdx index 10ac99d25..20c96daf5 100644 --- a/docs/v6/reference/tasks.mdx +++ b/docs/v6/reference/tasks.mdx @@ -158,7 +158,7 @@ job = await ts.run(agent, runtime=LocalRuntime("env.py"), group=8, max_concurren | Parameter | Type | Description | |-----------|------|-------------| | `agent` | `Agent` | The shared, stateless agent driven across every rollout. | -| `runtime` | `Provider \| HostedRuntime \| None` | Where each rollout runs; omit it and placement is inferred (local source serves itself, else the HUD tunnel by env name). See [runtime](/v6/reference/runtime). | +| `runtime` | `Provider \| HostedRuntime \| None` | Where each rollout runs; omit it and placement is inferred: a platform taskset (`Taskset.from_api`) runs over the HUD tunnel, a row whose `runtime_config` names an image or Compose project starts it under `DockerRuntime`, a row minted by a live environment runs against it, and any other row raises `ValueError`. See [runtime](/v6/reference/runtime). | | `group` | `int \| None` | Repeats each task N times for GRPO. | | `max_concurrent` | `int \| None` | Caps how many rollouts run in parallel. | | `job` | `Job \| None` | An open [`Job`](/v6/reference/types#job) to accumulate into; defaults to a fresh job per call. | diff --git a/docs/v6/reference/types.mdx b/docs/v6/reference/types.mdx index 21c77c91c..31c51237c 100644 --- a/docs/v6/reference/types.mdx +++ b/docs/v6/reference/types.mdx @@ -26,7 +26,7 @@ The live handle for one task - the lifecycle plus the agent's `Trace`. You get t | `run.trace` | `Trace` | The trajectory the agent fills. **The answer is `run.trace.content`.** | | `run.grade` | `Grade` | Structured grade result. | | `run.reward` | `float` | The graded reward (`grade.reward`, set on exit). | -| `run.evaluation` | `dict` | The raw grade payload (`grade.raw`). | +| `run.evaluation` | `dict` | A persistence-safe copy of the grade frame: `grade.raw` with each `subscores` entry reduced to its summary. | | `run.runtime` | `str \| None` | Control-channel url the run executed against (the placement record). | | `run.trace_id` | `str \| None` | Keys the trajectory for training. | | `run.slug` | `str \| None` | The task slug this run came from, set by the runner; how [`Job.results`](#job) keys runs back to their task without positional zip. | @@ -63,12 +63,14 @@ run reports under a job, so even a single `task.run` returns a job of one. You g | `runs` | `list[Run]` | The graded runs, in expansion order (task-major, then group). | | `group` | `int` | Rollouts per task. | | `taskset_id` | `str \| None` | Platform taskset id when the job runs a synced taskset (`Taskset.from_api`), else `None`. | -| `reward` | `float` | Property: mean reward across runs (`0.0` for an empty job). | +| `reward` | `float` | Property: mean reward across graded runs (`0.0` when none were graded). Runs in `errors` are excluded rather than averaged in as zero. | +| `errors` | `list[Run]` | Property: runs that ended without a valid grade - a grading error, or a failed trace with no grade frame. A run that timed out or raised after the task started still counts toward `reward` when its environment returned a grade. | | `results` | `dict[str, list[Run]]` | Property: runs grouped by task slug - the alignment-safe alternative to `zip(tasks, runs)`, list-valued since `group > 1` gives several runs per task. | ```python job = await ts.run(agent, runtime=LocalRuntime("env.py"), group=4) -print(job.reward) # mean across every run +print(job.reward) # mean across graded runs +print(len(job.errors)) # runs excluded for lack of a grade ``` Each `run` call mints its own job by default. To gather many calls under one id - a training session, a diff --git a/environments/argument-hints/tests/conftest.py b/environments/argument-hints/tests/conftest.py index 069464691..2a8e101e3 100644 --- a/environments/argument-hints/tests/conftest.py +++ b/environments/argument-hints/tests/conftest.py @@ -1,8 +1,126 @@ -"""Make the environment project importable when tests run from the SDK root.""" +"""Serve the environment the way HUD does, against a local stand-in for the HUD platform. +The stand-in answers the three things this environment calls: the data-file API +(metadata and a presigned download URL), the storage behind that URL, and the +inference gateway the LLM judge uses. The environment runs in its own process, +configured only through environment variables, exactly as ``hud eval`` serves it. +""" + +from __future__ import annotations + +import json import sys +import threading +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +import pytest + PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) + + +@dataclass +class DataFile: + filename: str + content: bytes + downloadable: bool + + +@dataclass +class Received: + method: str + path: str + authorization: str | None + body: dict = field(default_factory=dict) + + +class Platform: + """The HUD API, storage and gateway this environment talks to. + + The judge grades like a perfect answer: positive criteria MET, negative ones + UNMET. An answer listed in ``rejected_answers`` gets an HTTP 400 instead. + """ + + def __init__(self) -> None: + self.files: dict[str, DataFile] = {} + self.rejected_answers: set[str] = set() + self.received: list[Received] = [] + self.judging = threading.Barrier(1) + platform = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + platform.received.append(Received("GET", self.path, self.headers["Authorization"])) + parts = self.path.strip("/").split("/") + if parts[0] == "storage" and parts[1] in platform.files: + self.reply(200, platform.files[parts[1]].content, "application/octet-stream") + elif parts[:2] == ["v2", "data"] and parts[2] in platform.files: + data_file = platform.files[parts[2]] + if parts[3:] == ["download"]: + url = f"{platform.url}/storage/{parts[2]}" if data_file.downloadable else None + self.reply_json(200, {"url": url}) + else: + self.reply_json(200, {"filename": data_file.filename}) + else: + self.reply_json(404, {"detail": "not found"}) + + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + platform.received.append(Received("POST", self.path, self.headers["Authorization"], body)) + prompt = body["messages"][-1]["content"] + answer = prompt.split("\n", 1)[1].split("\n", 1)[0] + platform.judging.wait(timeout=10) + if answer in platform.rejected_answers: + self.reply_json(400, {"error": {"message": "judge rejected request"}}) + return + met = "\npositive" in prompt + verdict = {"criterion_status": "MET" if met else "UNMET", "explanation": "stand-in"} + self.reply_json(200, {"choices": [{"message": {"content": json.dumps(verdict)}}]}) + + def reply_json(self, status: int, payload: dict) -> None: + self.reply(status, json.dumps(payload).encode(), "application/json") + + def reply(self, status: int, body: bytes, content_type: str) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + del format, args + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.server.server_port}" + + def upload(self, file_id: str, filename: str, content: bytes = b"", *, downloadable: bool = True) -> None: + """Make ``file_id`` known to the data-file API; ``downloadable=False`` withholds its URL.""" + self.files[file_id] = DataFile(filename, content, downloadable) + + def requests(self, method: str, prefix: str) -> list[Received]: + return [item for item in self.received if item.method == method and item.path.startswith(prefix)] + + +@pytest.fixture +def workspace(tmp_path: Path) -> Path: + return tmp_path / "workspace" + + +@pytest.fixture(autouse=True) +def platform(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, workspace: Path): + """Point the environment at the stand-in platform with a runtime key and no user config.""" + stand_in = Platform() + thread = threading.Thread(target=stand_in.server.serve_forever, daemon=True) + thread.start() + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("WORKSPACE_DIR", str(workspace)) + monkeypatch.setenv("HUD_API_KEY", "runtime-key") + monkeypatch.setenv("HUD_API_URL", stand_in.url) + monkeypatch.setenv("HUD_GATEWAY_URL", stand_in.url) + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "0") + yield stand_in + stand_in.server.shutdown() + thread.join() diff --git a/environments/argument-hints/tests/test_env.py b/environments/argument-hints/tests/test_env.py index 11ddb097a..1641670f3 100644 --- a/environments/argument-hints/tests/test_env.py +++ b/environments/argument-hints/tests/test_env.py @@ -1,258 +1,216 @@ -"""What the example promises: hinted arguments, safe staging, weighted grading.""" +"""What the example promises: hinted arguments, safe staging, weighted grading, the runtime key. + +Each test starts a fresh environment process against the stand-in platform in +``conftest.py``, starts a task with the arguments a task row carries, and grades +an answer: what a rollout does, without an agent. +""" from __future__ import annotations import asyncio -import json -import sys -from functools import partial -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import threading +from contextlib import asynccontextmanager from pathlib import Path -from threading import Barrier, Thread -import httpx import pytest -from hud import LocalRuntime, connect +from hud import SubprocessRuntime, Task, connect from hud.clients import HudProtocolError -from hud.graders import SubScore - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -import env as env_module # noqa: E402 - - -def test_every_editable_argument_declares_its_hint(): - """The hints are the point of this environment; a rename must fail here.""" - properties = env_module.review_files.manifest_entry()["args"]["properties"] - assert properties["prompt"]["x-hud-hint"] == "prompt" - assert properties["attachments"]["x-hud-hint"] == "data-files" - assert properties["criteria"]["x-hud-hint"] == "grading" - assert "hud_api_key" in properties - required = env_module.review_files.manifest_entry()["args"].get("required") or [] - assert "hud_api_key" not in required - # The console resolves the reference through the $defs discovery bundles. - assert properties["attachments"]["items"]["$ref"] == "#/$defs/DataFileRef" - assert "file_id" in env_module.review_files.manifest_entry()["args"]["$defs"]["DataFileRef"]["properties"] - - -def test_staging_refuses_a_path_that_leaves_the_files_directory(tmp_path: Path): - """An uploaded file names its own destination, so the path is untrusted.""" - assert env_module._destination(tmp_path, "notes/summary.md") == tmp_path / "notes/summary.md" - for hostile in ("../escape.md", "/etc/passwd", "..\\escape.md"): - with pytest.raises(env_module.DataFileError): - env_module._destination(tmp_path, hostile) - - -async def test_criteria_reach_the_judge_unchanged(monkeypatch: pytest.MonkeyPatch): - """The argument is the grader's input, so nothing may be rewritten en route.""" - seen: dict[str, object] = {} - - async def fake_compute_score(**kwargs: object): - seen.update(kwargs) - return SubScore(name="LLMJudgeGrader", value=1.0) - - monkeypatch.setattr(env_module.LLMJudgeGrader, "compute_score", fake_compute_score) - criteria = [ - env_module.Criterion(requirement="Recommends an interview.", weight=3), - env_module.Criterion(requirement="Invents an employer.", weight=-2), - ] - - result = await env_module._grade("Hire her.", "Should we interview her?", criteria) - - assert seen["criteria"] == [("Recommends an interview.", 3.0), ("Invents an employer.", -2.0)] - assert seen["answer"] == "Hire her." - assert seen["question"] == "Should we interview her?" - assert result.reward == pytest.approx(1.0) - - -async def test_no_criteria_scores_zero_without_raising(): - """An empty list is a misconfigured task, not a crashed rollout.""" - result = await env_module._grade("Hire her.", "", []) - assert result.reward == pytest.approx(0.0) - - -async def test_staging_directory_is_cleared_between_tasks( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -): - """A prior task's leftovers (including symlinks) must not survive into staging.""" - async def fake_compute_score(**kwargs: object): - return SubScore(name="LLMJudgeGrader", value=1.0) +from env import review_files - async def fake_stage(refs, root: Path): - return [] +ENV_SOURCE = Path(__file__).resolve().parent.parent / "env.py" +CRITERIA = [ + {"requirement": "Names exactly three primes.", "weight": 2}, + {"requirement": "Explains each one.", "weight": 1}, + {"requirement": "Calls a composite number prime.", "weight": -2}, +] - monkeypatch.setattr(env_module.LLMJudgeGrader, "compute_score", fake_compute_score) - monkeypatch.setattr(env_module, "_stage", fake_stage) - monkeypatch.setattr(env_module, "WORKSPACE_ROOT", tmp_path) - files_dir = tmp_path / env_module.FILES_DIRNAME - files_dir.mkdir(parents=True) - (files_dir / "escape").symlink_to(tmp_path / "outside") +def row(attachments: list[dict] | None = None, criteria: list[dict] | None = None) -> Task: + return review_files( + prompt="Name three primes.", + attachments=attachments or [], + criteria=CRITERIA if criteria is None else criteria, + hud_api_key="task-key", + ) - task = env_module.review_files.func(prompt="p", attachments=[], criteria=[]) - await task.asend(None) - assert files_dir.is_dir() - assert list(files_dir.iterdir()) == [] +@asynccontextmanager +async def served(task: Task): + """A fresh environment process serving ``task``, and a client connected to it.""" + async with SubprocessRuntime(ENV_SOURCE)(task) as runtime, connect(runtime) as client: + yield client -async def test_arguments_arrive_as_plain_json( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -): - """A task run sends JSON, not model instances, so the template must validate.""" - seen: dict[str, object] = {} +async def test_every_editable_argument_declares_its_hint(): + async with served(row()) as client: + (template,) = await client.list_tasks() - async def fake_compute_score(**kwargs: object): - seen.update(kwargs) - return SubScore(name="LLMJudgeGrader", value=1.0) + arguments = template["args"] + hints = {name: spec.get("x-hud-hint") for name, spec in arguments["properties"].items()} + assert hints == { + "prompt": "prompt", + "attachments": "data-files", + "criteria": "grading", + "hud_api_key": None, + } + assert "hud_api_key" not in arguments["required"] + # The console resolves the attachment reference through $defs. + assert arguments["properties"]["attachments"]["items"]["$ref"] == "#/$defs/DataFileRef" + assert "file_id" in arguments["$defs"]["DataFileRef"]["properties"] - staged: list[env_module.DataFileRef] = [] - async def fake_stage( - refs: list[env_module.DataFileRef], - root: Path, - ): - staged.extend(refs) - return [{"path": "files/resume.pdf", "file_id": refs[0].file_id}] - - monkeypatch.setattr(env_module.LLMJudgeGrader, "compute_score", fake_compute_score) - monkeypatch.setattr(env_module, "_stage", fake_stage) - monkeypatch.setattr(env_module, "WORKSPACE_ROOT", tmp_path) - monkeypatch.setattr(env_module.settings, "api_key", None) - - task = env_module.review_files.func( - prompt="Summarise the role.", - attachments=[{"file_id": "8b1f", "path": "resume.pdf"}], - criteria=[{"requirement": "Recommends an interview.", "weight": 2}], - hud_api_key="test-key", - ) - frame = await task.asend(None) +async def test_attachments_are_staged_with_the_runtime_key_and_declared(platform, workspace): + platform.upload("notes", "notes.txt", b"three primes") + platform.upload("report", "report.pdf", b"%PDF") + task = row(attachments=[{"file_id": "notes", "path": "reading/notes.md"}, {"file_id": "report"}]) - # Each argument reached its model rather than staying a bare dict. - assert staged == [env_module.DataFileRef(file_id="8b1f", path="resume.pdf")] - assert "Summarise the role." in frame["prompt"] - assert frame["data_files"] == [{"path": "files/resume.pdf", "file_id": "8b1f"}] + async with served(task) as client: + started = await client.start_task(task.id, task.args) - # A task run sends the agent's final text, the same as a real grade call. - result = await task.asend("Hire her.") - - assert seen["criteria"] == [("Recommends an interview.", 2.0)] - assert result.reward == pytest.approx(1.0) - - -@pytest.mark.parametrize("fail_second", [False, True]) -async def test_concurrent_sessions_use_the_runtime_credential( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, fail_second: bool + assert started["data_files"] == [ + {"path": "files/reading/notes.md", "file_id": "notes"}, + {"path": "files/report.pdf", "file_id": "report"}, + ] + assert "- files/reading/notes.md\n- files/report.pdf" in started["prompt"] + assert started["prompt"].endswith("Name three primes.") + assert (workspace / "files/reading/notes.md").read_bytes() == b"three primes" + assert (workspace / "files/report.pdf").read_bytes() == b"%PDF" + assert {item.authorization for item in platform.requests("GET", "/v2/data")} == {"Bearer runtime-key"} + # The presigned URL carries its own credentials; the key never goes to storage. + assert {item.authorization for item in platform.requests("GET", "/storage")} == {None} + + +@pytest.mark.parametrize( + ("upload", "attachment", "message"), + [ + pytest.param( + {"filename": "notes.txt"}, + {"file_id": "notes", "path": "../escape.md"}, + "unsafe data file path", + id="hostile-destination", + ), + pytest.param( + {"filename": "../../escape.md"}, + {"file_id": "notes"}, + "unsafe data file path", + id="hostile-uploaded-filename", + ), + pytest.param( + {"filename": "..\\escape.md"}, + {"file_id": "notes"}, + "unsafe data file path", + id="backslash-filename", + ), + pytest.param(None, {"file_id": "notes"}, "GET /v2/data/notes failed: 404", id="unknown-file"), + pytest.param( + {"filename": "notes.txt", "downloadable": False}, + {"file_id": "notes"}, + "has no download url", + id="no-download-url", + ), + ], +) +async def test_a_file_that_cannot_be_staged_fails_the_start_before_anything_is_written( + upload, attachment, message, platform, workspace ): - monkeypatch.setattr(env_module, "WORKSPACE_ROOT", tmp_path) - monkeypatch.setattr(env_module.settings, "api_key", "process-key") - arrived = Barrier(2, timeout=5) - seen = {} - - class Gateway(BaseHTTPRequestHandler): - def do_POST(self): - body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) - prompt = body["messages"][-1]["content"] - answer = prompt.split("\n", 1)[1].split("\n", 1)[0] - seen[answer] = self.headers["Authorization"] - arrived.wait() - assert env_module.settings.api_key == "process-key" - rejected = fail_second and answer == "second" - payload = json.dumps( - {"error": {"message": "judge rejected request"}} - if rejected - else { - "choices": [ - { - "message": { - "content": '{"criterion_status":"MET","explanation":"fixture"}', - } - } - ] - } - ).encode() - self.send_response(400 if rejected else 200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - first_task = env_module.review_files( - prompt="Give an answer.", - attachments=[], - criteria=[{"requirement": "An answer is present.", "weight": 1}], - hud_api_key="first-key", - ) - second_task = first_task.model_copy(update={"args": {**first_task.args, "hud_api_key": "second-key"}}) - with ThreadingHTTPServer(("127.0.0.1", 0), Gateway) as server: - monkeypatch.setattr(env_module.settings, "hud_gateway_url", f"http://127.0.0.1:{server.server_port}") - thread = Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - async with ( - LocalRuntime(env_module.env)(first_task) as runtime, - connect(runtime) as first, - connect(runtime) as second, - ): - await first.start_task(first_task.id, first_task.args) - await second.start_task(second_task.id, second_task.args) - results = await asyncio.gather( - first.grade({"answer": "first"}), - second.grade({"answer": "second"}), - return_exceptions=True, - ) - finally: - server.shutdown() - thread.join() - - assert seen == {"first": "Bearer process-key", "second": "Bearer process-key"} - assert env_module.settings.api_key == "process-key" - for index, result in enumerate(results): - if fail_second and index == 1: - assert isinstance(result, HudProtocolError) - assert "judge rejected request" in str(result) - continue - assert isinstance(result, dict) - assert result["score"] == 1.0 - assert "first-key" not in json.dumps(result) - assert "second-key" not in json.dumps(result) - - -async def test_file_staging_uses_the_runtime_credential(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): - monkeypatch.setattr(env_module, "WORKSPACE_ROOT", tmp_path) - monkeypatch.setattr(env_module.settings, "api_key", "runtime-key") - monkeypatch.setattr(env_module.settings, "hud_api_url", "https://api.test") - - async def handler(request: httpx.Request) -> httpx.Response: - if request.url.host == "storage.test": - assert "Authorization" not in request.headers - return httpx.Response(200, content=b"notes") - assert request.headers["Authorization"] == "Bearer runtime-key" - if request.url.path.endswith("/download"): - return httpx.Response(200, json={"url": "https://storage.test/notes.txt"}) - return httpx.Response(200, json={"filename": "notes.txt"}) - - monkeypatch.setattr( - env_module.httpx, - "AsyncClient", - partial( - httpx.AsyncClient, - transport=httpx.MockTransport(handler), + if upload is not None: + platform.upload("notes", **upload) + task = row(attachments=[attachment]) + + async with served(task) as client: + with pytest.raises(HudProtocolError) as failure: + await client.start_task(task.id, task.args) + + assert message in failure.value.message + assert platform.requests("GET", "/storage") == [] + assert list((workspace / "files").rglob("*")) == [] + assert not (workspace / "escape.md").exists() + + +async def test_staging_without_a_hud_key_fails_the_start(monkeypatch, platform): + monkeypatch.setenv("HUD_API_KEY", "") + platform.upload("notes", "notes.txt") + task = row(attachments=[{"file_id": "notes"}]) + + async with served(task) as client: + with pytest.raises(HudProtocolError) as failure: + await client.start_task(task.id, task.args) + + assert "HUD_API_KEY is unset" in failure.value.message + assert platform.received == [] + + +async def test_a_previous_task_leaves_nothing_in_the_files_directory(workspace): + files = workspace / "files" + files.mkdir(parents=True) + (files / "escape").symlink_to(workspace.parent / "outside") + task = row() + + async with served(task) as client: + started = await client.start_task(task.id, task.args) + + assert started["data_files"] == [] + assert "- (none)" in started["prompt"] + assert list(files.iterdir()) == [] + + +@pytest.mark.parametrize( + ("criteria", "score", "judged"), + [ + pytest.param( + CRITERIA, + 1.0, + [ + ("positive", "Names exactly three primes."), + ("positive", "Explains each one."), + ("negative", "Calls a composite number prime."), + ], + id="weighted-criteria", ), - ) - task = env_module.review_files.func( - prompt="Read the file.", - attachments=[{"file_id": "notes"}], - criteria=[], - hud_api_key="task-key", - ) - try: - frame = await anext(task) - assert frame["data_files"] == [{"path": "files/notes.txt", "file_id": "notes"}] - assert (tmp_path / "files/notes.txt").read_text() == "notes" - assert env_module.settings.api_key == "runtime-key" - finally: - await task.aclose() + pytest.param([], 0.0, [], id="no-criteria"), + ], +) +async def test_the_criteria_reach_the_judge_unchanged(criteria, score, judged, platform): + task = row(criteria=criteria) + + async with served(task) as client: + await client.start_task(task.id, task.args) + grade = await client.grade({"answer": "2, 3 and 5"}) + + assert grade["score"] == pytest.approx(score) + prompts = [item.body["messages"][-1]["content"] for item in platform.requests("POST", "/")] + assert sorted( + (prompt.split("\n")[1].split("\n")[0], prompt.split("\n")[1].split("\n")[0]) + for prompt in prompts + ) == sorted(judged) + assert all("\n2, 3 and 5\n" in prompt for prompt in prompts) + assert {item.authorization for item in platform.requests("POST", "/")} <= {"Bearer runtime-key"} + + +@pytest.mark.parametrize("rejected", [set(), {"second"}], ids=["both-graded", "one-judge-rejects"]) +async def test_concurrent_sessions_grade_with_the_runtime_key(rejected, platform): + platform.rejected_answers = rejected + platform.judging = threading.Barrier(2) + first = row(criteria=[{"requirement": "An answer is present.", "weight": 1}]) + second = first.model_copy(update={"args": {**first.args, "hud_api_key": "second-key"}}) + + async with ( + SubprocessRuntime(ENV_SOURCE)(first) as runtime, + connect(runtime) as one, + connect(runtime) as two, + ): + await one.start_task(first.id, first.args) + await two.start_task(second.id, second.args) + grades = await asyncio.gather( + one.grade({"answer": "first"}), two.grade({"answer": "second"}), return_exceptions=True + ) + + assert {item.authorization for item in platform.requests("POST", "/")} == {"Bearer runtime-key"} + assert grades[0]["score"] == 1.0 + if rejected: + assert isinstance(grades[1], HudProtocolError) + assert "judge rejected request" in grades[1].message + else: + assert grades[1]["score"] == 1.0 + assert "task-key" not in str(grades) and "second-key" not in str(grades) diff --git a/environments/coding/README.md b/environments/coding/README.md index 7e0a126ff..04d4d7346 100644 --- a/environments/coding/README.md +++ b/environments/coding/README.md @@ -37,8 +37,12 @@ namespaces. uv run pytest tests/ -q ``` -The suite covers repository and session isolation, task reset, hidden-test application, selected -JUnit scoring, and both bundled reference fixes. +The grader and task-row tests run anywhere. The `sandbox` tests serve the environment with its +isolated workspace and cover task reset, hidden-test application and tampering, selected JUnit +scoring, grading as the agent's user, and both bundled reference fixes. They need Linux, root, +bubblewrap, and the environment's Python installed under `/usr` (the sandbox shows the grader +nothing else), so they skip elsewhere; run them in the coding container, where the image installs +it at `/usr/local/venv`. See the [coding-agent cookbook](https://docs.hud.ai/v6/cookbooks/coding-agent) for the grading lifecycle and a task-authoring walkthrough. diff --git a/environments/coding/pyproject.toml b/environments/coding/pyproject.toml index 37c40abf6..43dba699c 100644 --- a/environments/coding/pyproject.toml +++ b/environments/coding/pyproject.toml @@ -37,5 +37,9 @@ quote-style = "double" indent-style = "space" line-ending = "auto" +[tool.pytest.ini_options] +asyncio_mode = "auto" +markers = ["sandbox: serves the isolated workspace; needs Linux, root, bubblewrap, and the env under /usr"] + [tool.hatch.build.targets.wheel] packages = [ ".",] diff --git a/environments/coding/tests/conftest.py b/environments/coding/tests/conftest.py index 5450d19f3..f8b04f0b0 100644 --- a/environments/coding/tests/conftest.py +++ b/environments/coding/tests/conftest.py @@ -1,36 +1,85 @@ -"""Make the environment project importable when tests run from the SDK root.""" +"""Run the environment the way HUD does, against a small fixture repository. + +Tests marked ``sandbox`` serve the real workspace, which refuses to run without +isolation. They need Linux, root, bubblewrap, and this project's Python +environment under /usr, the only prefix the sandbox shows the grader (the image +installs it at /usr/local/venv). Run them in the coding container. +""" import os +import shutil +import subprocess import sys from pathlib import Path import pytest -import pytest_asyncio from hud.environment import Workspace -from hud.environment.workspace import usable_bwrap PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) +SANDBOXED = ( + sys.platform == "linux" + and os.geteuid() == 0 + and shutil.which("bwrap") is not None + and Path(sys.prefix).is_relative_to("/usr") +) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + if SANDBOXED: + return + skip = pytest.mark.skip(reason="needs Linux, root, bubblewrap, and the environment under /usr") + for item in items: + if "sandbox" in item.keywords: + item.add_marker(skip) + + +def git(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-c", f"safe.directory={cwd}", "-c", "user.name=t", "-c", "user.email=t@t", *args], + cwd=cwd, + check=False, + capture_output=True, + text=True, + ) + + +@pytest.fixture(scope="session") +def fixture_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: + """A repository whose ``bug_baseline`` branch is broken and ``bug_golden`` fixes it.""" + repo = tmp_path_factory.mktemp("fixture-repo") + (repo / "widget.py").write_text("BROKEN = True\n") + for args in ( + ("init", "-q", "-b", "main"), + ("add", "-A"), + ("commit", "-qm", "buggy widget"), + ("branch", "bug_baseline"), + ("checkout", "-qb", "bug_golden"), + ): + git(repo, *args).check_returncode() + (repo / "widget.py").write_text("BROKEN = False\n") + git(repo, "commit", "-qam", "reference fix").check_returncode() + return repo + + +@pytest.fixture +def repo(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, fixture_repo: Path) -> Path: + """Configure the environment to clone ``fixture_repo``; returns the agent's repository.""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "0") + monkeypatch.setenv("REPO_URL", str(fixture_repo)) + monkeypatch.setenv("REPO_DIR", str(tmp_path / "repo")) + monkeypatch.setenv("BASELINE_DIR", str(tmp_path / "baseline")) + return tmp_path / "repo" + @pytest.fixture -def isolated_workspace(): - if sys.platform != "linux" or os.geteuid() != 0 or usable_bwrap() is None: - pytest.skip("requires Linux, root, and bubblewrap (run in the coding container)") - - -@pytest_asyncio.fixture(loop_scope="session") -async def grading_workspace(tmp_path, monkeypatch, isolated_workspace): - import env as coding_env - - root = tmp_path / "repo" - workspace = Workspace(root, guest_path=str(root), network=False, shell_uid=1000, require_isolation=True) - await workspace.start() - monkeypatch.setattr(coding_env, "workspace", workspace) - monkeypatch.setattr(coding_env, "REPO_DIR", root) - monkeypatch.setattr(coding_env, "_GIT", (*coding_env._GIT, "-c", f"safe.directory={root}")) - try: - yield workspace - finally: - await workspace.stop() +async def workspace(tmp_path: Path): + """An isolated workspace like the environment's, for grading commands directly.""" + root = tmp_path / "workspace" + served = Workspace(root, guest_path=str(root), network=False, shell_uid=1000, require_isolation=True) + await served.start() + yield served + await served.stop() diff --git a/environments/coding/tests/test_env.py b/environments/coding/tests/test_env.py index 38f64513f..e41de02e4 100644 --- a/environments/coding/tests/test_env.py +++ b/environments/coding/tests/test_env.py @@ -1,36 +1,172 @@ -"""In-process wiring smoke tests for the served environment.""" +"""The coding environment, served the way HUD serves it and driven over its control channel. -from unittest.mock import AsyncMock +Each test starts a fresh environment process, starts a task, edits the agent's +repository the way an agent would, and grades: what a rollout does, without an +agent. The workspace refuses to run unisolated, so every test here is ``sandbox``. +""" + +import shlex +import subprocess +import sys +from collections.abc import Callable +from contextlib import asynccontextmanager +from pathlib import Path import pytest +from hud import SubprocessRuntime, Task, connect +from hud.clients import HudProtocolError -import env as coding_env -from env import env +import tasks +from env import coding_task +pytestmark = pytest.mark.sandbox -def test_coding_template_registered(): - assert "coding-task" in env.tasks - assert env.tasks["coding-task"].manifest_entry()["id"] == "coding-task" +ENV_SOURCE = Path(__file__).resolve().parent.parent / "env.py" +PYTEST = f"{shlex.quote(sys.executable)} -m pytest -q -c /dev/null --rootdir=. test_widget.py --junitxml={{junit_path}}" +TEST_PATCH = """diff --git a/test_widget.py b/test_widget.py +new file mode 100644 +--- /dev/null ++++ b/test_widget.py +@@ -0,0 +1,10 @@ ++import unittest ++import widget ++ ++ ++class TestWidget(unittest.TestCase): ++ def test_not_broken(self): ++ self.assertFalse(widget.BROKEN) ++ ++ def test_existing_behavior(self): ++ self.assertTrue(hasattr(widget, 'BROKEN')) +""" -def test_description_declares_prompt_hint(): - args = env.tasks["coding-task"].manifest_entry()["args"]["properties"] - assert args["description"]["x-hud-hint"] == "prompt" +def widget_task(**overrides: object) -> Task: + return coding_task( + **{ + "description": "\nFix the widget.\n", + "test_command": PYTEST, + "test_patch": TEST_PATCH, + "test_path": "test_widget.py", + "base_ref": "origin/bug_baseline", + "fail_to_pass": ["test_widget.TestWidget.test_not_broken"], + "pass_to_pass": ["test_widget.TestWidget.test_existing_behavior"], + **overrides, + } + ) -@pytest.mark.asyncio -async def test_coding_task_uses_description_as_prompt(monkeypatch): - monkeypatch.setattr(coding_env, "_setup", AsyncMock()) - workspace = AsyncMock() - monkeypatch.setattr(coding_env, "workspace", workspace) - task = coding_env.coding_task.func( - description="\nFix the bug.\n", - test_command="pytest -q --junitxml={junit_path}", - test_patch="diff --git a/test_bug.py b/test_bug.py\n", - test_path="tests", - base_ref="main", +def git(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-c", f"safe.directory={repo}", *args], cwd=repo, check=False, capture_output=True, text=True ) - assert await anext(task) == "Fix the bug." - await task.aclose() - workspace.discard_sandbox.assert_awaited_once_with() + +@asynccontextmanager +async def served(row: Task): + """A fresh environment process serving ``row``, and a client connected to it.""" + async with SubprocessRuntime(ENV_SOURCE)(row) as runtime, connect(runtime) as client: + yield client + + +def leave_untouched(repo: Path) -> None: + del repo + + +def fix_the_widget(repo: Path) -> None: + (repo / "fix.py").write_text("BROKEN = False\n") + (repo / "widget.py").write_text("from fix import BROKEN\n") + + +def tamper_with_the_tests(repo: Path) -> None: + """Rewrite the hidden test's path and plant a git filter that would rewrite it on apply.""" + git(repo, "config", "filter.agent.clean", "cat").check_returncode() + (repo / ".gitattributes").write_text("* filter=agent\n") + (repo / "test_widget.py").write_text("def test_not_broken():\n assert True\n") + + +async def test_the_description_is_the_prompt(repo): + row = widget_task() + async with served(row) as client: + (template,) = await client.list_tasks() + started = await client.start_task(row.id, row.args) + + assert template["id"] == "coding-task" + assert template["args"]["properties"]["description"]["x-hud-hint"] == "prompt" + assert started["prompt"] == "Fix the widget." + assert (repo / "widget.py").read_text() == "BROKEN = True\n" + assert not (repo / "test_widget.py").exists() + + +@pytest.mark.parametrize( + ("overrides", "edit", "score"), + [ + pytest.param({}, leave_untouched, 0.5, id="untouched-baseline-keeps-regression-credit"), + pytest.param({}, fix_the_widget, 1.0, id="agent-fix"), + pytest.param({"base_ref": "origin/bug_golden"}, leave_untouched, 1.0, id="golden-ref"), + pytest.param({}, tamper_with_the_tests, 0.5, id="agent-test-changes-are-discarded"), + ], +) +async def test_hidden_tests_grade_the_agents_repository( + overrides: dict, edit: Callable[[Path], None], score: float, repo: Path +): + row = widget_task(**overrides) + async with served(row) as client: + await client.start_task(row.id, row.args) + edit(repo) + grade = await client.grade({"answer": "done"}) + + assert not grade["isError"], grade + assert grade["score"] == score + assert git(repo, "config", "--get", "filter.agent.clean").returncode == 1 + + +async def test_a_command_without_a_junit_report_is_a_grading_error(repo): + row = widget_task(test_command="true {junit_path}") + async with served(row) as client: + await client.start_task(row.id, row.args) + grade = await client.grade({"answer": "done"}) + + assert grade["isError"] is True + assert grade["content"] == "test command did not write JUnit XML" + assert grade["score"] == 0.0 + + +@pytest.mark.parametrize("test_path", ["tests/test_widget.py", "/test_widget.py", ".."]) +async def test_hidden_tests_must_sit_at_the_top_level(test_path: str, repo: Path): + row = widget_task(test_path=test_path) + async with served(row) as client: + with pytest.raises(HudProtocolError) as failure: + await client.start_task(row.id, row.args) + + assert "test_path must be a top-level file or directory" in failure.value.message + assert not repo.exists() or list(repo.iterdir()) == [] + + +async def test_a_new_task_starts_from_a_clean_baseline(repo): + row = widget_task() + async with served(row) as client: + await client.start_task(row.id, row.args) + fix_the_widget(repo) + await client.grade({"answer": "done"}) + await client.start_task(row.id, row.args) + + assert not (repo / "test_widget.py").exists() + assert not (repo / "fix.py").exists() + assert (repo / "widget.py").read_text() == "BROKEN = True\n" + + +@pytest.mark.parametrize("bundled", tasks.tasks, ids=lambda task: task.slug) +@pytest.mark.parametrize(("ref", "score"), [("baseline", 0.0), ("golden", 1.0)]) +async def test_bundled_tasks_score_zero_at_baseline_and_one_with_the_reference_fix( + bundled: Task, ref: str, score: float, repo: Path, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delenv("REPO_URL") + base_ref = bundled.args["base_ref"].replace("_baseline", f"_{ref}") + row = bundled.model_copy(update={"args": {**bundled.args, "base_ref": base_ref}}) + async with served(row) as client: + await client.start_task(row.id, row.args) + grade = await client.grade({"answer": "done"}) + + assert not grade["isError"], grade + assert grade["score"] == score diff --git a/environments/coding/tests/test_grader.py b/environments/coding/tests/test_grader.py index a9519f8db..f16125bbb 100644 --- a/environments/coding/tests/test_grader.py +++ b/environments/coding/tests/test_grader.py @@ -90,20 +90,10 @@ def test_parse_junit_rejects_malformed_report(tmp_path: Path): parse_junit(report) -@pytest.mark.asyncio(loop_scope="session") -async def test_junit_grader_reports_missing_output(grading_workspace): - result = await grade_tests(grading_workspace, "true {junit_path}") - - assert result.value == 0.0 - assert result.info is not None - assert result.info["error"] == "test command did not write JUnit XML" - assert result.info["exit_code"] == 0 - - -@pytest.mark.asyncio(loop_scope="session") -async def test_binary_junit_grader_rejects_passing_report_from_failed_command(grading_workspace): +@pytest.mark.sandbox +async def test_binary_junit_grader_rejects_passing_report_from_failed_command(workspace): result = await grade_tests( - grading_workspace, + workspace, command=( 'printf \'\' > {junit_path}; false' @@ -118,14 +108,14 @@ async def test_binary_junit_grader_rejects_passing_report_from_failed_command(gr assert "all_testcases" not in result.info -@pytest.mark.asyncio(loop_scope="session") -async def test_grading_runs_as_agent_after_session_termination(grading_workspace): - session = await grading_workspace.run(["/bin/bash", "-lc", "echo agent > agent.txt"]) +@pytest.mark.sandbox +async def test_grading_runs_as_agent_after_session_termination(workspace): + session = await workspace.run(["/bin/bash", "-lc", "echo agent > agent.txt"]) assert session.returncode == 0 - await grading_workspace.terminate_sessions() + await workspace.terminate_sessions() result = await grade_tests( - grading_workspace, + workspace, 'test "$(id -u)" = 1000 && test "$(cat agent.txt)" = agent && ' "grep -q localhost /etc/hosts && " 'printf \'\' > {junit_path}', @@ -136,10 +126,10 @@ async def test_grading_runs_as_agent_after_session_termination(grading_workspace assert result.info["exit_code"] == 0 -@pytest.mark.asyncio(loop_scope="session") -async def test_timed_out_grader_does_not_score_partial_report(grading_workspace): +@pytest.mark.sandbox +async def test_timed_out_grader_does_not_score_partial_report(workspace): result = await grade_tests( - grading_workspace, + workspace, 'printf \'\' > {junit_path}; sleep 30', timeout_seconds=1, ) diff --git a/environments/coding/tests/test_local_rollout.py b/environments/coding/tests/test_local_rollout.py deleted file mode 100644 index b444ce9c4..000000000 --- a/environments/coding/tests/test_local_rollout.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Hermetic end-to-end validation of repository setup and grading.""" - -import os -import shlex -import subprocess -import sys -from pathlib import Path - -import pytest -from hud import LocalRuntime, Run, connect -from hud.environment import workspace as workspace_lib - -import tasks - -PROJECT_ROOT = Path(__file__).resolve().parent.parent -PYTEST_COMMAND = ( - f"{shlex.quote(sys.executable)} -m pytest -q -c /dev/null --rootdir=. test_widget.py --junitxml={{junit_path}}" -) -TEST_PATCH = """diff --git a/test_widget.py b/test_widget.py -new file mode 100644 ---- /dev/null -+++ b/test_widget.py -@@ -0,0 +1,10 @@ -+import unittest -+import widget -+ -+ -+class TestWidget(unittest.TestCase): -+ def test_not_broken(self): -+ self.assertFalse(widget.BROKEN) -+ -+ def test_existing_behavior(self): -+ self.assertTrue(hasattr(widget, 'BROKEN')) -""" - -pytestmark = pytest.mark.asyncio(loop_scope="session") - - -def _git(cwd: Path, *args: str) -> None: - subprocess.run( - ["git", "-c", f"safe.directory={cwd}", "-c", "user.name=t", "-c", "user.email=t@t", *args], - cwd=cwd, - check=True, - capture_output=True, - ) - - -@pytest.fixture(scope="session") -def fixture_repo(tmp_path_factory) -> Path: - repo = tmp_path_factory.mktemp("fixture-repo") - (repo / "widget.py").write_text("BROKEN = True\n") - _git(repo, "init", "-q", "-b", "main") - _git(repo, "add", "-A") - _git(repo, "commit", "-qm", "buggy widget") - _git(repo, "branch", "bug_baseline") - - _git(repo, "checkout", "-qb", "bug_golden") - (repo / "widget.py").write_text("BROKEN = False\n") - _git(repo, "add", "-A") - _git(repo, "commit", "-qm", "reference fix") - return repo - - -def _coding_task(): - from env import coding_task - - return coding_task( - description="Fix the widget.", - test_command=f'test -n "${{HOME}}" && {PYTEST_COMMAND}', - test_patch=TEST_PATCH, - test_path="test_widget.py", - base_ref="origin/bug_baseline", - fail_to_pass=["test_widget.TestWidget.test_not_broken"], - pass_to_pass=["test_widget.TestWidget.test_existing_behavior"], - ) - - -async def _run_task(fixture_repo: Path, task, monkeypatch) -> float: - monkeypatch.setenv("REPO_URL", str(fixture_repo)) - monkeypatch.delenv("REPO_DIR", raising=False) - monkeypatch.delenv("BASELINE_DIR", raising=False) - runtime = LocalRuntime(str(PROJECT_ROOT / "env.py")) - async with runtime(task) as addr, connect(addr) as client: - async with Run(client, task.id, task.args) as run: - pass - assert not run.evaluation.get("isError"), run.evaluation - return run.reward - - -async def test_agent_fix_scores_one(fixture_repo, tmp_path, monkeypatch, grading_workspace): - import env as coding_env - - repo = grading_workspace.root - subprocess.run(["git", "clone", "-q", str(fixture_repo), str(repo)], check=True) - monkeypatch.setattr(coding_env, "REPO_DIR", repo) - monkeypatch.setattr(coding_env, "BASELINE_DIR", tmp_path / "baseline") - monkeypatch.setattr(coding_env, "REPO_SOURCE", str(fixture_repo)) - - task = coding_env.coding_task.func(**_coding_task().args) - await task.asend(None) - (repo / "fix.py").write_text("BROKEN = False\n") - (repo / "widget.py").write_text("from fix import BROKEN\n") - result = await task.asend("done") - - assert result.reward == 1.0, result - assert (repo / "fix.py").exists() - - -async def test_missing_junit_report_is_grading_error( - fixture_repo, - tmp_path, - monkeypatch, - grading_workspace, -): - import env as coding_env - - repo = grading_workspace.root - subprocess.run(["git", "clone", "-q", str(fixture_repo), str(repo)], check=True) - monkeypatch.setattr(coding_env, "REPO_DIR", repo) - monkeypatch.setattr(coding_env, "BASELINE_DIR", tmp_path / "baseline") - monkeypatch.setattr(coding_env, "REPO_SOURCE", str(fixture_repo)) - - task = coding_env.coding_task.func( - description="Fix the widget.", - test_command="true {junit_path}", - test_patch=TEST_PATCH, - test_path="test_widget.py", - base_ref="origin/bug_baseline", - ) - await task.asend(None) - result = await task.asend("done") - - assert result.reward == 0.0 - assert result.isError is True - assert result.content == "test command did not write JUnit XML" - assert result.info["exit_code"] == 0 - - -async def test_untouched_baseline_gets_regression_credit(fixture_repo, isolated_workspace, monkeypatch): - assert await _run_task(fixture_repo, _coding_task(), monkeypatch) == 0.5 - - -@pytest.mark.parametrize("task", tasks.tasks, ids=lambda task: task.slug) -@pytest.mark.parametrize("fixed", [False, True], ids=["baseline", "reference-fix"]) -async def test_bundled_tasks_grade_in_workspace(task, fixed, isolated_workspace, tmp_path, monkeypatch): - bundle = tmp_path / "flask.bundle" - bundle.write_bytes((PROJECT_ROOT / "flask.bundle").read_bytes()) - if fixed: - task = task.model_copy( - update={"args": {**task.args, "base_ref": f"origin/{task.slug.replace('-', '_')}_golden"}} - ) - assert await _run_task(bundle, task, monkeypatch) == float(fixed) - - -async def test_local_runtime_refuses_unisolated_non_root_workspace( - fixture_repo, - monkeypatch, -): - monkeypatch.setattr(workspace_lib, "usable_bwrap", lambda: None) - monkeypatch.setattr(os, "geteuid", lambda: 501) - os.environ["REPO_URL"] = str(fixture_repo) - - runtime = LocalRuntime(str(PROJECT_ROOT / "env.py")) - with pytest.raises(RuntimeError, match="isolation was required"): - async with runtime(_coding_task()): - pass - - -async def test_grading_discards_agent_git_config_and_test_changes( - fixture_repo, - tmp_path, - monkeypatch, - grading_workspace, -): - import env as coding_env - - repo = grading_workspace.root - subprocess.run(["git", "clone", "-q", str(fixture_repo), str(repo)], check=True) - monkeypatch.setattr(coding_env, "REPO_DIR", repo) - monkeypatch.setattr(coding_env, "BASELINE_DIR", tmp_path / "baseline") - monkeypatch.setattr(coding_env, "REPO_SOURCE", str(fixture_repo)) - - task = coding_env.coding_task.func( - description="Fix the widget.", - test_command=PYTEST_COMMAND, - test_patch=TEST_PATCH, - test_path="test_widget.py", - base_ref="origin/bug_baseline", - fail_to_pass=["test_widget.TestWidget.test_not_broken"], - pass_to_pass=["test_widget.TestWidget.test_existing_behavior"], - ) - await task.asend(None) - - _git(repo, "config", "filter.agent.clean", "cat") - (repo / ".gitattributes").write_text("* filter=agent\n") - (repo / "test_widget.py").write_text("def test_not_broken():\n assert True\n") - - result = await task.asend("done") - - assert result.reward == 0.5, result - config = subprocess.run( - ["git", "-c", f"safe.directory={repo}", "config", "--get", "filter.agent.clean"], - cwd=repo, - capture_output=True, - check=False, - ) - assert config.returncode == 1 - - -async def test_setup_removes_files_from_the_previous_hidden_patch( - fixture_repo, - tmp_path, - monkeypatch, -): - import env as coding_env - - repo = tmp_path / "repo" - subprocess.run(["git", "clone", "-q", str(fixture_repo), str(repo)], check=True) - monkeypatch.setattr(coding_env, "REPO_DIR", repo) - monkeypatch.setattr(coding_env, "BASELINE_DIR", tmp_path / "baseline") - monkeypatch.setattr(coding_env, "REPO_SOURCE", str(fixture_repo)) - - await coding_env._setup("origin/bug_baseline") - subprocess.run( - ["git", "apply", "-"], - cwd=repo, - input=TEST_PATCH, - text=True, - check=True, - ) - assert (repo / "test_widget.py").exists() - - await coding_env._setup("origin/bug_baseline") - - assert not (repo / "test_widget.py").exists() diff --git a/environments/coding/tests/test_tasks.py b/environments/coding/tests/test_tasks.py index b9c2ef20b..d7448fa6a 100644 --- a/environments/coding/tests/test_tasks.py +++ b/environments/coding/tests/test_tasks.py @@ -1,10 +1,8 @@ """Task-row tests for the bundled coding tasks.""" -import subprocess from pathlib import Path import tasks -from grader import parse_junit, score_tests PROJECT_ROOT = Path(__file__).resolve().parent.parent GOLDEN_REFS = { @@ -28,42 +26,3 @@ def test_rows_parameterize_the_same_coding_task_template(): assert task.args["binary"] is True assert "test_ref" not in task.args assert "golden_ref" not in task.args - - -def test_bundled_task_baselines_fail_and_reference_fixes_pass(tmp_path: Path): - for task in tasks.tasks: - for ref, expected_exit_code, expected_reward in ( - (task.args["base_ref"], 1, 0.0), - (GOLDEN_REFS[task.slug], 0, 1.0), - ): - repo = tmp_path / f"{task.slug}-{ref.rsplit('/', 1)[-1]}" - subprocess.run( - ["git", "clone", "-q", str(PROJECT_ROOT / "flask.bundle"), str(repo)], - check=True, - ) - subprocess.run(["git", "checkout", "-qf", ref], cwd=repo, check=True) - subprocess.run( - ["git", "apply", "-"], - cwd=repo, - input=task.args["test_patch"], - text=True, - check=True, - ) - (repo / "conftest.py").write_text('raise RuntimeError("agent conftest loaded")\n') - (repo / "src" / "sitecustomize.py").write_text('raise RuntimeError("agent sitecustomize loaded")\n') - - junit_path = tmp_path / f"{repo.name}.xml" - command = task.args["test_command"].replace("{junit_path}", str(junit_path)) - completed = subprocess.run( - ["bash", "-lc", command], - cwd=repo, - check=False, - ) - assert completed.returncode == expected_exit_code - result = score_tests( - parse_junit(junit_path), - task.args["fail_to_pass"], - task.args["pass_to_pass"], - task.args["binary"], - ) - assert result.value == expected_reward diff --git a/environments/cua/env.py b/environments/cua/env.py index 3405d2d5a..cccb0bb60 100644 --- a/environments/cua/env.py +++ b/environments/cua/env.py @@ -4,6 +4,7 @@ # param crashes the sync/deploy manifest path (TypeAdapter on a string forward-ref). Keep # annotations as real objects. import asyncio +import os from collections.abc import Awaitable from typing import Annotated, Any @@ -17,7 +18,8 @@ _task_started = False _HOST = "127.0.0.1" -_VNC_PORT = 5900 +# supervisord runs x11vnc on 5900; tests point this at their own VNC listener. +_VNC_PORT = int(os.environ.get("VNC_PORT", "5900")) async def _listening(host: str, port: int, timeout: float = 30.0) -> None: @@ -38,7 +40,7 @@ async def _listening(host: str, port: int, timeout: float = 30.0) -> None: @env.initialize async def _up() -> None: await _listening(_HOST, _VNC_PORT) - env.add_capability(Capability.rfb(name="screen", url=f"rfb://{_HOST}", display=0)) + env.add_capability(Capability.rfb(name="screen", url=f"rfb://{_HOST}:{_VNC_PORT}")) @env.template() diff --git a/environments/cua/tests/conftest.py b/environments/cua/tests/conftest.py index 84d71f1a7..4409a269e 100644 --- a/environments/cua/tests/conftest.py +++ b/environments/cua/tests/conftest.py @@ -1,8 +1,73 @@ -"""Test fixtures for the CUA environment grader logic (offline, no desktop).""" +"""Serve the environment the way HUD does, against local stand-ins for the desktop and the judge. +Nothing here needs a real desktop or a HUD account: a TCP listener stands in for +the VNC server, and a small HTTP server stands in for the inference gateway the +LLM judge calls. The environment runs in its own process, configured only +through environment variables, exactly as ``hud eval`` serves it. +""" + +from __future__ import annotations + +import asyncio +import json import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -ROOT = str(Path(__file__).parent.parent) -if ROOT not in sys.path: - sys.path.insert(0, ROOT) +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + +class Judge: + """The inference gateway, judging every criterion MET. ``requests`` records what it was sent.""" + + def __init__(self) -> None: + self.requests: list[tuple[str, dict]] = [] + judge = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + judge.requests.append((self.headers["Authorization"], body)) + verdict = json.dumps({"criterion_status": "MET", "explanation": "stand-in judge"}) + reply = json.dumps({"choices": [{"message": {"content": verdict}}]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + + def log_message(self, format: str, *args: object) -> None: + del format, args + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.server.server_port}" + + +@pytest.fixture(autouse=True) +def judge(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + """Point the environment at the stand-in judge, with no HUD key and no user config.""" + stand_in = Judge() + thread = threading.Thread(target=stand_in.server.serve_forever, daemon=True) + thread.start() + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("HUD_API_KEY", "") + monkeypatch.setenv("HUD_GATEWAY_URL", stand_in.url) + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "0") + yield stand_in + stand_in.server.shutdown() + thread.join() + + +@pytest.fixture(autouse=True) +async def desktop(monkeypatch: pytest.MonkeyPatch): + """A listener on the environment's VNC port, so it starts without a real desktop.""" + server = await asyncio.start_server(lambda reader, writer: writer.close(), "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + monkeypatch.setenv("VNC_PORT", str(port)) + async with server: + yield port diff --git a/environments/cua/tests/test_env.py b/environments/cua/tests/test_env.py index 618eedcc2..cf92db1a2 100644 --- a/environments/cua/tests/test_env.py +++ b/environments/cua/tests/test_env.py @@ -1,138 +1,115 @@ -"""Offline tests for the CUA environment grader composition. +"""The CUA environment, served the way HUD serves it and driven over its control channel. -These do NOT touch the virtual desktop (rfb is Linux-only); they drive the @env.template -generator directly and exercise its configuration and grader composition with deterministic shell -commands. The end-to-end desktop run is a `hud eval ... --runtime hud` rollout. +Each test starts a fresh environment process, starts a task with the arguments a +task row carries, and grades an answer: what a rollout does, without an agent. """ -from unittest.mock import AsyncMock +from __future__ import annotations + +from contextlib import asynccontextmanager +from pathlib import Path import pytest +from hud import SubprocessRuntime, Task, connect +from hud.clients import HudProtocolError + +from env import cua_task + +ENV_SOURCE = Path(__file__).resolve().parent.parent / "env.py" +PASS = {"name": "ok", "command": "true", "weight": 1.0} + + +@asynccontextmanager +async def served(row: Task): + """A fresh environment process serving ``row``, and a client connected to it.""" + async with SubprocessRuntime(ENV_SOURCE)(row) as runtime, connect(runtime) as client: + yield client + -import env as M -import tasks - -GEN = M.cua_task.func - - -@pytest.fixture(autouse=True) -def fresh_substrate(monkeypatch): - monkeypatch.setattr(M, "_task_started", False) - - -class TestGrading: - async def test_missing_graders_fails_before_prompt(self): - gen = GEN(prompt="p") - with pytest.raises(ValueError, match="at least one grader"): - await gen.asend(None) - - async def test_bash_check_passes(self): - gen = GEN(prompt="p", bash_checks=[{"name": "ok", "command": "true", "weight": 1.0}]) - await gen.asend(None) - assert (await gen.asend("a")).reward == 1.0 - - async def test_bash_check_fails(self): - gen = GEN(prompt="p", bash_checks=[{"name": "no", "command": "false", "weight": 1.0}]) - await gen.asend(None) - assert (await gen.asend("a")).reward == 0.0 - - async def test_negative_bash_weight_fails_before_prompt(self): - gen = GEN(prompt="p", bash_checks=[{"name": "no", "command": "false", "weight": -1.0}]) - with pytest.raises(ValueError, match="nonnegative"): - await gen.asend(None) - - @pytest.mark.parametrize("hud_api_key", [None, "task-key"]) - async def test_no_runtime_key_judge_fails_before_prompt(self, monkeypatch, hud_api_key): - monkeypatch.setattr(M.settings, "api_key", "", raising=False) - gen = GEN(prompt="p", grading_criteria=["anything"], hud_api_key=hud_api_key) - with pytest.raises(RuntimeError, match="HUD_API_KEY"): - await gen.asend(None) - - async def test_weights_normalize_across_bash_and_judge(self, monkeypatch): - monkeypatch.setattr(M.settings, "api_key", "key", raising=False) - monkeypatch.setattr( - M.LLMJudgeGrader, - "grade", - AsyncMock(return_value=M.SubScore(name="llm_judge", value=1.0, weight=0.5)), - ) - gen = GEN( - prompt="p", - bash_checks=[{"name": "ok", "command": "true", "weight": 0.3}], - grading_criteria=["x"], - ) - await gen.asend(None) - result = await gen.asend("answer") - assert result.reward == 1.0 - weights = {subscore.name: subscore.weight for subscore in result.subscores} - assert weights == {"ok": 0.5, "llm_judge": 0.5} - - async def test_second_task_requires_fresh_substrate(self): - first = GEN(prompt="first", bash_checks=[{"name": "ok", "command": "true"}]) - await first.asend(None) - - second = GEN(prompt="second", bash_checks=[{"name": "ok", "command": "true"}]) - with pytest.raises(RuntimeError, match="one task per substrate"): - await second.asend(None) - - await first.aclose() - - async def test_named_bash_subscores_preserved(self): - gen = GEN( - prompt="p", - bash_checks=[ - {"name": "alpha", "command": "true", "weight": 0.4}, - {"name": "beta", "command": "true", "weight": 0.6}, - ], - ) - await gen.asend(None) - result = await gen.asend("a") - names = {s.name for s in result.subscores} - assert {"alpha", "beta"} <= names - assert result.reward == 1.0 - - -def test_create_document_requires_exact_contents(): - command = tasks._create_document.args["bash_checks"][1]["command"] - assert command == "cmp -s /home/ubuntu/Desktop/hello.txt <(printf 'Hello from HUD!\\n')" - - -def test_open_website_requires_wikipedia_page(): - check = tasks._open_website.args["bash_checks"][0] - assert check == { - "name": "visited_wikipedia", - "command": ( - "curl -fsS http://127.0.0.1:9222/json/list " - "| jq -e 'any(.[]; .url == \"https://www.wikipedia.org/\")' >/dev/null" +async def test_the_manifest_publishes_the_screen_and_a_prompt_box(desktop): + row = cua_task(prompt="p", bash_checks=[PASS]) + async with served(row) as client: + (template,) = await client.list_tasks() + screens = [binding.url for binding in client.manifest.bindings if binding.name == "screen"] + + assert screens == [f"rfb://127.0.0.1:{desktop}"] + arguments = template["args"] + assert arguments["properties"]["prompt"]["x-hud-hint"] == "prompt" + assert "hud_api_key" in arguments["properties"] + assert arguments["required"] == ["prompt"] + + +@pytest.mark.parametrize( + ("arguments", "message"), + [ + pytest.param({}, "at least one grader", id="no-graders"), + pytest.param( + {"bash_checks": [{"name": "no", "command": "false", "weight": -1.0}]}, + "nonnegative", + id="negative-weight", + ), + pytest.param({"grading_criteria": ["Anything."]}, "HUD_API_KEY is required", id="judge-without-key"), + pytest.param( + {"grading_criteria": ["Anything."], "hud_api_key": "task-key"}, + "HUD_API_KEY is required", + id="a-task-key-is-not-the-runtime-key", + ), + ], +) +async def test_a_misconfigured_task_fails_before_the_agent_sees_a_prompt(arguments, message): + row = cua_task(prompt="p", **arguments) + async with served(row) as client: + with pytest.raises(HudProtocolError) as failure: + await client.start_task(row.id, row.args) + + assert message in failure.value.message + + +@pytest.mark.parametrize( + ("arguments", "score", "weights"), + [ + pytest.param({"bash_checks": [PASS]}, 1.0, {"ok": 1.0}, id="passing-check"), + pytest.param({"bash_checks": [{"name": "no", "command": "false"}]}, 0.0, {"no": 1.0}, id="failing-check"), + pytest.param( + { + "bash_checks": [ + {"name": "alpha", "command": "true", "weight": 0.4}, + {"name": "beta", "command": "false", "weight": 0.6}, + ] + }, + 0.4, + {"alpha": 0.4, "beta": 0.6}, + id="weighted-checks", + ), + pytest.param( + { + "bash_checks": [{"name": "ok", "command": "true", "weight": 0.3}], + "grading_criteria": ["Says hello."], + }, + 1.0, + {"ok": 0.5, "llm_judge": 0.5}, + id="checks-and-judge-split-the-score", ), - "weight": 1.0, - } - - -def test_shannon_research_requires_exact_file_and_open_pages(): - assert tasks._shannon_research.args["bash_checks"] == [ - { - "name": "file_contents", - "command": ( - "cmp -s /home/ubuntu/Desktop/shannon.txt " - "<(printf 'born: 1916\\nphd: Massachusetts Institute of Technology\\n" - "city: Cambridge, Massachusetts\\n')" - ), - "weight": 1.0, - }, - { - "name": "research_pages_open", - "command": ( - "curl -fsS http://127.0.0.1:9222/json/list | jq -e " - "--arg shannon https://en.wikipedia.org/wiki/Claude_Shannon " - "--arg mit https://en.wikipedia.org/wiki/Massachusetts_Institute_of_Technology " - "'map(.url) | contains([$shannon, $mit])' >/dev/null" - ), - "weight": 1.0, - }, - ] - - -def test_platform_args_declared(): - args = M.cua_task.manifest_entry()["args"]["properties"] - assert args["prompt"]["x-hud-hint"] == "prompt" - assert "hud_api_key" in args + ], +) +async def test_the_grade_weighs_named_checks_and_the_judge(arguments, score, weights, monkeypatch, judge): + monkeypatch.setenv("HUD_API_KEY", "runtime-key") + row = cua_task(prompt="Say hello.", hud_api_key="task-key", **arguments) + async with served(row) as client: + started = await client.start_task(row.id, row.args) + grade = await client.grade({"answer": "hello"}) + + assert started["prompt"].endswith("Say hello.") + assert grade["score"] == pytest.approx(score) + assert {sub["name"]: sub["weight"] for sub in grade["subscores"]} == pytest.approx(weights) + assert {authorization for authorization, _ in judge.requests} <= {"Bearer runtime-key"} + + +async def test_a_desktop_serves_one_task(): + row = cua_task(prompt="first", bash_checks=[PASS]) + async with served(row) as client: + await client.start_task(row.id, row.args) + with pytest.raises(HudProtocolError) as failure: + await client.start_task(row.id, {**row.args, "prompt": "second"}) + + assert "one task per substrate" in failure.value.message diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 5f4d821ae..90077770d 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -51,7 +51,7 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: else: entry = resolve_gateway_model(model) agent_type = AgentType(entry.sdk_agent_type) - kwargs.setdefault("model", entry.model_name or model) + kwargs.setdefault("model", entry.model_name or entry.id) kwargs["gateway"] = True return cast("GatewayAgent", agent_type.cls.load(kwargs)) diff --git a/hud/agents/browser_use/__init__.py b/hud/agents/browser_use/__init__.py deleted file mode 100644 index e175afb1c..000000000 --- a/hud/agents/browser_use/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""browser-use SDK integration (optional dependency ``hud[browseruse]``).""" - -from .agent import BrowserUseAgent, BrowserUseConfig - -__all__ = ["BrowserUseAgent", "BrowserUseConfig"] diff --git a/hud/agents/browser_use/agent.py b/hud/agents/browser_use/agent.py deleted file mode 100644 index ee88bec3b..000000000 --- a/hud/agents/browser_use/agent.py +++ /dev/null @@ -1,137 +0,0 @@ -"""BrowserUseAgent — delegates browser control to the ``browser-use`` SDK. - -The env publishes a ``cdp/1.3`` capability (a Chromium DevTools endpoint); this -agent reads that binding's URL and hands it to ``browser-use``, which drives -the browser over its own CDP client. We do **not** ``open`` one of our own -``CapabilityClient`` connections — browser-use owns the session — so this -agent uses ``client.binding(...)`` (wire data) rather than ``client.open(...)`` -(managed client). - -The agent is stateless w.r.t. the env: it holds only config and is driven by -``await agent(run)``, receiving the run handle per call. ``browser-use`` is an -optional dependency -(``hud[browseruse]``), imported lazily inside ``rollout``. -""" - -from __future__ import annotations - -import contextlib -import importlib -import logging -from typing import TYPE_CHECKING, Protocol, cast -from urllib.parse import urlsplit, urlunsplit - -from hud.agents.base import Agent -from hud.agents.types import AgentStep, BrowserUseConfig -from hud.settings import settings - -if TYPE_CHECKING: - from hud.eval.run import Run - -LOGGER = logging.getLogger("hud.agents.browser_use") - -CDP_PROTOCOL = "cdp/1.3" - - -class _Browser(Protocol): - async def stop(self) -> None: ... - - -class _History(Protocol): - def is_successful(self) -> bool | None: ... - - def final_result(self) -> str | None: ... - - def number_of_steps(self) -> int: ... - - def urls(self) -> list[str]: ... - - def is_done(self) -> bool: ... - - -class _BrowserAgent(Protocol): - async def run(self, *, max_steps: int) -> _History: ... - - -class _ChatAnthropicFactory(Protocol): - def __call__(self, *, model: str, api_key: str, base_url: str | None) -> object: ... - - -class _BrowserFactory(Protocol): - def __call__(self, *, cdp_url: str) -> _Browser: ... - - -class _AgentFactory(Protocol): - def __call__(self, *, task: str, llm: object, browser: _Browser) -> _BrowserAgent: ... - - -class _BrowserUseModule(Protocol): - ChatAnthropic: _ChatAnthropicFactory - Browser: _BrowserFactory - Agent: _AgentFactory - - -class BrowserUseAgent(Agent[BrowserUseConfig]): - """Run the ``browser-use`` agent against an env's ``cdp/1.3`` capability.""" - - config_cls = BrowserUseConfig - - async def __call__(self, run: Run) -> None: - """Drive browser-use over the run's CDP capability, filling ``run.trace``. - - Reads ``run.prompt_text`` and the CDP binding off the run, runs the browser-use - loop, and writes the final answer + trajectory metadata onto ``run.trace`` - (graded on exit). - """ - browser_use = cast("_BrowserUseModule", importlib.import_module("browser_use")) - - trace = run.trace - cdp_url = _ws_to_http(run.client.binding(CDP_PROTOCOL).url) - LOGGER.info("browser-use attaching to %s", cdp_url) - - api_key = self.config.api_key or settings.anthropic_api_key - if not api_key: - raise ValueError("BrowserUseAgent needs an Anthropic API key (set ANTHROPIC_API_KEY)") - - llm = browser_use.ChatAnthropic( - model=self.config.model, - api_key=api_key, - base_url=self.config.base_url, - ) - browser = browser_use.Browser(cdp_url=cdp_url) - sdk_agent = browser_use.Agent(task=run.prompt_text, llm=llm, browser=browser) - - try: - history = await sdk_agent.run(max_steps=self.config.max_steps) - finally: - with contextlib.suppress(Exception): - await browser.stop() - - successful = history.is_successful() - content = history.final_result() or "" - trace.content = content - trace.extra.update( - { - "is_successful": successful, - "steps": history.number_of_steps(), - "urls": history.urls(), - } - ) - # browser-use owns its own loop; record the run as one coarse agent step - # (per-action fidelity would need a browser-use-native serializer). - run.record( - AgentStep( - content=content, - done=history.is_done(), - ), - ) - - -def _ws_to_http(url: str) -> str: - """Map a ``ws(s)://`` CDP endpoint to the ``http(s)://`` form browser-use expects.""" - parts = urlsplit(url) - scheme = {"ws": "http", "wss": "https"}.get(parts.scheme, parts.scheme) - return urlunsplit((scheme, parts.netloc, parts.path, parts.query, parts.fragment)) - - -__all__ = ["BrowserUseAgent", "BrowserUseConfig"] diff --git a/hud/agents/claude/agent.py b/hud/agents/claude/agent.py index 54e15e9e6..edbd33564 100644 --- a/hud/agents/claude/agent.py +++ b/hud/agents/claude/agent.py @@ -45,6 +45,7 @@ from .images import image_source from .tools.coding import ClaudeBashTool, ClaudeTextEditorTool from .tools.computer import ClaudeComputerTool +from .tools.hosted import ClaudeToolSearchTool from .tools.mcp_proxy import ClaudeMCPProxyTool if TYPE_CHECKING: @@ -311,7 +312,7 @@ async def get_response( ) betas: list[str] | Omit = list(required_betas) if required_betas else Omit() tool_choice = BetaToolChoiceAutoParam(type="auto", disable_parallel_tool_use=True) - tools = cast("list[BetaToolUnionParam]", list(state.params)) + tools = self._defer_for_tool_search(cast("list[BetaToolUnionParam]", list(state.params))) system = system_prompt if system_prompt is not None else Omit() is_bedrock = isinstance(self.anthropic_client, AsyncAnthropicBedrock) @@ -388,6 +389,27 @@ async def get_response( return self.message_to_agent_step(response, citations_enabled=citations_enabled) + def _defer_for_tool_search(self, tools: list[BetaToolUnionParam]) -> list[BetaToolUnionParam]: + """Defer custom tools to tool search once there are more than its threshold.""" + search = next( + ( + hosted + for hosted in self.config.hosted_tools + if isinstance(hosted, ClaudeToolSearchTool) + and hosted.supports_model(self.config.model) + ), + None, + ) + custom = [tool for tool in tools if "type" not in tool or tool["type"] == "custom"] + if search is None or len(custom) <= search.threshold: + return tools + return [ + cast("BetaToolUnionParam", {**tool, "defer_loading": True}) + if "type" not in tool or tool["type"] == "custom" + else tool + for tool in tools + ] + @staticmethod def _prune_tool_result_images(messages: list[BetaMessageParam], limit: int) -> None: retained = 0 diff --git a/hud/agents/claude/tools/hosted.py b/hud/agents/claude/tools/hosted.py index d7076acb3..4a6e34413 100644 --- a/hud/agents/claude/tools/hosted.py +++ b/hud/agents/claude/tools/hosted.py @@ -81,7 +81,11 @@ def to_params(self) -> BetaWebFetchTool20250910Param: @dataclass(frozen=True, kw_only=True) class ClaudeToolSearchTool(ClaudeHostedTool): - """Claude tool search for large tool sets.""" + """Claude tool search for large tool sets. + + When the request carries more custom tools than ``threshold``, the agent + marks them ``defer_loading`` so the model finds them through search. + """ threshold: int = 10 diff --git a/hud/agents/claude/tools/tests/test_computer.py b/hud/agents/claude/tools/tests/test_computer.py deleted file mode 100644 index 2d0f22bdd..000000000 --- a/hud/agents/claude/tools/tests/test_computer.py +++ /dev/null @@ -1,227 +0,0 @@ -"""``ClaudeComputerTool`` — key translation, per-model spec gating, and the -computer-use action dispatch (translation to RFB primitives), without a live VNC. -""" - -from __future__ import annotations - -from io import BytesIO -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, patch - -import mcp.types as mcp_types -from PIL import Image - -from hud.agents.claude.tools.computer import ( - CLAUDE_COMPUTER_SPECS, - GENERIC_COMPUTER_SPEC, - ClaudeComputerTool, - _crop_png, - _hold_keys, - _split_keys, - _translate_key, -) - -if TYPE_CHECKING: - from collections.abc import Iterable -from hud.agents.tools.base import result_text, tool_ok -from hud.capabilities.rfb import WebPScreenshotEncoding - - -class RecordingComputer(ClaudeComputerTool): - """Bypasses RFBTool init; records the primitive calls dispatch makes.""" - - client: Any - - def __init__(self) -> None: - self.calls: list[tuple[Any, ...]] = [] - self.screenshot_encoding = WebPScreenshotEncoding(quality=42) - self.client = SimpleNamespace(width=200, height=100) - - async def screenshot(self) -> Any: - self.calls.append(("screenshot",)) - return tool_ok("shot") - - async def click( - self, - x: int | None = None, - y: int | None = None, - *, - button: Any = "left", - hold_keys: Iterable[str] | None = None, - count: int = 1, - interval_ms: int = 0, - ) -> None: - kw: dict[str, Any] = {} - if button != "left": - kw["button"] = button - if hold_keys is not None: - kw["hold_keys"] = hold_keys - if count != 1: - kw["count"] = count - if interval_ms: - kw["interval_ms"] = interval_ms - self.calls.append(("click", x, y, kw)) - - async def move(self, x: Any, y: Any) -> None: - self.calls.append(("move", x, y)) - - async def mouse_down(self, button: Any = "left") -> None: - self.calls.append(("down", button)) - - async def mouse_up(self, button: Any = "left") -> None: - self.calls.append(("up", button)) - - async def type_text(self, text: Any) -> None: - self.calls.append(("type", text)) - - async def press_keys(self, keys: Any, **kw: Any) -> None: - self.calls.append(("keys", tuple(keys), kw)) - - async def hold_key(self, key: Any, **kw: Any) -> None: - self.calls.append(("hold", key, kw)) - - async def scroll( - self, - x: int | None = None, - y: int | None = None, - *, - scroll_x: int = 0, - scroll_y: int = 0, - hold_keys: Iterable[str] | None = None, - ) -> None: - kw = {"scroll_x": scroll_x, "scroll_y": scroll_y, "hold_keys": hold_keys} - self.calls.append(("scroll", x, y, kw)) - - async def drag(self, path: Any, **kw: Any) -> None: - self.calls.append(("drag", tuple(path), kw)) - - -# ─── key translation helpers ────────────────────────────────────────── - - -def test_translate_key_maps_anthropic_to_x11() -> None: - assert _translate_key("Return") == "Return" - assert _translate_key("ctrl") == "Control_L" - assert _translate_key("ctrl+c") == "Control_L+c" - - -def test_split_and_hold_keys() -> None: - assert _split_keys("ctrl+c") == ["Control_L", "c"] - assert _split_keys(None) == [] - assert _split_keys("") == [] - assert _hold_keys(None) is None - assert _hold_keys("alt") == ["Alt_L"] - - -# ─── spec gating + params ───────────────────────────────────────────── - - -def test_default_spec_per_model() -> None: - spec_45 = ClaudeComputerTool.default_spec("claude-sonnet-4-5-20250101") - assert spec_45 is not None - assert spec_45.api_type == "computer_20250124" - # Unknown Claude model falls back to the latest spec. - spec_unknown = ClaudeComputerTool.default_spec("claude-totally-unknown") - assert spec_unknown is not None - assert spec_unknown.api_type == "computer_20251124" - # Non-Anthropic models get the generic input_schema spec. - assert ClaudeComputerTool.default_spec("qwen/qwen3.8-max") is GENERIC_COMPUTER_SPEC - - -def test_to_params_reflects_spec_version() -> None: - tool = RecordingComputer() - tool.spec = CLAUDE_COMPUTER_SPECS[0] - assert tool.to_params().get("type") == "computer_20251124" - tool.spec = CLAUDE_COMPUTER_SPECS[1] - assert tool.to_params().get("type") == "computer_20250124" - - -def test_to_params_generic_spec_uses_input_schema() -> None: - tool = RecordingComputer() - tool.spec = GENERIC_COMPUTER_SPEC - params: Any = tool.to_params() - assert "type" not in params - assert params["name"] == "computer" - assert "200x100" in params["description"] - assert params["input_schema"]["required"] == ["action"] - assert "left_click" in params["input_schema"]["properties"]["action"]["enum"] - - -# ─── action dispatch ────────────────────────────────────────────────── - - -async def test_left_click_then_screenshot() -> None: - tool = RecordingComputer() - await tool.execute({"action": "left_click", "coordinate": [10, 20], "text": "ctrl"}) - assert tool.calls[0] == ("click", 10, 20, {"hold_keys": ["Control_L"]}) - assert tool.calls[-1] == ("screenshot",) - - -async def test_type_action() -> None: - tool = RecordingComputer() - await tool.execute({"action": "type", "text": "hello"}) - assert ("type", "hello") in tool.calls - - -async def test_key_action_translates_chord() -> None: - tool = RecordingComputer() - await tool.execute({"action": "key", "text": "ctrl+c"}) - assert any(c[0] == "keys" and c[1] == ("Control_L", "c") for c in tool.calls) - - -async def test_mouse_move_and_down() -> None: - tool = RecordingComputer() - await tool.execute({"action": "mouse_move", "coordinate": [5, 6]}) - await tool.execute({"action": "left_mouse_down"}) - assert ("move", 5, 6) in tool.calls - assert ("down", "left") in tool.calls - - -async def test_screenshot_only() -> None: - tool = RecordingComputer() - await tool.execute({"action": "screenshot"}) - assert tool.calls == [("screenshot",)] - - -async def test_key_without_text_errors() -> None: - tool = RecordingComputer() - result = await tool.execute({"action": "key"}) - assert result.isError - - -async def test_unsupported_action_errors() -> None: - tool = RecordingComputer() - result = await tool.execute({"action": "frobnicate"}) - assert result.isError - assert "unsupported" in result_text(result).lower() - - -def test_crop_reports_encoded_mime_type() -> None: - source = BytesIO() - Image.effect_noise((128, 128), 100).convert("RGB").save(source, format="PNG") - - cropped, mime_type = _crop_png( - source.getvalue(), - (0, 0, 128, 128), - WebPScreenshotEncoding(quality=42), - ) - - assert mime_type == "image/webp" - assert cropped.startswith(b"RIFF") - - -async def test_zoom_reports_encoded_mime_type() -> None: - tool = RecordingComputer() - tool.client.screenshot_png = AsyncMock(return_value=(b"png", "image/png")) - - with patch( - "hud.agents.claude.tools.computer._crop_png", - return_value=(b"webp", "image/webp"), - ) as crop: - result = await tool._zoom({"region": [0, 0, 10, 10]}) - - tool.client.screenshot_png.assert_awaited_once_with("image/png") - crop.assert_called_once_with(b"png", (0, 0, 10, 10), tool.screenshot_encoding) - image = next(block for block in result.content if isinstance(block, mcp_types.ImageContent)) - assert image.mimeType == "image/webp" diff --git a/hud/agents/gemini/agent.py b/hud/agents/gemini/agent.py index 649ce27a8..6b6459dac 100644 --- a/hud/agents/gemini/agent.py +++ b/hud/agents/gemini/agent.py @@ -17,6 +17,8 @@ if TYPE_CHECKING: from google import genai + from hud.agents.tools.base import AgentTool + from .settings import gemini_agent_settings from .tools import ( @@ -86,7 +88,6 @@ def __init__(self, config: GeminiConfig | None = None) -> None: self.max_output_tokens = config.max_output_tokens self.thinking_level = config.thinking_level self.include_thoughts = config.include_thoughts - self.excluded_predefined_functions = list(config.excluded_predefined_functions) self.max_tool_result_chars = config.max_tool_result_chars self.max_recent_turn_with_screenshots = ( gemini_agent_settings.MAX_RECENT_TURN_WITH_SCREENSHOTS @@ -94,6 +95,12 @@ def __init__(self, config: GeminiConfig | None = None) -> None: # ─── ToolAgent hooks ────────────────────────────────────────────── + def _tool_options(self, tool_cls: type[AgentTool[Any]]) -> dict[str, Any]: + options = super()._tool_options(tool_cls) + if issubclass(tool_cls, GeminiComputerTool): + options["excluded_predefined_functions"] = self.config.excluded_predefined_functions + return options + async def _initialize_state( self, *, prompt: list[mcp_types.PromptMessage] ) -> RunState[genai_types.Content]: diff --git a/hud/agents/gemini/tools/computer.py b/hud/agents/gemini/tools/computer.py index 9532debc8..57fe33958 100644 --- a/hud/agents/gemini/tools/computer.py +++ b/hud/agents/gemini/tools/computer.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import platform from typing import TYPE_CHECKING, Any from google.genai import types as genai_types @@ -14,12 +13,18 @@ from .base import GeminiToolSpec if TYPE_CHECKING: + from collections.abc import Sequence + + from hud.agents.tools.base import AgentToolSpec + from hud.capabilities import RFBClient + from hud.capabilities.rfb import ScreenshotEncoding from hud.types import MCPToolResult logger = logging.getLogger(__name__) GEMINI_DRAG_INSET = 25 -IS_MAC = platform.system().lower() == "darwin" +#: Gemini places every coordinate on a 1000x1000 grid regardless of the screen size. +GEMINI_COORDINATE_SCALE = 1000 PREDEFINED_COMPUTER_USE_FUNCTIONS = ( "open_web_browser", @@ -44,13 +49,25 @@ class GeminiComputerTool(RFBTool): - """Translate Gemini predefined computer functions into RFBTool primitives.""" + """Translate Gemini predefined computer functions into RFBTool primitives. + + Browser chords (select all, back, forward, the address bar) are the Linux + and Windows ones: the keys land on the environment's display, which the + RFB capability does not describe, and environments run Linux. + """ name = "computer_use" - def __init__(self, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.excluded_predefined_functions: list[str] = [] + def __init__( + self, + *, + spec: AgentToolSpec, + client: RFBClient, + screenshot_encoding: ScreenshotEncoding, + excluded_predefined_functions: Sequence[str] = (), + ) -> None: + super().__init__(spec=spec, client=client, screenshot_encoding=screenshot_encoding) + self.excluded_predefined_functions = list(excluded_predefined_functions) @classmethod def default_spec(cls, model: str) -> GeminiToolSpec: @@ -80,25 +97,23 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: return await self.screenshot() if action == "click_at": - await self.click(args.get("x"), args.get("y")) + await self.click(*self._point(args)) return await self.screenshot() if action == "hover_at": - x, y = args.get("x"), args.get("y") + x, y = self._point(args) if x is not None and y is not None: - await self.move(int(x), int(y)) + await self.move(x, y) return await self.screenshot() if action == "type_text_at": - x, y = args.get("x"), args.get("y") + x, y = self._point(args) if x is not None and y is not None: - await self.move(int(x), int(y)) - await self.click(int(x), int(y)) + await self.move(x, y) + await self.click(x, y) if args.get("clear_before_typing", True): - select_all = ["Super_L", "a"] if IS_MAC else ["Control_L", "a"] - delete_key = "BackSpace" if IS_MAC else "Delete" - await self.press_keys(select_all) - await self.press_keys([delete_key]) + await self.press_keys(["Control_L", "a"]) + await self.press_keys(["Delete"]) text = args.get("text") if isinstance(text, str) and text: await self.type_text(text) @@ -118,14 +133,8 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: sx = magnitude elif direction == "left": sx = -magnitude - x = args.get("x") if action == "scroll_at" else None - y = args.get("y") if action == "scroll_at" else None - await self.scroll( - int(x) if x is not None else None, - int(y) if y is not None else None, - scroll_x=sx, - scroll_y=sy, - ) + x, y = self._point(args) if action == "scroll_at" else (None, None) + await self.scroll(x, y, scroll_x=sx, scroll_y=sy) return await self.screenshot() if action == "wait_5_seconds": @@ -133,26 +142,22 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: return await self.screenshot() if action == "go_back": - keys = ["Super_L", "bracketleft"] if IS_MAC else ["Alt_L", "Left"] - await self.press_keys(keys) + await self.press_keys(["Alt_L", "Left"]) return await self.screenshot() if action == "go_forward": - keys = ["Super_L", "bracketright"] if IS_MAC else ["Alt_L", "Right"] - await self.press_keys(keys) + await self.press_keys(["Alt_L", "Right"]) return await self.screenshot() if action == "search": target = args.get("url") or "https://www.google.com" - keys = ["Super_L", "l"] if IS_MAC else ["Control_L", "l"] - await self.press_keys(keys) + await self.press_keys(["Control_L", "l"]) await self.type_text(str(target)) await self.press_keys(["Return"]) return await self.screenshot() if action == "navigate": - keys = ["Super_L", "l"] if IS_MAC else ["Control_L", "l"] - await self.press_keys(keys) + await self.press_keys(["Control_L", "l"]) url = args.get("url") or "" await self.type_text(str(url)) await self.press_keys(["Return"]) @@ -167,7 +172,7 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: "ctrl": "Control_L", "cmd": "Super_L", "command": "Super_L", - "meta": "Super_L" if IS_MAC else "Control_L", + "meta": "Control_L", "alt": "Alt_L", "shift": "Shift_L", "return": "Return", @@ -180,21 +185,31 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: return await self.screenshot() if action == "drag_and_drop": - max_coord = max(self.display_width, self.display_height) - def clamp(v: Any) -> int: - if not isinstance(v, int | float): - return 0 - return min(max(int(v), GEMINI_DRAG_INSET), max_coord - GEMINI_DRAG_INSET) + def inset(value: Any, size: int) -> int: + pixel = self._pixel(value if isinstance(value, int | float) else 0, size) + return min(max(pixel, GEMINI_DRAG_INSET), size - GEMINI_DRAG_INSET) + width, height = self.display_width, self.display_height path = [ - (clamp(args.get("x")), clamp(args.get("y"))), - (clamp(args.get("destination_x")), clamp(args.get("destination_y"))), + (inset(args.get("x"), width), inset(args.get("y"), height)), + (inset(args.get("destination_x"), width), inset(args.get("destination_y"), height)), ] await self.drag(path) return await self.screenshot() return tool_err(f"Unknown Gemini computer action: {action}") + @staticmethod + def _pixel(value: float, size: int) -> int: + """Map a coordinate on Gemini's normalized 0-999 grid onto a ``size``-pixel axis.""" + return int(value / GEMINI_COORDINATE_SCALE * size) + + def _point(self, args: dict[str, Any]) -> tuple[int | None, int | None]: + x, y = args.get("x"), args.get("y") + if x is None or y is None: + return None, None + return self._pixel(x, self.display_width), self._pixel(y, self.display_height) + __all__ = ["GEMINI_COMPUTER_SPEC", "PREDEFINED_COMPUTER_USE_FUNCTIONS", "GeminiComputerTool"] diff --git a/hud/agents/gemini/tools/tests/test_computer.py b/hud/agents/gemini/tools/tests/test_computer.py deleted file mode 100644 index 305fe876b..000000000 --- a/hud/agents/gemini/tools/tests/test_computer.py +++ /dev/null @@ -1,124 +0,0 @@ -"""``GeminiComputerTool`` — predefined computer-use functions dispatched to RFB. - -No live VNC: a recording subclass captures the primitive calls. -""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, patch - -from hud.agents.gemini.tools.computer import GEMINI_COMPUTER_SPEC, GeminiComputerTool -from hud.agents.tools.base import tool_ok - -if TYPE_CHECKING: - from collections.abc import Iterable - - -class RecordingGemini(GeminiComputerTool): - client: Any - - def __init__(self) -> None: - self.calls: list[tuple[Any, ...]] = [] - self.client = SimpleNamespace(width=400, height=300) - self.excluded_predefined_functions = [] - - async def screenshot(self) -> Any: - self.calls.append(("screenshot",)) - return tool_ok("shot") - - async def click( - self, - x: int | None = None, - y: int | None = None, - *, - button: Any = "left", - hold_keys: Iterable[str] | None = None, - count: int = 1, - interval_ms: int = 0, - ) -> None: - self.calls.append(("click", x, y)) - - async def move(self, x: Any, y: Any) -> None: - self.calls.append(("move", x, y)) - - async def type_text(self, text: Any) -> None: - self.calls.append(("type", text)) - - async def press_keys(self, keys: Any, **kw: Any) -> None: - self.calls.append(("keys", tuple(keys))) - - async def scroll( - self, - x: int | None = None, - y: int | None = None, - *, - scroll_x: int = 0, - scroll_y: int = 0, - hold_keys: Iterable[str] | None = None, - ) -> None: - kw = {"scroll_x": scroll_x, "scroll_y": scroll_y, "hold_keys": hold_keys} - self.calls.append(("scroll", x, y, kw)) - - async def drag(self, path: Any, **kw: Any) -> None: - self.calls.append(("drag", tuple(path))) - - -def test_default_spec() -> None: - assert GeminiComputerTool.default_spec("any") is GEMINI_COMPUTER_SPEC - - -async def test_click_at() -> None: - tool = RecordingGemini() - await tool.execute({"action": "click_at", "x": 10, "y": 20}) - assert ("click", 10, 20) in tool.calls - assert tool.calls[-1] == ("screenshot",) - - -async def test_type_text_at_without_clear() -> None: - tool = RecordingGemini() - await tool.execute( - {"action": "type_text_at", "x": 5, "y": 6, "text": "hi", "clear_before_typing": False} - ) - assert ("move", 5, 6) in tool.calls - assert ("type", "hi") in tool.calls - - -async def test_scroll_at_down() -> None: - tool = RecordingGemini() - await tool.execute({"action": "scroll_at", "x": 0, "y": 0, "direction": "down", "magnitude": 3}) - scrolls = [c for c in tool.calls if c[0] == "scroll"] - assert scrolls and scrolls[0][3]["scroll_y"] == 3 - - -async def test_key_combination() -> None: - tool = RecordingGemini() - await tool.execute({"action": "key_combination", "keys": "ctrl+c"}) - assert ("keys", ("Control_L", "c")) in tool.calls - - -async def test_key_combination_requires_string() -> None: - tool = RecordingGemini() - assert (await tool.execute({"action": "key_combination", "keys": 123})).isError - - -async def test_wait_and_drag() -> None: - tool = RecordingGemini() - with patch("hud.agents.tools.rfb.asyncio.sleep", new_callable=AsyncMock) as sleep: - await tool.execute({"action": "wait_5_seconds"}) - await tool.execute( - {"action": "drag_and_drop", "x": 50, "y": 50, "destination_x": 100, "destination_y": 100} - ) - sleep.assert_awaited_once_with(5) - assert any(c[0] == "drag" for c in tool.calls) - - -async def test_missing_action_errors() -> None: - tool = RecordingGemini() - assert (await tool.execute({})).isError - - -async def test_unknown_action_errors() -> None: - tool = RecordingGemini() - assert (await tool.execute({"action": "fly_to_moon"})).isError diff --git a/hud/agents/misc/response_automation.py b/hud/agents/misc/response_automation.py index b28eaea17..5a420cf87 100644 --- a/hud/agents/misc/response_automation.py +++ b/hud/agents/misc/response_automation.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from functools import cache from typing import TYPE_CHECKING, Literal, cast import mcp.types as types @@ -60,13 +59,6 @@ async def auto_respond( ) -@cache -def _client() -> AsyncOpenAI: - from hud.utils.gateway import build_gateway_client - - return cast("AsyncOpenAI", build_gateway_client("openai")) - - @instrument(name="response_automation") async def _determine_response( agent_message: str, @@ -74,18 +66,24 @@ async def _determine_response( model: str = "gpt-5", system_prompt: str = DEFAULT_SYSTEM_PROMPT, ) -> ResponseType: - response = await _client().responses.create( - model=model, - instructions=system_prompt, - input=[ - { - "role": "user", - "content": f"Agent message: {agent_message}\n\nWhat is the appropriate response?", - }, - ], - reasoning={"effort": "low"}, - max_output_tokens=256, - ) + from hud.utils.gateway import build_gateway_client + + # A client per call: its connection pool belongs to the event loop that opened it. + async with cast("AsyncOpenAI", build_gateway_client("openai")) as client: + response = await client.responses.create( + model=model, + instructions=system_prompt, + input=[ + { + "role": "user", + "content": ( + f"Agent message: {agent_message}\n\nWhat is the appropriate response?" + ), + }, + ], + reasoning={"effort": "low"}, + max_output_tokens=256, + ) text_parts: list[str] = [] for item in response.output: diff --git a/hud/agents/openai/tools/tests/test_computer.py b/hud/agents/openai/tools/tests/test_computer.py deleted file mode 100644 index a42221ebf..000000000 --- a/hud/agents/openai/tools/tests/test_computer.py +++ /dev/null @@ -1,174 +0,0 @@ -"""``OpenAIComputerTool`` — key mapping + computer-call dispatch to RFB primitives. - -No live VNC: a recording subclass captures the primitive calls. -""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, patch - -import mcp.types as mcp_types -import pytest - -from hud.agents.openai.tools.computer import ( - OpenAIComputerTool, - _hold_keys, - _map_key, -) -from hud.agents.tools.base import result_text -from hud.types import MCPToolResult - -if TYPE_CHECKING: - from collections.abc import Iterable - - -class RecordingOpenAI(OpenAIComputerTool): - client: Any - - def __init__(self) -> None: - self.calls: list[tuple[Any, ...]] = [] - self.client = SimpleNamespace(width=200, height=100) - - async def screenshot(self) -> MCPToolResult: - self.calls.append(("screenshot",)) - return MCPToolResult( - content=[mcp_types.ImageContent(type="image", data="c2hvdA==", mimeType="image/png")], - ) - - async def click( - self, - x: int | None = None, - y: int | None = None, - *, - button: Any = "left", - hold_keys: Iterable[str] | None = None, - count: int = 1, - interval_ms: int = 0, - ) -> None: - kw = {"button": button, "hold_keys": hold_keys} - if count != 1: - kw["count"] = count - if interval_ms: - kw["interval_ms"] = interval_ms - self.calls.append(("click", x, y, kw)) - - async def move(self, x: Any, y: Any) -> None: - self.calls.append(("move", x, y)) - - async def type_text(self, text: Any) -> None: - self.calls.append(("type", text)) - - async def press_keys(self, keys: Any, **kw: Any) -> None: - self.calls.append(("keys", tuple(keys))) - - async def scroll( - self, - x: int | None = None, - y: int | None = None, - *, - scroll_x: int = 0, - scroll_y: int = 0, - hold_keys: Iterable[str] | None = None, - ) -> None: - kw = {"scroll_x": scroll_x, "scroll_y": scroll_y, "hold_keys": hold_keys} - self.calls.append(("scroll", x, y, kw)) - - async def drag(self, path: Any, **kw: Any) -> None: - self.calls.append(("drag", tuple(path))) - - -def test_key_mapping() -> None: - assert _map_key("ctrl") == "Control_L" - assert _map_key("x") == "x" - assert _map_key("ESC") == "Escape" - assert _map_key("RIGHT") == "Right" - assert _hold_keys(["ctrl", "c"]) == ["Control_L", "c"] - assert _hold_keys(None) is None - - -def test_to_params() -> None: - assert RecordingOpenAI().to_params() == {"type": "computer"} - - -async def test_click_returns_screenshot() -> None: - tool = RecordingOpenAI() - result = await tool.execute({"type": "click", "x": 1, "y": 2, "button": "left"}) - assert ("click", 1, 2, {"button": "left", "hold_keys": None}) in tool.calls - assert not result.isError - - -async def test_type_and_keypress() -> None: - tool = RecordingOpenAI() - await tool.execute({"type": "type", "text": "hi"}) - await tool.execute({"type": "keypress", "keys": ["ctrl", "c"]}) - assert ("type", "hi") in tool.calls - assert ("keys", ("Control_L", "c")) in tool.calls - - -@pytest.mark.parametrize(("ms", "seconds"), [(500, 0.5), (0, 0)]) -async def test_drag_and_wait(ms: int, seconds: float) -> None: - tool = RecordingOpenAI() - await tool.execute({"type": "drag", "path": [{"x": 0, "y": 0}, {"x": 5, "y": 5}]}) - with patch("hud.agents.tools.rfb.asyncio.sleep", new_callable=AsyncMock) as sleep: - result = await tool.execute({"type": "wait", "ms": ms}) - assert ("drag", ((0, 0), (5, 5))) in tool.calls - assert not result.isError - sleep.assert_awaited_once_with(seconds) - - -async def test_response_action_returns_text_and_screenshot() -> None: - tool = RecordingOpenAI() - result = await tool.execute({"type": "response", "text": "all done"}) - assert result_text(result) == "all done" - assert any(isinstance(block, mcp_types.ImageContent) for block in result.content) - assert tool.calls[-1] == ("screenshot",) - - -async def test_actions_list_runs_each() -> None: - tool = RecordingOpenAI() - await tool.execute( - {"actions": [{"type": "move", "x": 3, "y": 4}, {"type": "type", "text": "a"}]} - ) - assert ("move", 3, 4) in tool.calls - assert ("type", "a") in tool.calls - - -@pytest.mark.parametrize( - "arguments", [{"actions": []}, {"actions": "invalid", "type": "move", "x": 3, "y": 4}] -) -async def test_invalid_actions_errors(arguments: dict[str, Any]) -> None: - tool = RecordingOpenAI() - assert (await tool.execute(arguments)).isError - assert tool.calls == [("screenshot",)] - - -async def test_invalid_type_errors() -> None: - tool = RecordingOpenAI() - assert (await tool.execute({"type": "frobnicate"})).isError - assert (await tool.execute({})).isError - - -@pytest.mark.parametrize( - "action", - [ - {"type": "type"}, - {"type": "move", "x": 10}, - {"type": "keypress", "keys": "ESC"}, - {"type": "keypress", "keys": []}, - {"type": "drag", "path": [{"x": 1, "y": 2}, {"x": 3}]}, - {"type": "click", "button": 1}, - {"type": "wait", "ms": -1}, - ], -) -async def test_invalid_action_stops_batch_and_returns_screenshot(action: dict[str, Any]) -> None: - tool = RecordingOpenAI() - result = await tool.execute( - {"actions": [{"type": "move", "x": 3, "y": 4}, action, {"type": "type", "text": "later"}]}, - ) - - assert result.isError - assert result_text(result) - assert any(isinstance(block, mcp_types.ImageContent) for block in result.content) - assert tool.calls == [("move", 3, 4), ("screenshot",)] diff --git a/hud/agents/openai/tools/tests/test_strict_schema.py b/hud/agents/openai/tools/tests/test_strict_schema.py deleted file mode 100644 index f0ff27370..000000000 --- a/hud/agents/openai/tools/tests/test_strict_schema.py +++ /dev/null @@ -1,74 +0,0 @@ -"""``ensure_strict_json_schema`` — coerce a JSON schema to OpenAI strict-mode form.""" - -from __future__ import annotations - -from typing import Any - -from hud.agents.openai.tools.strict_schema import ensure_strict_json_schema - - -def test_empty_schema_becomes_closed_object() -> None: - result = ensure_strict_json_schema({}) - assert result == { - "additionalProperties": False, - "type": "object", - "properties": {}, - "required": [], - } - - -def test_object_gets_additional_properties_false_and_all_required() -> None: - schema: dict[str, Any] = { - "type": "object", - "properties": {"a": {"type": "string"}, "b": {"type": "integer"}}, - } - result = ensure_strict_json_schema(schema) - - assert result["additionalProperties"] is False - assert set(result["required"]) == {"a", "b"} # strict mode requires every property - - -def test_additional_properties_true_is_converted_to_false() -> None: - result = ensure_strict_json_schema( - {"type": "object", "properties": {}, "additionalProperties": True} - ) - assert result["additionalProperties"] is False - - -def test_unsupported_keywords_are_stripped() -> None: - schema: dict[str, Any] = { - "type": "object", - "properties": { - "name": { - "type": "string", - "title": "Name", # unsupported meta keyword - "minLength": 1, # unsupported string constraint - }, - }, - } - name_schema = ensure_strict_json_schema(schema)["properties"]["name"] - assert "title" not in name_schema - assert "minLength" not in name_schema - assert name_schema["type"] == "string" - - -def test_nested_objects_are_recursively_strict() -> None: - schema: dict[str, Any] = { - "type": "object", - "properties": { - "inner": {"type": "object", "properties": {"x": {"type": "number"}}}, - }, - } - inner = ensure_strict_json_schema(schema)["properties"]["inner"] - assert inner["additionalProperties"] is False - assert inner["required"] == ["x"] - - -def test_is_idempotent() -> None: - schema: dict[str, Any] = { - "type": "object", - "properties": {"a": {"type": "string", "title": "A"}}, - } - once = ensure_strict_json_schema(dict(schema)) - twice = ensure_strict_json_schema(once) - assert once == twice diff --git a/hud/agents/tests/cli_fakes.py b/hud/agents/tests/cli_fakes.py deleted file mode 100644 index a1cb5c246..000000000 --- a/hud/agents/tests/cli_fakes.py +++ /dev/null @@ -1,111 +0,0 @@ -from __future__ import annotations - -import asyncio -from types import SimpleNamespace -from typing import Any - - -class FakeReader: - def __init__(self, value: str, *, pause_after: int | None = None) -> None: - self._raw = value.encode() - self._lines = self._raw.splitlines(keepends=True) - self._pause_after = pause_after - self._index = 0 - self.blocked = asyncio.Event() - self.release = asyncio.Event() - - async def readline(self) -> bytes: - if self._pause_after == self._index: - self.blocked.set() - await self.release.wait() - self._pause_after = None - if self._index == len(self._lines): - return b"" - line = self._lines[self._index] - self._index += 1 - return line - - async def read(self) -> bytes: - return self._raw - - -class FakeWriter: - def __init__(self) -> None: - self.data = bytearray() - self.eof = False - - def write(self, data: bytes) -> None: - self.data.extend(data) - - async def drain(self) -> None: - pass - - def write_eof(self) -> None: - self.eof = True - - -class FakeProcess: - def __init__( - self, - stdout: str, - *, - stderr: str = "", - exit_status: int | None = 0, - returncode: int | None = None, - pause_after: int | None = None, - ) -> None: - self.stdin = FakeWriter() - self.stdout = FakeReader(stdout, pause_after=pause_after) - self.stderr = FakeReader(stderr) - self.exit_status = exit_status - self.returncode = exit_status if returncode is None else returncode - self.closed = False - self.terminated = False - - def terminate(self) -> None: - self.terminated = True - - def close(self) -> None: - self.closed = True - - async def wait_closed(self) -> None: - pass - - -class FakeClient: - """The slice of a run's client the CLI agents use: SSH, the manifest, routed bindings.""" - - def __init__( - self, - ssh: Any, - bindings: list[Any] | None = None, - routed: dict[str, Any] | None = None, - ) -> None: - self.ssh = ssh - self.manifest = SimpleNamespace(bindings=[ssh.capability, *(bindings or [])]) - self.routed = routed or {} - - async def open(self, ref: str) -> Any: - assert ref == "ssh" - return self.ssh - - def binding(self, ref: str) -> Any: - return self.routed[ref] - - -def fake_run( - client: FakeClient | None = None, - prompt_text: str = "", - connections: dict[str, Any] | None = None, -) -> Any: - trace = SimpleNamespace(status=None, content="", extra={}) - steps: list[Any] = [] - return SimpleNamespace( - trace=trace, - record=steps.append, - steps=steps, - client=client, - prompt_text=prompt_text, - runtime_config=None, - connections=connections or {}, - ) diff --git a/hud/agents/tests/test_apply_patch.py b/hud/agents/tests/test_apply_patch.py deleted file mode 100644 index c050f9a36..000000000 --- a/hud/agents/tests/test_apply_patch.py +++ /dev/null @@ -1,77 +0,0 @@ -"""The OpenAI V4A apply-patch engine: parse a patch + apply it via callbacks. - -``_text_to_patch`` parses the V4A diff text against the current files; ``_apply_patch`` -applies the parsed actions through ``write``/``remove`` callbacks. Both are pure, so -the file store is just a dict captured by the callbacks. -""" - -from __future__ import annotations - -from typing import Any - -import pytest - -from hud.agents.openai.tools.apply_patch import DiffError, _apply_patch, _text_to_patch - - -def _apply(patch_text: str, orig: dict[str, str]) -> tuple[dict[str, str | None], list[str]]: - actions, _fuzz = _text_to_patch(patch_text, orig) - writes: dict[str, str | None] = {} - removed: list[str] = [] - _apply_patch(actions, orig, lambda p, c: writes.__setitem__(p, c), removed.append) - return writes, removed - - -def test_add_file_writes_new_content() -> None: - patch = "*** Begin Patch\n*** Add File: hello.txt\n+hello\n+world\n*** End Patch" - writes, removed = _apply(patch, {}) - assert writes == {"hello.txt": "hello\nworld"} - assert removed == [] - - -def test_update_file_replaces_a_line_in_context() -> None: - orig = {"f.txt": "line1\nline2\nline3"} - patch = ( - "*** Begin Patch\n*** Update File: f.txt\n@@\n line1\n-line2\n+LINE2\n line3\n*** End Patch" - ) - writes, _ = _apply(patch, orig) - assert writes["f.txt"] == "line1\nLINE2\nline3" - - -def test_delete_file_calls_remove() -> None: - writes, removed = _apply( - "*** Begin Patch\n*** Delete File: f.txt\n*** End Patch", {"f.txt": "gone"} - ) - assert removed == ["f.txt"] - assert writes == {} - - -def test_update_with_move_renames_file() -> None: - orig = {"a.txt": "hi"} - patch = ( - "*** Begin Patch\n*** Update File: a.txt\n*** Move to: b.txt\n@@\n-hi\n+bye\n*** End Patch" - ) - writes, removed = _apply(patch, orig) - assert writes == {"b.txt": "bye"} - assert removed == ["a.txt"] - - -def test_invalid_patch_without_sentinels_raises() -> None: - with pytest.raises(DiffError): - _text_to_patch("just some text", {}) - - -def test_update_missing_file_raises() -> None: - patch = "*** Begin Patch\n*** Update File: ghost.txt\n@@\n-x\n+y\n*** End Patch" - with pytest.raises(DiffError, match="Missing File"): - _text_to_patch(patch, {}) - - -def test_duplicate_update_path_raises() -> None: - orig: dict[str, Any] = {"f.txt": "a"} - patch = ( - "*** Begin Patch\n*** Update File: f.txt\n@@\n-a\n+b\n" - "*** Update File: f.txt\n@@\n-b\n+c\n*** End Patch" - ) - with pytest.raises(DiffError, match="Duplicate Path"): - _text_to_patch(patch, orig) diff --git a/hud/agents/tests/test_base.py b/hud/agents/tests/test_base.py deleted file mode 100644 index b0ce8d580..000000000 --- a/hud/agents/tests/test_base.py +++ /dev/null @@ -1,242 +0,0 @@ -"""The agent base contract: the ``Agent`` ABC and gateway routing. - -These cover the model-agnostic surface that doesn't need provider SDKs or -network: the stateless ``Agent`` contract and ``AgentType`` / ``create_agent`` -resolution. -""" - -from __future__ import annotations - -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from hud.agents import ( - ClaudeCLIAgent, - CodexCLIAgent, - OpenAIAgent, - OpenAIChatAgent, - create_agent, -) -from hud.agents.base import Agent -from hud.agents.types import AgentConfig, OpenAIConfig -from hud.types import AgentType -from hud.utils.exceptions import HudAuthenticationError - - -class _FillingAgent(Agent): - async def __call__(self, run: Any) -> None: - run.trace.content = "done" - - -def test_agent_requires_call_implementation() -> None: - with pytest.raises(TypeError): - Agent() - - -async def test_agent_call_fills_trace() -> None: - from types import SimpleNamespace - - run = SimpleNamespace(trace=SimpleNamespace(content="")) - await _FillingAgent()(run) - assert run.trace.content == "done" - - -def test_agent_type_maps_value_to_class_and_provider() -> None: - assert AgentType("openai").cls is OpenAIAgent - assert AgentType("openai_compatible").cls is OpenAIChatAgent - assert isinstance(AgentType("openai").gateway_provider, str) - - -def test_agent_type_registers_cli_agent() -> None: - assert AgentType("claude_cli").cls is ClaudeCLIAgent - assert AgentType.of(ClaudeCLIAgent()) == AgentType.CLAUDE_CLI - assert AgentType("codex_cli").cls is CodexCLIAgent - assert AgentType.of(CodexCLIAgent()) == AgentType.CODEX_CLI - - -@pytest.mark.parametrize("agent", [ClaudeCLIAgent(), CodexCLIAgent()]) -def test_cli_agent_load_rebuilds_dumped_config(agent: ClaudeCLIAgent | CodexCLIAgent) -> None: - loaded = type(agent).load(agent.dump()) - - assert type(loaded) is type(agent) - assert loaded.config == agent.config - - -def test_custom_agent_round_trips_base_config() -> None: - agent = _FillingAgent(AgentConfig(model="custom", timeout_seconds=30)) - - loaded = _FillingAgent.load(agent.dump()) - - assert loaded.config == agent.config - - -def test_missing_provider_dependency_points_at_agents_extra( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Base installs (no [agents] extra) get an actionable error, not a raw import failure.""" - import sys - - import hud.agents - - class _Blocker: - def find_spec(self, fullname: str, path: Any = None, target: Any = None) -> None: - if fullname == "anthropic" or fullname.startswith("anthropic."): - raise ModuleNotFoundError(f"No module named {fullname!r}", name=fullname) - return None - - for module in list(sys.modules): - if module == "anthropic" or module.startswith(("anthropic.", "hud.agents.claude")): - monkeypatch.delitem(sys.modules, module) - monkeypatch.setattr(sys, "meta_path", [_Blocker(), *sys.meta_path]) - if "ClaudeAgent" in vars(hud.agents): # drop any cached lazy export - monkeypatch.delitem(hud.agents.__dict__, "ClaudeAgent") - - with pytest.raises(ImportError, match=r"hud\[agents\]"): - _ = hud.agents.ClaudeAgent - - with pytest.raises(ImportError, match=r"hud\[agents\]"): - _ = AgentType.CLAUDE.cls - - -@pytest.fixture(autouse=True) -def gateway_api_key(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("hud.agents.settings.api_key", "test-key") - - -def test_create_agent_unknown_model_raises(monkeypatch: pytest.MonkeyPatch) -> None: - # No gateway models available -> a bare unknown model can't be resolved. - monkeypatch.setattr("hud.utils.gateway.list_gateway_models", lambda *_: []) - with pytest.raises(ValueError, match="not found"): - create_agent("totally-unknown-model-xyz") - - -def test_create_agent_value_shortcut_leaves_client_out_of_config( - monkeypatch: pytest.MonkeyPatch, -) -> None: - sentinel = object() - - def _build_client(_provider: str) -> object: - return sentinel - - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", _build_client) - - agent = create_agent("openai") # AgentType.OPENAI shortcut - - assert isinstance(agent, OpenAIAgent) - assert agent.config.model == OpenAIConfig().model - assert agent.config.model_client is None - assert agent.openai_client is sentinel - assert agent.dump()["prompt_cache_key"] == agent.config.prompt_cache_key - - -def test_create_agent_uses_the_gateway_even_with_a_provider_key( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr("hud.utils.gateway.settings.openai_api_key", "provider-key") - direct = MagicMock(return_value=object()) - via_gateway = MagicMock(return_value=object()) - monkeypatch.setattr("hud.utils.gateway.AsyncOpenAI", direct) - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", via_gateway) - - agent = create_agent("openai") - - assert isinstance(agent, OpenAIAgent) - assert agent.openai_client is via_gateway.return_value - direct.assert_not_called() - # The same config built directly honours the provider key. - assert OpenAIAgent(OpenAIConfig()).openai_client is direct.return_value - # Routing is config, not a client, so it survives serialization. - reloaded = OpenAIAgent.load(agent.dump()) - assert reloaded.config.gateway is True - assert reloaded.openai_client is via_gateway.return_value - - -def test_create_agent_resolves_gateway_model_metadata( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.utils.gateway import GatewayModelInfo, GatewayProviderInfo - - model = GatewayModelInfo( - id="ft:custom-123", - model_name="gpt-5.5", - sdk_agent_type="openai_compatible", - provider=GatewayProviderInfo(name="openai"), - ) - monkeypatch.setattr("hud.utils.gateway.list_gateway_models", lambda *_: [model]) - - def _build_client(_provider: str) -> object: - return object() - - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", _build_client) - - agent = create_agent("ft:custom-123") - - assert isinstance(agent, OpenAIChatAgent) - assert agent.config.model == "gpt-5.5" # resolved to the model's real name - assert agent.config.model_client is None - - -@pytest.mark.parametrize( - ("alias", "canonical"), - [ - ("deepseek-v4-flash", "deepseek/deepseek-v4-flash"), # slug without provider prefix - ("glm-5.2", "z-ai/glm-5.2"), - ("kimi-k2.6", "moonshotai/kimi-k2.6"), - ("minimax-m3", "MiniMax-M3"), # case-insensitive - ], -) -def test_create_agent_accepts_gateway_model_aliases( - alias: str, - canonical: str, - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.utils.gateway import GatewayModelInfo, GatewayProviderInfo - - model = GatewayModelInfo( - id=canonical, - model_name=canonical, - sdk_agent_type="openai_compatible", - provider=GatewayProviderInfo(name="openai"), - ) - monkeypatch.setattr("hud.utils.gateway.list_gateway_models", lambda *_: [model]) - - def _build_client(_provider: str) -> object: - return object() - - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", _build_client) - - agent = create_agent(alias) - - assert isinstance(agent, OpenAIChatAgent) - assert agent.config.model == canonical - - -def test_create_agent_requires_hud_api_key(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("hud.agents.settings.api_key", None) - - with pytest.raises(HudAuthenticationError, match="HUD_API_KEY"): - create_agent("openai") - - -@pytest.mark.parametrize( - ("credential", "value"), - [ - ("api_key", "provider-key"), - ("api_key", None), - ("base_url", "https://provider.example"), - ("base_url", None), - ], -) -@pytest.mark.parametrize("hud_api_key", ["hud-key", None]) -def test_create_agent_rejects_direct_provider_credentials( - monkeypatch: pytest.MonkeyPatch, - credential: str, - value: str | None, - hud_api_key: str | None, -) -> None: - monkeypatch.setattr("hud.agents.settings.api_key", hud_api_key) - - with pytest.raises(ValueError, match="provider agent directly"): - create_agent("openai_compatible", **{credential: value}) diff --git a/hud/agents/tests/test_claude_agent.py b/hud/agents/tests/test_claude_agent.py deleted file mode 100644 index 38571b214..000000000 --- a/hud/agents/tests/test_claude_agent.py +++ /dev/null @@ -1,596 +0,0 @@ -"""``ClaudeAgent`` — ``get_response`` parsing over a fake streaming Messages client, -plus the pure ``_citation`` / ``_cache_last_user_block`` helpers. -""" - -from __future__ import annotations - -import base64 -import random -import struct -import zlib -from contextlib import AsyncExitStack -from io import BytesIO -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, cast -from unittest.mock import AsyncMock, Mock, call - -import fastmcp -import httpx -import httpx2 -import mcp.types as mcp_types -import pytest -from anthropic import APIStatusError, Omit -from fastmcp.tools import ToolResult -from mcp.types import ImageContent -from PIL import Image -from pydantic import ValidationError - -from hud.agents.claude.agent import ClaudeAgent -from hud.agents.claude.tools.computer import ClaudeComputerTool -from hud.agents.tool_agent import RunState -from hud.agents.types import ClaudeConfig, ToolStep -from hud.capabilities import Capability, MCPClient, RFBClient -from hud.settings import settings -from hud.types import MCPToolCall, MCPToolResult, Trace - -if TYPE_CHECKING: - from hud.eval.run import Run - - -class FakeStream: - def __init__(self, outcome: Any) -> None: - self._outcome = outcome - - async def __aenter__(self) -> FakeStream: - return self - - async def __aexit__(self, *_a: Any) -> bool: - return False - - def __aiter__(self) -> FakeStream: - return self - - async def __anext__(self) -> Any: - if isinstance(self._outcome, BaseException): - raise self._outcome - raise StopAsyncIteration - - async def get_final_message(self) -> Any: - return self._outcome - - -class FakeMessages: - def __init__(self, *outcomes: Any) -> None: - self._outcomes = list(outcomes) - self.calls = 0 - self.requests: list[dict[str, Any]] = [] - - def stream(self, **kwargs: Any) -> FakeStream: - self.requests.append(kwargs) - outcome = self._outcomes[self.calls] - self.calls += 1 - return FakeStream(outcome) - - -class FakeAnthropic: - def __init__(self, *outcomes: Any) -> None: - self.beta = SimpleNamespace(messages=FakeMessages(*outcomes)) - - -def _final(*content: Any, stop_reason: str, stop_details: Any = None) -> Any: - """A fake ``BetaMessage``: content blocks plus the always-present envelope.""" - return SimpleNamespace( - content=list(content), - stop_reason=stop_reason, - stop_details=stop_details, - model="claude-test-v9", - usage=SimpleNamespace(input_tokens=11, output_tokens=7, cache_read_input_tokens=3), - ) - - -def _agent(*outcomes: Any) -> ClaudeAgent: - from hud.agents.types import ClaudeConfig - - return ClaudeAgent( - ClaudeConfig(model="claude-test", max_tokens=1024, model_client=FakeAnthropic(*outcomes)) - ) - - -def _state(agent: ClaudeAgent) -> Any: - from hud.agents.tool_agent import RunState - - return RunState(messages=[agent._format_message("user", "go")]) - - -def _inline_error(type_: str, *, headers: dict[str, str] | None = None) -> APIStatusError: - body = {"type": "error", "error": {"type": type_, "message": "stream failed"}} - response = httpx.Response( - 200, - headers=headers, - request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), - ) - return APIStatusError(str(body), response=response, body=body) - - -def test_format_message_shape() -> None: - agent = _agent(SimpleNamespace(content=[], stop_reason="end_turn")) - msg = agent._format_message("assistant", "hi") - assert msg["role"] == "assistant" - - -async def test_get_response_text_and_tool_use() -> None: - final = _final( - SimpleNamespace(type="text", text="hello", citations=None), - SimpleNamespace(type="tool_use", id="t1", name="bash", input={"command": "ls"}), - stop_reason="tool_use", - ) - agent = _agent(final) - state = _state(agent) - - result = await agent.get_response(state) - - assert result.content == "hello" - assert [tc.name for tc in result.tool_calls] == ["bash"] - assert result.tool_calls[0].arguments == {"command": "ls"} - assert result.done is False - assert result.finish_reason == "tool_use" - # Model and usage are normalized off the provider response. - assert result.model == "claude-test-v9" - assert result.usage is not None - assert result.usage.prompt_tokens == 11 - assert result.usage.completion_tokens == 7 - assert result.usage.cached_tokens == 3 - - -async def test_get_response_done_on_text_only() -> None: - final = _final( - SimpleNamespace(type="text", text="done", citations=None), - stop_reason="end_turn", - ) - agent = _agent(final) - result = await agent.get_response(_state(agent)) - assert result.done is True - assert result.content == "done" - assert result.tool_calls == [] - assert result.refusal is None - - -async def test_get_response_surfaces_refusal_explanation() -> None: - explanation = ( - "This request triggered restrictions on violative cyber content and was " - "blocked under Anthropic's Usage Policy." - ) - final = _final( - stop_reason="refusal", - stop_details=SimpleNamespace( - type="refusal", - category="cyber", - explanation=explanation, - ), - ) - agent = _agent(final) - result = await agent.get_response(_state(agent)) - - assert result.finish_reason == "refusal" - assert result.refusal == explanation - assert result.done is True - assert result.tool_calls == [] - - -async def test_get_response_surfaces_refusal_category_when_explanation_missing() -> None: - final = _final( - stop_reason="refusal", - stop_details=SimpleNamespace( - type="refusal", - category="cyber", - explanation=None, - ), - ) - agent = _agent(final) - result = await agent.get_response(_state(agent)) - - assert result.finish_reason == "refusal" - assert result.refusal is not None - assert "cyber" in result.refusal - assert result.done is True - assert result.tool_calls == [] - - -async def test_get_response_collects_thinking() -> None: - final = _final( - SimpleNamespace(type="thinking", thinking="pondering"), - SimpleNamespace(type="text", text="answer", citations=None), - stop_reason="end_turn", - ) - agent = _agent(final) - result = await agent.get_response(_state(agent)) - assert result.reasoning == "pondering" - - -@pytest.mark.parametrize("error_type", ["upstream_error", "timeout_error"]) -async def test_get_response_retries_transient_inline_stream_error( - monkeypatch: pytest.MonkeyPatch, - error_type: str, -) -> None: - final = _final( - SimpleNamespace(type="text", text="recovered", citations=None), - stop_reason="end_turn", - ) - agent = _agent(_inline_error(error_type), _inline_error(error_type), final) - state = _state(agent) - sleep = AsyncMock() - jitter = Mock(side_effect=[0.25, 1.5]) - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", sleep) - monkeypatch.setattr("hud.agents.claude.agent._STREAM_JITTER.uniform", jitter) - - result = await agent.get_response(state) - - messages = cast("FakeMessages", cast("Any", agent.anthropic_client).beta.messages) - assert messages.calls == 3 - assert result.content == "recovered" - assert len(state.messages) == 2 - assert sleep.await_args_list == [call(0.25), call(1.5)] - assert jitter.call_args_list == [call(0.0, 1.0), call(0.0, 2.0)] - - -async def test_get_response_raises_after_transient_stream_retry_is_exhausted( - monkeypatch: pytest.MonkeyPatch, -) -> None: - agent = _agent( - _inline_error("gateway_timeout"), - _inline_error("gateway_timeout"), - _inline_error("gateway_timeout"), - ) - state = _state(agent) - sleep = AsyncMock() - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", sleep) - monkeypatch.setattr("hud.agents.claude.agent._STREAM_JITTER.uniform", Mock(return_value=0.0)) - - with pytest.raises(APIStatusError, match="gateway_timeout"): - await agent.get_response(state) - - messages = cast("FakeMessages", cast("Any", agent.anthropic_client).beta.messages) - assert messages.calls == 3 - assert len(state.messages) == 1 - assert sleep.await_count == 2 - - -@pytest.mark.parametrize( - ("headers", "expected_delay"), - [ - ({"retry-after": "1.75"}, 1.75), - ({"retry-after-ms": "1750"}, 1.75), - ({"retry-after": "60"}, 5.0), - ], - ids=["seconds", "milliseconds", "capped"], -) -async def test_get_response_honors_retry_after( - monkeypatch: pytest.MonkeyPatch, - headers: dict[str, str], - expected_delay: float, -) -> None: - final = _final( - SimpleNamespace(type="text", text="recovered", citations=None), - stop_reason="end_turn", - ) - agent = _agent(_inline_error("overloaded_error", headers=headers), final) - sleep = AsyncMock() - jitter = Mock() - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", sleep) - monkeypatch.setattr("hud.agents.claude.agent._STREAM_JITTER.uniform", jitter) - - result = await agent.get_response(_state(agent)) - - assert result.content == "recovered" - sleep.assert_awaited_once_with(expected_delay) - jitter.assert_not_called() - - -@pytest.mark.parametrize( - "error", - [ - httpx.ReadError( - "stream interrupted", - request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), - ), - httpx.ReadTimeout( - "stream interrupted", - request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), - ), - httpx2.ReadError( - "stream interrupted", - request=httpx2.Request("POST", "https://api.anthropic.com/v1/messages"), - ), - httpx2.ReadTimeout( - "stream interrupted", - request=httpx2.Request("POST", "https://api.anthropic.com/v1/messages"), - ), - ], - ids=["httpx-read-error", "httpx-read-timeout", "httpx2-read-error", "httpx2-read-timeout"], -) -async def test_get_response_retries_interrupted_stream( - monkeypatch: pytest.MonkeyPatch, - error: httpx.TransportError | httpx2.TransportError, -) -> None: - final = _final( - SimpleNamespace(type="text", text="recovered", citations=None), - stop_reason="end_turn", - ) - agent = _agent(error, final) - state = _state(agent) - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", AsyncMock()) - - result = await agent.get_response(state) - - messages = cast("FakeMessages", cast("Any", agent.anthropic_client).beta.messages) - assert messages.calls == 2 - assert result.content == "recovered" - assert len(state.messages) == 2 - - -async def test_get_response_does_not_retry_non_transient_inline_stream_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - agent = _agent(_inline_error("invalid_request_error")) - state = _state(agent) - sleep = AsyncMock() - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", sleep) - - with pytest.raises(APIStatusError, match="invalid_request_error"): - await agent.get_response(state) - - messages = cast("FakeMessages", cast("Any", agent.anthropic_client).beta.messages) - assert messages.calls == 1 - assert len(state.messages) == 1 - sleep.assert_not_awaited() - - -def test_citation_char_location() -> None: - raw = SimpleNamespace( - type="char_location", - cited_text="quote", - document_index=2, - document_title="doc", - start_char_index=0, - end_char_index=5, - ) - cit = ClaudeAgent._citation(cast("Any", raw)) - assert cit.type == "document_citation" - assert cit.source == "2" - assert cit.start_index == 0 - - -def test_cache_last_user_block_marks_content() -> None: - agent = _agent(SimpleNamespace(content=[], stop_reason="end_turn")) - messages = [agent._format_message("user", "hi")] - out = ClaudeAgent._cache_last_user_block(messages) - content = cast("list[Any]", out[-1]["content"]) - block = cast("dict[str, Any]", content[0]) - assert block.get("cache_control") == {"type": "ephemeral"} - - -def _image_result(data: bytes, *, computer: bool = False) -> Any: - agent = _agent() - state = RunState() - if computer: - spec = ClaudeComputerTool.default_spec("claude") - assert spec is not None - tool = ClaudeComputerTool(spec=spec, client=Mock(spec=RFBClient)) - state.tools[tool.provider_name] = tool - name = tool.provider_name - else: - name = "read_file" - content = ImageContent(type="image", mimeType="image/png", data=base64.b64encode(data).decode()) - result = MCPToolResult(content=[content]) - original = result.model_dump() - message = agent._format_result( - MCPToolCall(id="image-call", name=name, arguments={}), result, state - ) - assert result.model_dump() == original - assert isinstance(message, dict) - return next(iter(message["content"])) - - -@pytest.mark.parametrize("size", [(4000, 2000), (2000, 4000), (2000, 2000), (32, 24)]) -def test_tool_images_fit_claude_limits_without_upscaling(size: tuple[int, int]) -> None: - buffer = BytesIO() - Image.new("RGB", size, "white").save(buffer, format="PNG") - result = _image_result(buffer.getvalue()) - assert not result["is_error"] - source = result["content"][0]["source"] - assert len(source["data"]) <= 5_000_000 - with Image.open(BytesIO(base64.b64decode(source["data"]))) as image: - assert max(image.size) <= 1568 - assert image.width * image.height <= 1_150_000 - assert image.width <= size[0] and image.height <= size[1] - assert image.width / image.height == pytest.approx(size[0] / size[1], rel=0.005) - - -def test_claude_compresses_noisy_transparent_tool_images() -> None: - size = (1072, 1072) - image = Image.frombytes("RGBA", size, random.Random(0).randbytes(size[0] * size[1] * 4)) - buffer = BytesIO() - image.save(buffer, format="PNG") - assert len(base64.b64encode(buffer.getvalue())) > 5_000_000 - result = _image_result(buffer.getvalue()) - assert not result["is_error"] - source = result["content"][0]["source"] - assert source["media_type"] == "image/jpeg" - assert len(source["data"]) <= 5_000_000 - with Image.open(BytesIO(base64.b64decode(source["data"]))) as viewed: - assert viewed.size == size - assert viewed.mode == "RGB" - - -def test_claude_applies_image_orientation() -> None: - image = Image.new("RGB", (40, 20), "red") - exif = Image.Exif() - exif[274] = 6 - buffer = BytesIO() - image.save(buffer, format="JPEG", exif=exif) - source = _image_result(buffer.getvalue())["content"][0]["source"] - with Image.open(BytesIO(base64.b64decode(source["data"]))) as viewed: - assert viewed.size == (20, 40) - assert viewed.getexif().get(274, 1) == 1 - - -def test_claude_preserves_small_png_transparency() -> None: - buffer = BytesIO() - Image.new("RGBA", (32, 24), (255, 0, 0, 128)).save(buffer, format="PNG") - source = _image_result(buffer.getvalue())["content"][0]["source"] - assert source["media_type"] == "image/png" - with Image.open(BytesIO(base64.b64decode(source["data"]))) as viewed: - assert viewed.getpixel((0, 0)) == (255, 0, 0, 128) - - -@pytest.mark.filterwarnings("ignore:Image size.*:PIL.Image.DecompressionBombWarning") -@pytest.mark.parametrize("size", [(4001, 4000), (10000, 10000)]) -def test_claude_rejects_oversized_source_before_decoding(size: tuple[int, int]) -> None: - buffer = BytesIO() - Image.new("RGB", (1, 1)).save(buffer, format="PNG") - data = bytearray(buffer.getvalue()) - data[16:24] = struct.pack(">II", *size) - data[29:33] = struct.pack(">I", zlib.crc32(data[12:29])) - result = _image_result(bytes(data)) - assert result["is_error"] - assert result["content"][0]["type"] == "text" - assert "16,000,000-pixel source limit" in result["content"][0]["text"] - - -def test_claude_returns_invalid_image_as_tool_error() -> None: - result = _image_result(b"invalid image") - assert result["is_error"] - assert result["content"][0]["type"] == "text" - assert "Cannot view image" in result["content"][0]["text"] - - -def test_claude_preserves_computer_screenshot_coordinate_space() -> None: - buffer = BytesIO() - Image.new("RGB", (2000, 1000), "white").save(buffer, format="PNG") - data = buffer.getvalue() - result = _image_result(data, computer=True) - assert not result["is_error"] - source = result["content"][0]["source"] - assert source["media_type"] == "image/png" - assert base64.b64decode(source["data"]) == data - - -@pytest.mark.parametrize("limit", [None, 1]) -async def test_image_history_preserves_tool_results_and_thinking( - limit: int | None, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(settings, "telemetry_enabled", False) - buffer = BytesIO() - Image.new("RGB", (32, 24), "white").save(buffer, format="PNG") - image = ImageContent( - type="image", mimeType="image/png", data=base64.b64encode(buffer.getvalue()).decode() - ) - server = fastmcp.FastMCP("screenshots") - - @server.tool - def screenshot() -> ToolResult: - """Capture the current screen.""" - return ToolResult(content=[mcp_types.TextContent(type="text", text="screen"), image]) - - client = FakeAnthropic( - *( - _final( - SimpleNamespace( - type="thinking", thinking=f"reason-{turn}", signature=f"sig-{turn}" - ), - SimpleNamespace(type="tool_use", id=f"shot-{turn}", name="screenshot", input={}), - stop_reason="tool_use", - ) - for turn in (1, 2) - ), - _final(SimpleNamespace(type="text", text="done", citations=None), stop_reason="end_turn"), - ) - config = ClaudeConfig( - model="claude-test", - model_client=client, - max_steps=3, - max_tool_result_images=limit, - thinking={"type": "adaptive"} if limit is not None else None, - ) - restored = ClaudeConfig.model_validate(config.model_dump(mode="json")) - assert restored.max_tool_result_images == limit - assert restored.thinking == config.thinking - async with fastmcp.Client(server) as mcp_client: - cap = Capability.mcp(name="screenshots", url="https://tools.invalid/mcp") - connection = MCPClient(cap, mcp_client, AsyncExitStack()) - trace = Trace() - run = cast( - "Run", - SimpleNamespace( - trace=trace, - record=trace.record, - client=SimpleNamespace( - manifest=SimpleNamespace(bindings=[cap]), - open=AsyncMock(return_value=connection), - ), - prompt_messages=[ - mcp_types.PromptMessage( - role="user", content=mcp_types.TextContent(type="text", text="go") - ) - ], - ), - ) - await ClaudeAgent(config)(run) - - assert trace.status == "completed" - requests = client.beta.messages.requests - assert len(requests) == 3 - for request in requests: - if limit is None: - assert isinstance(request.get("thinking", Omit()), Omit) - assert isinstance(request.get("betas", Omit()), Omit) - else: - assert "thinking-binding-controls-2026-08-01" in request["betas"] - assert request["thinking"] == { - "type": "adaptive", - "block_binding": {"prefix_mismatch_behavior": "drop_block"}, - } - messages = requests[-1]["messages"] - results = [message["content"][0] for message in messages[2::2]] - assert [result["tool_use_id"] for result in results] == ["shot-1", "shot-2"] - assert [result["content"][0]["text"] for result in results] == ["screen", "screen"] - assert [result["content"][1]["type"] for result in results] == ( - ["image", "image"] if limit is None else ["text", "image"] - ) - assert [ - (message["content"][0].thinking, message["content"][0].signature) - for message in messages[1::2] - ] == [("reason-1", "sig-1"), ("reason-2", "sig-2")] - for step in trace.steps: - if isinstance(step, ToolStep): - assert step.result is not None - recorded_image = step.result.content[1] - assert isinstance(recorded_image, ImageContent) - assert recorded_image.data == image.data - - -@pytest.mark.parametrize("thinking", [None, {"type": "disabled"}]) -def test_image_history_requires_enabled_thinking(thinking: dict[str, str] | None) -> None: - with pytest.raises(ValidationError, match="requires explicit adaptive or enabled thinking"): - ClaudeConfig.model_validate({"max_tool_result_images": 1, "thinking": thinking}) - - -@pytest.mark.parametrize( - ("max_tokens", "budget_tokens"), - [(16384, 16384), (16384, 16385), (2048, 2048), (2048, 2049)], -) -def test_enabled_thinking_budget_must_be_below_max_tokens( - max_tokens: int, budget_tokens: int -) -> None: - with pytest.raises(ValidationError, match="budget_tokens must be less than max_tokens"): - ClaudeConfig( - max_tokens=max_tokens, thinking={"type": "enabled", "budget_tokens": budget_tokens} - ) - - -@pytest.mark.parametrize("max_tokens", [1025, 2048]) -def test_enabled_thinking_budget_below_max_tokens_is_allowed(max_tokens: int) -> None: - config = ClaudeConfig( - max_tokens=max_tokens, thinking={"type": "enabled", "budget_tokens": max_tokens - 1} - ) - assert config.thinking == {"type": "enabled", "budget_tokens": max_tokens - 1} diff --git a/hud/agents/tests/test_claude_cli_agent.py b/hud/agents/tests/test_claude_cli_agent.py deleted file mode 100644 index a3cb5ac37..000000000 --- a/hud/agents/tests/test_claude_cli_agent.py +++ /dev/null @@ -1,920 +0,0 @@ -"""ClaudeCLIAgent remote-command construction over the workspace SSH. - -The agent runs the ``claude`` CLI on the remote workspace. These cover how the -command is assembled per login shell — especially the Windows path, where the -command must ride a batch file invoked via ``cmd /c``. Bare ``.hud_run.bat`` is -rejected by the remote shell (and silently fails under PowerShell), so the -``cmd /c`` prefix is a regression guard for local Windows setups. -""" - -from __future__ import annotations - -import asyncio -import base64 -import json -import re -import sys -from contextlib import asynccontextmanager -from types import SimpleNamespace -from typing import Any, Literal, cast -from unittest.mock import AsyncMock, Mock - -import fastmcp -import pytest -from mcp.types import BlobResourceContents, EmbeddedResource, ImageContent, TextContent - -from hud.agents import create_agent -from hud.agents.claude.sdk import computer_mcp -from hud.agents.claude.sdk.agent import ClaudeCLIAgent -from hud.agents.claude.sdk.events import ClaudeEvents -from hud.agents.tests.cli_fakes import FakeClient as _FakeClient -from hud.agents.tests.cli_fakes import FakeProcess as _FakeStreamProcess -from hud.agents.tests.cli_fakes import fake_run as _fake_run -from hud.agents.types import AgentStep, ClaudeCLIConfig, ToolStep -from hud.capabilities import Capability, Connection, SSHClient -from hud.capabilities.rfb import WebPScreenshotEncoding -from hud.settings import settings -from hud.telemetry.context import set_trace_context -from hud.types import MCPToolResult - - -@pytest.fixture(autouse=True) -def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", None) - monkeypatch.setattr(settings, "anthropic_api_key", None) - monkeypatch.setattr( - "hud.agents.claude.sdk.agent.resolve_executable", - AsyncMock(return_value="claude"), - ) - - -async def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - - gateway = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig(gateway=True))) - provider = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig(gateway=False))) - - assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in gateway - assert "ANTHROPIC_API_KEY=hud-key" in gateway - assert "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1" in gateway - assert "DISABLE_AUTO_COMPACT" not in gateway - assert "ANTHROPIC_API_KEY=anthropic-key" in provider - assert "ANTHROPIC_BASE_URL" not in provider - assert "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS" not in provider - assert "DISABLE_AUTO_COMPACT" not in provider - assert "ANTHROPIC_MODEL=claude-sonnet-5" in provider - - -async def test_default_routing_uses_gateway_only_without_a_provider_key( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - - without_provider_key = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig())) - monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - with_provider_key = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig())) - - assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in without_provider_key - assert "ANTHROPIC_API_KEY=anthropic-key" in with_provider_key - assert "ANTHROPIC_BASE_URL" not in with_provider_key - - -async def test_create_agent_routes_cli_agent_through_gateway( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - - agent = create_agent("claude_cli") - - assert isinstance(agent, ClaudeCLIAgent) - assert agent.config.model == ClaudeCLIConfig().model - command = await _sent_command(agent) - assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in command - assert "ANTHROPIC_API_KEY=hud-key" in command - - -async def test_command_uses_process_bound_connection_without_its_credential( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - - with set_trace_context("trace-123"): - command = await _sent_command(ClaudeCLIAgent(), connection=connection) - - assert f"ANTHROPIC_BASE_URL={connection.client_url}" in command - assert "ANTHROPIC_AUTH_TOKEN=hud-process-bound" in command - assert "ANTHROPIC_API_KEY" not in command - assert "scoped-runtime-token" not in command - assert "hud-key" not in command - assert "Trace-Id" not in command - assert "exec env" in command - for name in ( - "ANTHROPIC_MODEL", - "ANTHROPIC_SMALL_FAST_MODEL", - "ANTHROPIC_DEFAULT_SONNET_MODEL", - "ANTHROPIC_DEFAULT_OPUS_MODEL", - "ANTHROPIC_DEFAULT_HAIKU_MODEL", - "CLAUDE_CODE_SUBAGENT_MODEL", - ): - assert f"{name}=claude-sonnet-5" in command - - -async def test_windows_command_encodes_environment_and_arguments( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud&key's") - config = ClaudeCLIConfig( - gateway=True, - max_steps=3, - system_prompt="don't $expand", - ) - command = await _sent_command(ClaudeCLIAgent(config), "powershell") - - encoded = command.rsplit(" ", 1)[1] - script = base64.b64decode(encoded).decode("utf-16-le") - assert "$env:ANTHROPIC_API_KEY='hud&key''s'" in script - assert "$env:CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS='1'" in script - assert "'--system-prompt' 'don''t $expand'" in script - assert "Get-Content -Raw -Encoding UTF8 '.hud_input.jsonl' | & 'claude'" in script - assert "'--input-format=stream-json'" in script - assert "python" not in script - - -class _FakeCompletedProcess: - async def wait(self, *, check: bool, **kwargs: Any) -> Any: - del check - assert kwargs == {"timeout": None} - return SimpleNamespace(stdout=b"", stderr=b"", exit_status=0, returncode=0) - - def terminate(self) -> None: - pass - - def close(self) -> None: - pass - - async def wait_closed(self) -> None: - pass - - -class _FakeConn: - def __init__(self, sink: dict[str, bytes], process: _FakeStreamProcess) -> None: - self._sink = sink - self._process = process - self.ran: list[str] = [] - self.write_commands: list[str] = [] - self.written: dict[str, bytes] = {} - self.deleted: list[str] = [] - - def is_closed(self) -> bool: - return False - - async def create_process(self, cmd: str, **kwargs: Any) -> Any: - input_value = kwargs.get("input") - if cmd.startswith(("rm -f -- ", "cmd /c del /f /q ")): - paths = [path for path in self._sink if path in cmd] - for path in paths: - self._sink.pop(path) - self.deleted.extend(paths) - return _FakeCompletedProcess() - if input_value is not None or cmd.startswith("powershell "): - self.write_commands.append(cmd) - script = cmd - if match := re.search(r"-EncodedCommand (\S+)", cmd): - script = base64.b64decode(match.group(1)).decode("utf-16-le") - name = next( - path - for path in (".hud_input.jsonl", ".hud_run.bat", ".hud_mcp_config.json") - if path in script - ) - if input_value is not None: - self._sink[name] = str(input_value).encode() - elif match := re.search(r"FromBase64String\('([^']+)'\)", script): - self._sink[name] += base64.b64decode(match.group(1)) - else: - self._sink[name] = b"" - self.written[name] = self._sink[name] - return _FakeCompletedProcess() - assert kwargs["encoding"] is None - self.env = kwargs.get("env") - self.ran.append(cmd) - return self._process - - -async def run_claude( - config: ClaudeCLIConfig, - run: Any, - *, - ssh: SSHClient, - prompt: str, - bindings: list[Capability] | None = None, - routed: dict[str, Capability] | None = None, -) -> None: - run.client = _FakeClient(ssh, bindings, routed) - run.prompt_text = prompt - await ClaudeCLIAgent(config)(run) - - -_STREAM_JSON = ( - '{"type":"assistant","message":{"id":"msg-1","type":"message",' - '"role":"assistant","model":"claude-test","content":[{"type":"text",' - '"text":"editing"},{"type":"tool_use","id":"tool-1","name":"Write","input":{}}],' - '"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":11,' - '"output_tokens":7,"cache_read_input_tokens":3}}}\n' - '{"type":"user","message":{"content":[{"type":"tool_result",' - '"tool_use_id":"tool-1","content":[{"type":"text","text":"wrote a.txt"},' - '{"type":"image","source":{"type":"base64","media_type":"image/png",' - '"data":"aW1hZ2U="}}],"is_error":false}]}}\n' - '{"type":"assistant","message":{"id":"msg-2","type":"message",' - '"role":"assistant","model":"claude-test","content":[{"type":"text",' - '"text":"done"}],"stop_reason":"end_turn","stop_sequence":null,' - '"usage":{"input_tokens":11,"output_tokens":7,"cache_read_input_tokens":3}}}\n' - '{"type":"result","is_error":false,"result":"done","session_id":"s",' - '"duration_ms":5,"num_turns":2,"total_cost_usd":0.01}\n' -) - - -def _ssh_with_conn(shell: str, conn: _FakeConn) -> SSHClient: - capability = Capability( - name="shell", - protocol="ssh/2", - url="ssh://localhost:22", - params={"shell": shell, "process_connections": True}, - ) - return SSHClient(capability, cast("Any", conn)) - - -async def _sent_command( - agent: ClaudeCLIAgent, shell: str = "bash", connection: Connection | None = None -) -> str: - """The CLI command the agent sends over SSH (unwrapped from the batch file on Windows).""" - conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - connections = {"inference": connection} if connection is not None else None - await agent(_fake_run(_FakeClient(_ssh_with_conn(shell, conn)), "x", connections)) - if shell in ("cmd", "powershell"): - return conn.written[".hud_run.bat"].decode().split("\r\n")[1] - (command,) = conn.ran - return command - - -async def test_command_passes_the_configured_reasoning_effort() -> None: - command = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig(reasoning_effort="max"))) - assert "--effort max" in command - - -async def test_command_omits_effort_when_unconfigured() -> None: - command = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig())) - assert "--effort" not in command - - -async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) - ssh = _ssh_with_conn("cmd", conn) - - run = _fake_run() - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="build it") - - assert conn.ran == ["cmd /c .hud_run.bat"] - assert all(command.startswith("powershell ") for command in conn.write_commands) - assert conn.written[".hud_run.bat"].startswith(b"@echo off\r\n") - assert conn.written[".hud_input.jsonl"].endswith(b"\n") - assert json.loads(conn.written[".hud_input.jsonl"]) == { - "type": "user", - "message": { - "role": "user", - "content": [{"type": "text", "text": "build it"}], - }, - } - assert sink == {} - assert set(conn.deleted) == {".hud_input.jsonl", ".hud_run.bat"} - assert run.trace.status is None - assert run.trace.content == "done" - assert "messages" not in run.trace.extra - - -async def test_exec_on_bash_runs_inline_without_batch() -> None: - sink: dict[str, bytes] = {} - process = _FakeStreamProcess(_STREAM_JSON) - conn = _FakeConn(sink, process) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="build it") - - assert sink == {} - assert conn.write_commands == [] - assert conn.deleted == [] - assert len(conn.ran) == 1 - assert "claude" in conn.ran[0] - assert "--input-format=stream-json" in conn.ran[0] - assert "build it" not in conn.ran[0] - assert process.stdin.data.endswith(b"\n") - assert json.loads(process.stdin.data) == { - "type": "user", - "message": { - "role": "user", - "content": [{"type": "text", "text": "build it"}], - }, - } - assert process.stdin.eof is True - assert run.trace.status is None - assert run.trace.content == "done" - assert "messages" not in run.trace.extra - - -async def test_exec_removes_mcp_config_after_run() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) - await run_claude( - ClaudeCLIConfig(), - _fake_run(), - ssh=_ssh_with_conn("bash", conn), - bindings=[ - Capability.mcp(name="database", url="http://db/mcp", transport="streamable-http") - ], - prompt="build it", - ) - - config = json.loads(conn.written[".hud_mcp_config.json"]) - assert config == {"mcpServers": {"database": {"type": "http", "url": "http://db:80/mcp"}}} - assert sink == {} - assert conn.deleted == [".hud_mcp_config.json"] - assert "--mcp-config .hud_mcp_config.json" in conn.ran[0] - - -async def test_exec_records_steps_before_process_exit() -> None: - process = _FakeStreamProcess(_STREAM_JSON, pause_after=1) - conn = _FakeConn({}, process) - ssh = _ssh_with_conn("bash", conn) - run = _fake_run() - - execution = asyncio.create_task( - run_claude( - ClaudeCLIConfig(), - run, - ssh=ssh, - prompt="edit it", - ) - ) - await process.stdout.blocked.wait() - - assert not execution.done() - assert len(run.steps) == 1 - first = run.steps[0] - assert isinstance(first, AgentStep) - assert first.content == "editing" - assert first.tool_calls[0].id == "tool-1" - - process.stdout.release.set() - await execution - - assert [type(step) for step in run.steps] == [AgentStep, ToolStep, AgentStep] - tool = cast("ToolStep", run.steps[1]) - assert tool.started_at == first.ended_at - assert tool.result is not None - text = tool.result.content[0] - assert isinstance(text, TextContent) - assert text.text == "wrote a.txt" - image = tool.result.content[1] - assert isinstance(image, ImageContent) - assert image.mimeType == "image/png" - assert image.data == "aW1hZ2U=" - final = cast("AgentStep", run.steps[2]) - assert final.started_at == tool.ended_at - assert run.trace.status is None - assert run.trace.content == "done" - - -async def test_exec_forwards_trace_id_only_to_hud_gateway( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - - gateway_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - gateway = ClaudeCLIConfig(gateway=True) - with set_trace_context("trace-123"): - await run_claude( - gateway, - _fake_run(), - ssh=_ssh_with_conn("bash", gateway_conn), - prompt="build it", - ) - assert "ANTHROPIC_CUSTOM_HEADERS='Trace-Id: trace-123'" in gateway_conn.ran[0] - - provider_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - provider = ClaudeCLIConfig(gateway=False) - with set_trace_context("trace-123"): - await run_claude( - provider, - _fake_run(), - ssh=_ssh_with_conn("bash", provider_conn), - prompt="build it", - ) - assert "ANTHROPIC_CUSTOM_HEADERS" not in provider_conn.ran[0] - - -async def test_exec_closes_streaming_process_when_cancelled() -> None: - process = _FakeStreamProcess(_STREAM_JSON, pause_after=0) - conn = _FakeConn({}, process) - execution = asyncio.create_task( - run_claude( - ClaudeCLIConfig(), - _fake_run(), - ssh=_ssh_with_conn("bash", conn), - prompt="build it", - ) - ) - await process.stdout.blocked.wait() - execution.cancel() - - with pytest.raises(asyncio.CancelledError): - await execution - - assert process.closed - - -async def test_exec_nonzero_exit_with_no_stdout_raises() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn(sink, _FakeStreamProcess("", stderr="boom", exit_status=1)) - ssh = _ssh_with_conn("cmd", conn) - - run = _fake_run() - with pytest.raises(RuntimeError, match="boom"): - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.extra["returncode"] == 1 - - -async def test_exec_signal_exit_records_the_returncode() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn( - sink, - _FakeStreamProcess("", exit_status=None, returncode=-15), - ) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - with pytest.raises(RuntimeError, match="return code -15"): - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.extra["returncode"] == -15 - - -async def test_exec_nonzero_exit_with_result_stream_remains_an_error() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn( - sink, - _FakeStreamProcess(_STREAM_JSON, stderr="transport failed", exit_status=1), - ) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - with pytest.raises(RuntimeError, match="transport failed"): - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.content == "done" - assert run.trace.extra["returncode"] == 1 - assert run.trace.extra["stderr"] == "transport failed" - assert "messages" not in run.trace.extra - - -async def test_exec_nonzero_exit_with_result_stream_and_no_stderr_is_graded() -> None: - """The CLI exits nonzero after a successful result; the run stays gradeable.""" - sink: dict[str, bytes] = {} - conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON, exit_status=1)) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.content == "done" - assert run.trace.extra["returncode"] == 1 - assert "stderr" not in run.trace.extra - - -async def test_exec_zero_exit_without_result_event_is_an_error() -> None: - sink: dict[str, bytes] = {} - stdout = _STREAM_JSON.rsplit('{"type":"result"', 1)[0] - conn = _FakeConn(sink, _FakeStreamProcess(stdout)) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - with pytest.raises(RuntimeError, match="without a result event"): - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.content == "done" - - -@pytest.mark.parametrize( - ("transport", "claude_type"), - [("streamable-http", "http"), ("sse", "sse")], -) -async def test_manifest_mcp_capability_is_written_for_remote_claude( - transport: Literal["streamable-http", "sse"], - claude_type: str, -) -> None: - mcp = Capability.mcp(name="database", url="http://database:8000/mcp", transport=transport) - conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - - await run_claude( - ClaudeCLIConfig(), - _fake_run(), - ssh=_ssh_with_conn("bash", conn), - prompt="call the tool", - bindings=[mcp], - ) - - assert json.loads(conn.written[".hud_mcp_config.json"]) == { - "mcpServers": {"database": {"type": claude_type, "url": "http://database:8000/mcp"}} - } - - -async def test_remote_claude_bridges_computer_mcp_for_the_cli_run( - monkeypatch: pytest.MonkeyPatch, -) -> None: - screen = Capability.rfb(name="screen", url="rfb://localhost:5900", display=0) - routed = Capability.rfb(name="screen", url="rfb://127.0.0.1:41000", display=0) - encoding = WebPScreenshotEncoding(quality=42) - conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - ssh = _ssh_with_conn("bash", conn) - runs_while_bridged: list[int] = [] - - @asynccontextmanager - async def bridge( - bridge_ssh: SSHClient, - capability: Capability, - screenshot_encoding: WebPScreenshotEncoding, - *, - shell: str, - ) -> Any: - assert bridge_ssh is ssh - assert capability == routed - assert screenshot_encoding == encoding - assert shell == "bash" - yield {"type": "stdio", "command": "sh", "args": ["-c", "relay"]} - runs_while_bridged.append(len(conn.ran)) - - monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - - await run_claude( - ClaudeCLIConfig(screenshot_encoding=encoding), - _fake_run(), - ssh=ssh, - prompt="use the computer", - bindings=[screen], - routed={"screen": routed}, - ) - - assert runs_while_bridged == [1] - assert json.loads(conn.written[".hud_mcp_config.json"]) == { - "mcpServers": {"computer-use": {"type": "stdio", "command": "sh", "args": ["-c", "relay"]}} - } - - -async def test_remote_claude_preserves_multiple_rfb_bindings( - monkeypatch: pytest.MonkeyPatch, -) -> None: - screens = [ - Capability.rfb(name="screen-0", url="rfb://display-0:5900", display=0), - Capability.rfb(name="screen-1", url="rfb://display-1:5901", display=1), - ] - routed = { - cap.name: Capability.rfb(name=cap.name, url=f"rfb://127.0.0.1:{41000 + i}", display=i) - for i, cap in enumerate(screens) - } - conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - - @asynccontextmanager - async def bridge( - _ssh: SSHClient, - capability: Capability, - _encoding: WebPScreenshotEncoding, - *, - shell: str, - ) -> Any: - yield {"type": "stdio", "command": "sh", "args": ["-c", capability.name]} - - monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - - await run_claude( - ClaudeCLIConfig(), - _fake_run(), - ssh=_ssh_with_conn("bash", conn), - prompt="use both screens", - bindings=screens, - routed=routed, - ) - - assert json.loads(conn.written[".hud_mcp_config.json"]) == { - "mcpServers": { - f"computer-use-{name}": {"type": "stdio", "command": "sh", "args": ["-c", name]} - for name in ("screen-0", "screen-1") - } - } - - -async def test_computer_mcp_stdio_owns_rfb_lifetime( - monkeypatch: pytest.MonkeyPatch, -) -> None: - screen = Capability.rfb(name="screen", url="rfb://localhost:5900", display=0) - encoding = WebPScreenshotEncoding(quality=42) - rfb = SimpleNamespace(close=AsyncMock()) - connect = AsyncMock(return_value=rfb) - server = SimpleNamespace(run_async=AsyncMock()) - create = Mock(return_value=server) - monkeypatch.setattr(computer_mcp.RFBClient, "connect", connect) - monkeypatch.setattr(computer_mcp, "create_computer_mcp", create) - - await computer_mcp.run_computer_mcp( - { - computer_mcp.RFB_CAPABILITY_ENV: json.dumps(screen.to_manifest()), - computer_mcp.SCREENSHOT_ENCODING_ENV: encoding.model_dump_json(), - } - ) - - connect.assert_awaited_once_with(screen) - create.assert_called_once_with(rfb, encoding) - server.run_async.assert_awaited_once_with(transport="stdio", show_banner=False) - rfb.close.assert_awaited_once() - - -async def test_computer_mcp_preserves_tool_result(monkeypatch: pytest.MonkeyPatch) -> None: - result = MCPToolResult( - content=[TextContent(type="text", text="failed")], - isError=True, - ) - execute = AsyncMock(return_value=result) - monkeypatch.setattr(computer_mcp.ClaudeComputerTool, "execute", execute) - server = computer_mcp.create_computer_mcp(cast("Any", object()), WebPScreenshotEncoding()) - - async with fastmcp.Client(server) as client: - received = await client.call_tool_mcp( - "computer", - {"action": "left_click", "coordinate": [10, 20]}, - ) - - execute.assert_awaited_once_with({"action": "left_click", "coordinate": [10, 20]}) - assert received.isError is True - assert received.content == result.content - - -class _ByteWriter: - def __init__(self) -> None: - self.closed = False - self.data = bytearray() - - def write(self, data: bytes) -> None: - self.data.extend(data) - - async def drain(self) -> None: - pass - - def close(self) -> None: - self.closed = True - - -class _LocalComputerProcess: - def __init__(self) -> None: - self.stdin = _ByteWriter() - self.stdout = asyncio.StreamReader() - self.stderr = asyncio.StreamReader() - self.returncode: int | None = None - self.terminated = False - self.killed = False - - def terminate(self) -> None: - self.terminated = True - self.returncode = -15 - - def kill(self) -> None: - self.killed = True - self.returncode = -9 - - async def wait(self) -> int: - assert self.returncode is not None - return self.returncode - - -async def test_computer_mcp_bridge_uses_controller_python_and_owns_resources( - monkeypatch: pytest.MonkeyPatch, -) -> None: - bridge_stdout = asyncio.StreamReader() - bridge_stderr = asyncio.StreamReader() - bridge_stderr.feed_data(b"ready\n") - bridge_stdin = _ByteWriter() - bridge = SimpleNamespace( - stdin=bridge_stdin, - stdout=bridge_stdout, - stderr=bridge_stderr, - channel=SimpleNamespace(close=Mock()), - wait_closed=AsyncMock(), - ) - connection = SimpleNamespace(create_process=AsyncMock(return_value=bridge)) - ssh = SimpleNamespace(create_process=connection.create_process) - local = _LocalComputerProcess() - spawn = AsyncMock(return_value=local) - monkeypatch.setattr(computer_mcp.asyncio, "create_subprocess_exec", spawn) - monkeypatch.setattr(computer_mcp.secrets, "token_hex", lambda _length: "bridge-token") - screen = Capability.rfb(name="screen", url="rfb://127.0.0.1:41000", display=0) - encoding = WebPScreenshotEncoding(quality=42) - - async with computer_mcp.bridge_computer_mcp( - cast("Any", ssh), - screen, - encoding, - shell="bash", - ) as config: - assert config == { - "type": "stdio", - "command": "sh", - "args": [ - "-c", - "cat /tmp/hud-computer-bridge-token.response & reader=$!; " - "cat > /tmp/hud-computer-bridge-token.request; wait $reader", - ], - } - assert not bridge_stdin.closed - assert not local.stdin.closed - - bridge_command = connection.create_process.await_args.args[0] - assert "mkfifo -- /tmp/hud-computer-bridge-token.request" in bridge_command - assert "printf 'ready\\n' >&2" in bridge_command - spawn.assert_awaited_once() - spawn_call = spawn.await_args - assert spawn_call is not None - spawn_args = spawn_call.args - assert spawn_args[:3] == ( - sys.executable, - "-m", - "hud.agents.claude.sdk.computer_mcp", - ) - environ = spawn_call.kwargs["env"] - assert json.loads(environ[computer_mcp.RFB_CAPABILITY_ENV]) == screen.to_manifest() - assert environ[computer_mcp.SCREENSHOT_ENCODING_ENV] == encoding.model_dump_json() - bridge.channel.close.assert_called_once() - bridge.wait_closed.assert_awaited_once() - assert bridge_stdin.closed - assert local.stdin.closed - assert local.terminated - assert not local.killed - - -async def test_computer_mcp_bridge_rejects_windows_before_starting_resources() -> None: - screen = Capability.rfb(name="screen", url="rfb://127.0.0.1:41000", display=0) - ssh = SimpleNamespace(create_process=AsyncMock()) - - with pytest.raises(RuntimeError, match="requires a POSIX workspace"): - async with computer_mcp.bridge_computer_mcp( - cast("Any", ssh), - screen, - WebPScreenshotEncoding(), - shell="powershell", - ): - pass - - ssh.create_process.assert_not_awaited() - - -@pytest.mark.skipif(sys.platform == "win32", reason="POSIX FIFO relay") -async def test_computer_mcp_bridge_relays_both_directions(monkeypatch: pytest.MonkeyPatch) -> None: - """The remote FIFO bridge and the relay Claude runs form one duplex pipe to the controller.""" - spawn = asyncio.create_subprocess_exec - - async def remote_shell(command: str) -> Any: - process = await asyncio.create_subprocess_shell( - command, - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - return SimpleNamespace( - stdin=process.stdin, - stdout=process.stdout, - stderr=process.stderr, - channel=SimpleNamespace(close=lambda: None), - wait_closed=process.wait, - ) - - async def echo_controller(*_args: Any, **kwargs: Any) -> asyncio.subprocess.Process: - return await spawn( - "cat", stdin=kwargs["stdin"], stdout=kwargs["stdout"], stderr=kwargs["stderr"] - ) - - monkeypatch.setattr(computer_mcp.asyncio, "create_subprocess_exec", echo_controller) - screen = Capability.rfb(name="screen", url="rfb://127.0.0.1:41000", display=0) - - async with asyncio.timeout(10): - async with computer_mcp.bridge_computer_mcp( - cast("Any", SimpleNamespace(create_process=remote_shell)), - screen, - WebPScreenshotEncoding(), - shell="bash", - ) as config: - relay = await spawn( - config["command"], - *config["args"], - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - ) - assert relay.stdin is not None and relay.stdout is not None - relay.stdin.write(b'{"method":"tools/list"}\n') - await relay.stdin.drain() - assert await relay.stdout.readline() == b'{"method":"tools/list"}\n' - relay.stdin.close() - await relay.wait() - - -async def test_concurrent_runs_keep_their_ssh_state_isolated() -> None: - first_process = _FakeStreamProcess(_STREAM_JSON, pause_after=0) - conn_a = _FakeConn({}, first_process) - conn_b = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - run_a, run_b = _fake_run(), _fake_run() - - first = asyncio.create_task( - run_claude(ClaudeCLIConfig(), run_a, ssh=_ssh_with_conn("bash", conn_a), prompt="first") - ) - await first_process.stdout.blocked.wait() - await run_claude( - ClaudeCLIConfig(), run_b, ssh=_ssh_with_conn("powershell", conn_b), prompt="second" - ) - first_process.stdout.release.set() - await first - - assert conn_a.write_commands == [] - assert "--input-format=stream-json" in conn_a.ran[0] - assert json.loads(first_process.stdin.data)["message"]["content"][0]["text"] == "first" - assert conn_b.ran == ["cmd /c .hud_run.bat"] - assert json.loads(conn_b.written[".hud_input.jsonl"])["message"]["content"][0]["text"] == ( - "second" - ) - assert run_a.trace.content == run_b.trace.content == "done" - - -def test_events_record_document_and_unknown_tool_result_blocks() -> None: - """A PDF Read returns a document block; neither it nor an unknown block ends the run.""" - run = _fake_run() - events = ClaudeEvents(run, started_at="2026-01-01T00:00:00Z") - events.consume( - json.dumps( - { - "type": "assistant", - "message": { - "id": "msg-1", - "type": "message", - "role": "assistant", - "model": "claude-test", - "content": [{"type": "tool_use", "id": "tool-1", "name": "Read", "input": {}}], - "stop_reason": "tool_use", - "stop_sequence": None, - "usage": {"input_tokens": 1, "output_tokens": 1}, - }, - } - ) - ) - events.consume( - json.dumps( - { - "type": "user", - "message": { - "content": [ - { - "type": "tool_result", - "tool_use_id": "tool-1", - "content": [ - { - "type": "document", - "source": { - "type": "base64", - "media_type": "application/pdf", - "data": "JVBERi0=", - }, - }, - {"type": "novelty", "payload": 1}, - ], - } - ] - }, - } - ) - ) - - tool = cast("ToolStep", run.steps[1]) - assert tool.result is not None - document, unknown = tool.result.content - assert isinstance(document, EmbeddedResource) - assert isinstance(document.resource, BlobResourceContents) - assert document.resource.mimeType == "application/pdf" - assert document.resource.blob == "JVBERi0=" - assert str(document.resource.uri) == "document://tool-1/0" - assert isinstance(unknown, TextContent) - assert unknown.text == "[unsupported novelty block]" - assert events.error is None diff --git a/hud/agents/tests/test_codex_cli_agent.py b/hud/agents/tests/test_codex_cli_agent.py deleted file mode 100644 index 84c77b452..000000000 --- a/hud/agents/tests/test_codex_cli_agent.py +++ /dev/null @@ -1,391 +0,0 @@ -"""CodexCLIAgent command construction and JSONL trajectory mapping.""" - -from __future__ import annotations - -import asyncio -import base64 -from types import SimpleNamespace -from typing import Any, cast -from unittest.mock import AsyncMock - -import pytest -from mcp.types import TextContent - -from hud.agents.cli import resolve_executable -from hud.agents.codex import CodexCLIAgent -from hud.agents.tests.cli_fakes import FakeClient as _FakeClient -from hud.agents.tests.cli_fakes import FakeProcess as _FakeProcess -from hud.agents.tests.cli_fakes import fake_run as _fake_run -from hud.agents.types import AgentStep, CodexCLIConfig, ToolStep -from hud.capabilities import Capability, Connection -from hud.eval.runtime import RuntimeConfig, RuntimeResources -from hud.settings import settings -from hud.telemetry.context import set_trace_context - - -@pytest.fixture(autouse=True) -def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", None) - monkeypatch.setattr(settings, "openai_api_key", None) - monkeypatch.setattr( - "hud.agents.codex.agent.resolve_executable", - AsyncMock(return_value="codex"), - ) - - -class _FakeSSH: - def __init__( - self, process: _FakeProcess, *, shell: str = "bash", executable_head: bytes = b"\x7fE" - ) -> None: - self.process = process - self.executable_head = executable_head - self.capability = Capability( - name="shell", - protocol="ssh/2", - url="ssh://localhost:22", - params={"shell": shell}, - ) - self.commands: list[str] = [] - - async def run(self, command: str, **_: Any) -> SimpleNamespace: - assert command.startswith("head -c 2 -- ") - return SimpleNamespace(returncode=0, stdout=self.executable_head) - - async def create_process( - self, command: str, *, connections: tuple[Connection, ...] = () - ) -> _FakeProcess: - self.commands.append(command) - return self.process - - -async def _sent_command( - config: CodexCLIConfig, shell: str = "bash", connection: Connection | None = None -) -> str: - ssh = _FakeSSH(_FakeProcess(_STREAM_JSON), shell=shell) - connections = {"inference": connection} if connection is not None else None - await CodexCLIAgent(config)(_fake_run(_FakeClient(ssh), "Fix it", connections)) - (command,) = ssh.commands - return command - - -_STREAM_JSON = ( - '{"type":"thread.started","thread_id":"thread-1"}\n' - '{"type":"turn.started"}\n' - '{"type":"item.started","item":{"id":"cmd-1","type":"command_execution",' - '"command":"pytest -q","aggregated_output":"","exit_code":null,' - '"status":"in_progress"}}\n' - '{"type":"item.completed","item":{"id":"cmd-1","type":"command_execution",' - '"command":"pytest -q","aggregated_output":"1 passed\\n","exit_code":0,' - '"status":"completed"}}\n' - '{"type":"item.completed","item":{"id":"patch-1","type":"file_change",' - '"changes":[{"path":"calc.py","kind":"update"}],"status":"completed"}}\n' - '{"type":"item.started","item":{"id":"mcp-1","type":"mcp_tool_call",' - '"server":"db","tool":"query","arguments":{"sql":"select 42"},' - '"result":null,"error":null,"status":"in_progress"}}\n' - '{"type":"item.completed","item":{"id":"mcp-1","type":"mcp_tool_call",' - '"server":"db","tool":"query","arguments":{"sql":"select 42"},' - '"result":{"content":[{"type":"text","text":"42"}],' - '"structured_content":{"answer":42}},"error":null,"status":"completed"}}\n' - '{"type":"item.completed","item":{"id":"search-1","type":"web_search",' - '"query":"HUD evals","action":{"type":"search"}}}\n' - '{"type":"item.completed","item":{"id":"reason-1","type":"reasoning",' - '"text":"The test now passes."}}\n' - '{"type":"item.completed","item":{"id":"message-1","type":"agent_message",' - '"text":"Implemented and verified."}}\n' - '{"type":"turn.completed","usage":{"input_tokens":20,"cached_input_tokens":5,' - '"output_tokens":8,"reasoning_output_tokens":3}}\n' -) - - -async def test_command_passes_the_configured_reasoning_effort() -> None: - command = await _sent_command(CodexCLIConfig(reasoning_effort="max")) - assert 'model_reasoning_effort="max"' in command - - -async def test_command_omits_reasoning_effort_when_unconfigured() -> None: - command = await _sent_command(CodexCLIConfig()) - assert "model_reasoning_effort" not in command - - -async def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - monkeypatch.setattr(settings, "openai_api_key", "openai-key") - - with set_trace_context("trace-123"): - gateway = await _sent_command(CodexCLIConfig(gateway=True)) - provider = await _sent_command(CodexCLIConfig(gateway=False)) - - assert "HUD_API_KEY=hud-key" in gateway - assert 'model_provider="hud"' in gateway - assert f'model_providers.hud.base_url="{settings.hud_gateway_url}"' in gateway - assert "Trace-Id" in gateway - assert "CODEX_API_KEY=openai-key" in provider - assert "model_provider" not in provider - for command in (gateway, provider): - assert "codex exec" in command - assert "--json" in command - assert "--ephemeral" in command - assert "--ignore-user-config" not in command - assert "mktemp -d" in command - assert 'export CODEX_HOME="$codex_home"' in command - assert "--sandbox workspace-write" in command - assert "--model gpt-5.6-sol" in command - assert command.endswith(" -") - - -@pytest.mark.parametrize("sandbox", ["read-only", "workspace-write", "danger-full-access"]) -async def test_command_uses_process_bound_connection_without_its_credential( - monkeypatch: pytest.MonkeyPatch, sandbox: str -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - - with set_trace_context("trace-123"): - command = await _sent_command( - CodexCLIConfig.model_validate({"sandbox": sandbox}), connection=connection - ) - - assert "scoped-runtime-token" not in command - assert "hud-key" not in command - assert "HUD_CONNECTION_CREDENTIAL=hud-process-bound" in command - assert 'model_providers.hud.env_key="HUD_CONNECTION_CREDENTIAL"' in command - assert f'model_providers.hud.base_url="{connection.client_url}"' in command - assert "Trace-Id" not in command - assert "exec env" in command - assert "--sandbox danger-full-access" in command - - -async def test_process_bound_connection_rejects_a_launcher_script_before_starting() -> None: - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - ssh = _FakeSSH(_FakeProcess(_STREAM_JSON), executable_head=b"#!") - - with pytest.raises(RuntimeError, match="launcher script"): - await CodexCLIAgent()(_fake_run(_FakeClient(ssh), "Fix it", {"inference": connection})) - - assert ssh.commands == [] - - -@pytest.mark.parametrize("shell", ["bash", "powershell"]) -async def test_command_preserves_ambient_codex_login_without_explicit_credentials( - shell: str, -) -> None: - command = await _sent_command(CodexCLIConfig(), shell) - script = ( - base64.b64decode(command.rsplit(" ", 1)[1]).decode("utf-16-le") - if shell == "powershell" - else command - ) - - assert "CODEX_HOME" not in script - assert "CODEX_API_KEY" not in script - assert "HUD_API_KEY" not in script - assert "mktemp" not in script - assert "codex exec" in script or "& 'codex' 'exec'" in script - - -async def test_windows_command_encodes_environment_and_arguments( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "openai_api_key", "key&value's") - config = CodexCLIConfig(gateway=False, sandbox="danger-full-access") - command = await _sent_command(config, "powershell") - - script = base64.b64decode(command.rsplit(" ", 1)[1]).decode("utf-16-le") - assert "$env:CODEX_API_KEY='key&value''s'" in script - assert "$env:CODEX_HOME=$codexHome" in script - assert "[System.Guid]::NewGuid()" in script - assert "Remove-Item -Recurse -Force $codexHome" in script - assert "--ignore-user-config" not in script - assert "'--sandbox' 'danger-full-access'" in script - assert "& 'codex' 'exec'" in script - assert script.endswith(";exit $hudExitCode") - - -async def test_exec_streams_prompt_and_records_codex_items() -> None: - process = _FakeProcess(_STREAM_JSON) - run = _fake_run(_FakeClient(_FakeSSH(process)), "Fix the failing test") - - await CodexCLIAgent()(run) - - assert process.stdin.data == b"Fix the failing test" - assert process.stdin.eof - assert [type(step) for step in run.steps] == [ - ToolStep, - ToolStep, - ToolStep, - ToolStep, - AgentStep, - AgentStep, - ] - command = cast("ToolStep", run.steps[0]) - assert command.call is not None - assert command.call.name == "shell" - assert command.call.arguments == {"command": "pytest -q"} - assert command.result is not None - assert command.result.isError is False - output = command.result.content[0] - assert isinstance(output, TextContent) - assert output.text == "1 passed\n" - patch = cast("ToolStep", run.steps[1]) - assert patch.call is not None - assert patch.call.name == "apply_patch" - mcp = cast("ToolStep", run.steps[2]) - assert mcp.call is not None - assert mcp.call.name == "query" - assert mcp.call.provider_name == "db.query" - assert mcp.result is not None - assert mcp.result.structuredContent == {"answer": 42} - search = cast("ToolStep", run.steps[3]) - assert search.call is not None - assert search.call.name == "web_search" - assert cast("AgentStep", run.steps[4]).reasoning == "The test now passes." - assert cast("AgentStep", run.steps[5]).content == "Implemented and verified." - assert run.trace.content == "Implemented and verified." - assert run.trace.extra["codex_thread_id"] == "thread-1" - assert run.trace.extra["usage"]["cached_input_tokens"] == 5 - assert run.trace.status is None - - -async def test_exec_records_completed_items_before_process_exit() -> None: - process = _FakeProcess(_STREAM_JSON, pause_after=4) - run = _fake_run(_FakeClient(_FakeSSH(process)), "Fix it") - execution = asyncio.create_task(CodexCLIAgent()(run)) - await process.stdout.blocked.wait() - - assert not execution.done() - assert len(run.steps) == 1 - assert isinstance(run.steps[0], ToolStep) - - process.stdout.release.set() - await execution - - -async def test_exec_turn_failure_raises() -> None: - stream = ( - '{"type":"thread.started","thread_id":"thread-1"}\n' - '{"type":"turn.started"}\n' - '{"type":"turn.failed","error":{"message":"model unavailable"}}\n' - ) - run = _fake_run(_FakeClient(_FakeSSH(_FakeProcess(stream))), "Fix it") - - with pytest.raises(RuntimeError, match="model unavailable"): - await CodexCLIAgent()(run) - - -async def test_exec_nonzero_exit_raises_stderr() -> None: - process = _FakeProcess("", stderr="authentication failed", returncode=1) - run = _fake_run(_FakeClient(_FakeSSH(process)), "Fix it") - - with pytest.raises(RuntimeError, match="authentication failed"): - await CodexCLIAgent()(run) - - assert run.trace.extra["returncode"] == 1 - - -async def test_exec_nonzero_exit_prefers_structured_error() -> None: - stream = '{"type":"error","message":"gateway rejected streaming"}\n' - process = _FakeProcess(stream, stderr="noisy warning", returncode=1) - run = _fake_run(_FakeClient(_FakeSSH(process)), "Fix it") - - with pytest.raises(RuntimeError, match="gateway rejected streaming"): - await CodexCLIAgent()(run) - - assert "stderr" not in run.trace.extra - - -async def test_exec_closes_process_when_cancelled() -> None: - process = _FakeProcess(_STREAM_JSON, pause_after=0) - execution = asyncio.create_task( - CodexCLIAgent()(_fake_run(_FakeClient(_FakeSSH(process)), "Fix it")) - ) - await process.stdout.blocked.wait() - execution.cancel() - - with pytest.raises(asyncio.CancelledError): - await execution - - assert process.closed - - -async def test_agent_runs_in_the_workspace_shell_with_the_run_prompt() -> None: - process = _FakeProcess(_STREAM_JSON) - ssh = _FakeSSH(process, shell="powershell") - - await CodexCLIAgent()(_fake_run(_FakeClient(ssh), "Fix it")) - - (command,) = ssh.commands - script = base64.b64decode(command.rsplit(" ", 1)[1]).decode("utf-16-le") - assert "& 'codex' 'exec'" in script - assert process.stdin.data == b"Fix it" - - -async def test_executable_resolution_prefers_matching_managed_bundle() -> None: - ssh = SimpleNamespace( - capability=Capability.ssh(url="ssh://localhost:22", host_pubkey="key", shell="bash"), - run=AsyncMock( - side_effect=[ - SimpleNamespace(returncode=0, stdout=b"Linux\nx86_64\ngnu\n"), - SimpleNamespace(returncode=0, stdout=b""), - ] - ), - ) - - executable = await resolve_executable( - cast("Any", ssh), - "codex", - {"linux-x64": "/usr/local/lib/agents/codex/bin/codex"}, - RuntimeConfig(resources=RuntimeResources(os="linux")), - ) - - assert executable == "/usr/local/lib/agents/codex/bin/codex" - assert ssh.run.await_count == 2 - - -@pytest.mark.parametrize(("reported", "platform"), [("X64", "x64"), ("Arm64", "arm64")]) -async def test_executable_resolution_understands_windows_architectures( - reported: str, platform: str -) -> None: - ssh = SimpleNamespace( - capability=Capability.ssh(url="ssh://localhost:22", host_pubkey="key", shell="powershell"), - run=AsyncMock( - side_effect=[ - SimpleNamespace(returncode=0, stdout=f"{reported}\r\n".encode()), - SimpleNamespace(returncode=0, stdout=b"C:\\bin\\codex.exe\r\n"), - ] - ), - ) - - executable = await resolve_executable( - cast("Any", ssh), - "codex", - {f"linux-{platform}": "/usr/local/lib/agents/codex/bin/codex"}, - RuntimeConfig(resources=RuntimeResources(os="windows")), - ) - - assert executable == "C:\\bin\\codex.exe" - assert ssh.run.await_args_list[1].args[0] == "where.exe codex" - - -async def test_executable_resolution_rejects_runtime_os_mismatch() -> None: - ssh = SimpleNamespace( - capability=Capability.ssh(url="ssh://localhost:22", host_pubkey="key", shell="bash"), - run=AsyncMock(return_value=SimpleNamespace(returncode=0, stdout=b"Linux\nx86_64\ngnu\n")), - ) - - with pytest.raises(RuntimeError, match=r"requested 'windows'.*reports 'linux'"): - await resolve_executable( - cast("Any", ssh), - "codex", - {}, - RuntimeConfig(resources=RuntimeResources(os="windows")), - ) diff --git a/hud/agents/tests/test_gemini_agent.py b/hud/agents/tests/test_gemini_agent.py deleted file mode 100644 index 30b672dda..000000000 --- a/hud/agents/tests/test_gemini_agent.py +++ /dev/null @@ -1,319 +0,0 @@ -"""``GeminiAgent`` — ``get_response`` parsing over a fake Generate Content client, -plus ``_make_tool_call`` mapping and ``_grounding_citations``. -""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, cast - -import mcp.types as mcp_types - -from hud.agents.gemini.agent import GeminiAgent, _grounding_citations -from hud.agents.tool_agent import RunState -from hud.agents.types import GeminiConfig -from hud.types import MCPToolResult, Step, Trace - -if TYPE_CHECKING: - from hud.eval.run import Run - - -class FakeModels: - def __init__(self, response: Any) -> None: - self._response = response - - async def generate_content(self, **_kwargs: Any) -> Any: - return self._response - - -class FakeGenai: - def __init__(self, response: Any) -> None: - self.aio = SimpleNamespace(models=FakeModels(response)) - - -def _agent(response: Any) -> GeminiAgent: - return GeminiAgent( - GeminiConfig(model="gemini-test", include_thoughts=False, model_client=FakeGenai(response)) - ) - - -def _state(agent: GeminiAgent) -> Any: - return RunState(messages=[agent._format_message("user", "go")]) - - -def test_format_message_uses_model_role() -> None: - agent = _agent(SimpleNamespace(candidates=[])) - assert agent._format_message("assistant", "hi").role == "model" - assert agent._format_message("user", "hi").role == "user" - - -def _api_response(*candidates: Any) -> Any: - """A fake ``GenerateContentResponse``: candidates plus the response envelope.""" - return SimpleNamespace( - candidates=list(candidates), - model_version="gemini-test-v2", - usage_metadata=SimpleNamespace( - prompt_token_count=5, - candidates_token_count=3, - cached_content_token_count=None, - ), - ) - - -async def test_get_response_text_and_function_call() -> None: - resp_content = SimpleNamespace( - role="model", - parts=[ - SimpleNamespace(function_call=None, text="hi", thought=None), - SimpleNamespace( - function_call=SimpleNamespace(name="bash", args={"command": "ls"}), - text=None, - thought=None, - ), - ], - ) - response = _api_response( - SimpleNamespace( - content=resp_content, - grounding_metadata=None, - finish_reason=SimpleNamespace(name="STOP"), - ) - ) - agent = _agent(response) - - result = await agent.get_response(_state(agent)) - - assert result.content == "hi" - assert [tc.name for tc in result.tool_calls] == ["bash"] - assert result.done is False - assert result.finish_reason == "STOP" - # Model and usage are normalized off the provider response. - assert result.model == "gemini-test-v2" - assert result.usage is not None - assert result.usage.prompt_tokens == 5 - assert result.usage.completion_tokens == 3 - - -async def test_get_response_done_text_only() -> None: - resp_content = SimpleNamespace( - role="model", - parts=[SimpleNamespace(function_call=None, text="answer", thought=None)], - ) - response = _api_response( - SimpleNamespace(content=resp_content, grounding_metadata=None, finish_reason=None) - ) - agent = _agent(response) - result = await agent.get_response(_state(agent)) - assert result.done is True - assert result.content == "answer" - - -async def test_get_response_no_candidates_raises() -> None: - agent = _agent(SimpleNamespace(candidates=[])) - try: - await agent.get_response(_state(agent)) - except RuntimeError: - pass - else: # pragma: no cover - raise AssertionError("expected RuntimeError for empty candidates") - - -def test_make_tool_call_maps_predefined_to_computer() -> None: - agent = _agent(SimpleNamespace(candidates=[])) - fc = SimpleNamespace(name="click_at", args={"x": 1}) - tc = agent._make_tool_call(cast("Any", fc), cast("Any", SimpleNamespace(name="computer_use"))) - assert tc.name == "computer_use" - assert tc.arguments == {"action": "click_at", "x": 1} - assert tc.provider_name == "click_at" - - -def test_make_tool_call_plain_function() -> None: - agent = _agent(SimpleNamespace(candidates=[])) - fc = cast("Any", SimpleNamespace(name="bash", args={"command": "ls"})) - tc = agent._make_tool_call(fc, None) - assert tc.name == "bash" - assert tc.arguments == {"command": "ls"} - - -def test_grounding_citations() -> None: - meta = SimpleNamespace( - grounding_chunks=[SimpleNamespace(web=SimpleNamespace(uri="http://x", title="T"))], - grounding_supports=[ - SimpleNamespace( - segment=SimpleNamespace(text="seg", start_index=0, end_index=3), - grounding_chunk_indices=[0], - ) - ], - ) - cites = _grounding_citations(cast("Any", meta)) - assert len(cites) == 1 - assert cites[0].source == "http://x" - assert cites[0].type == "grounding" - - -async def test_malformed_function_call_is_a_normalized_error() -> None: - response = _api_response( - SimpleNamespace( - content=None, - grounding_metadata=None, - finish_reason=SimpleNamespace(name="MALFORMED_FUNCTION_CALL"), - ) - ) - agent = _agent(response) - result = await agent.get_response(_state(agent)) - assert result.error == "Provider returned a malformed function call" - assert result.stop_reason == "malformed_tool_call" - assert result.finish_reason == "MALFORMED_FUNCTION_CALL" - assert result.usage is not None - - -class ConfigRecordingModels: - def __init__(self, response: Any) -> None: - self._response = response - self.configs: list[Any] = [] - - async def generate_content(self, **kwargs: Any) -> Any: - self.configs.append(kwargs["config"]) - return self._response - - -async def _sent_output_cap(**config: Any) -> int | None: - resp_content = SimpleNamespace( - role="model", - parts=[SimpleNamespace(function_call=None, text="answer", thought=None)], - ) - models = ConfigRecordingModels( - _api_response( - SimpleNamespace(content=resp_content, grounding_metadata=None, finish_reason=None) - ) - ) - agent = GeminiAgent( - GeminiConfig( - model="gemini-test", - include_thoughts=False, - model_client=SimpleNamespace(aio=SimpleNamespace(models=models)), - **config, - ) - ) - await agent.get_response(_state(agent)) - return models.configs[0].max_output_tokens - - -async def test_output_cap_defaults_to_the_model_limit() -> None: - assert await _sent_output_cap() is None - - -async def test_configured_output_cap_is_sent() -> None: - assert await _sent_output_cap(max_output_tokens=4096) == 4096 - - -class RecordingModels: - """Replays scripted responses, recording the contents each request carried.""" - - def __init__(self, responses: list[Any]) -> None: - self._responses = list(responses) - self.requests: list[Any] = [] - - async def generate_content(self, **kwargs: Any) -> Any: - # Snapshot: the agent goes on mutating the list it passed. - self.requests.append(list(kwargs["contents"])) - return self._responses.pop(0) - - -class RecordingGenai: - def __init__(self, responses: list[Any]) -> None: - self.models = RecordingModels(responses) - self.aio = SimpleNamespace(models=self.models) - - -class BigOutputTool: - """Tool stub whose text block exceeds the configured budget.""" - - name = "big" - provider_name = "big" - - def __init__(self, text: str) -> None: - self._text = text - - async def execute(self, arguments: dict[str, Any]) -> MCPToolResult: - del arguments - return MCPToolResult(content=[mcp_types.TextContent(type="text", text=self._text)]) - - -class FakeRun: - """Offline stand-in for ``Run``: records steps onto a local trace only.""" - - def __init__(self) -> None: - self.trace = Trace() - - def record(self, step: Step) -> None: - self.trace.record(step) - - -def _model_turn(*parts: Any) -> Any: - return _api_response( - SimpleNamespace( - content=SimpleNamespace(role="model", parts=list(parts)), - grounding_metadata=None, - finish_reason=None, - ) - ) - - -def _tool_output(contents: Any) -> str: - parts = contents[-1].parts - assert parts is not None - function_response = parts[0].function_response - assert function_response is not None - response = function_response.response - assert response is not None - return cast("str", response["output"]) - - -async def _run_with_tool_output(text: str, budget: int) -> list[Any]: - """Drive one tool call through the loop, returning the requests Gemini saw.""" - client = RecordingGenai( - [ - _model_turn( - SimpleNamespace( - function_call=SimpleNamespace(name="big", args={}), - text=None, - thought=None, - ) - ), - _model_turn(SimpleNamespace(function_call=None, text="done", thought=None)), - ] - ) - agent = GeminiAgent( - GeminiConfig( - model="gemini-test", - include_thoughts=False, - model_client=client, - max_tool_result_chars=budget, - max_steps=3, - ) - ) - state = _state(agent) - state.tools = {"big": cast("Any", BigOutputTool(text))} - - await agent._loop(cast("Run", FakeRun()), state) - - assert len(client.models.requests) == 2 - return client.models.requests - - -async def test_oversized_tool_output_is_bounded_before_the_next_request() -> None: - requests = await _run_with_tool_output("h" * 500 + "t" * 500, budget=100) - - output = _tool_output(requests[1]) - assert "900 omitted" in output - assert "[900 characters omitted]" in output - assert "h" * 20 in output - assert "h" * 21 not in output - assert "t" * 80 in output - - -async def test_tool_output_within_budget_reaches_the_model_intact() -> None: - requests = await _run_with_tool_output("short", budget=100) - - assert _tool_output(requests[1]) == "short" diff --git a/hud/agents/tests/test_openai_agent.py b/hud/agents/tests/test_openai_agent.py deleted file mode 100644 index f733f268c..000000000 --- a/hud/agents/tests/test_openai_agent.py +++ /dev/null @@ -1,340 +0,0 @@ -"""``OpenAIAgent`` — construction + ``get_response`` parsing of the Responses API, -with a fake ``AsyncOpenAI`` client (no network). -""" - -from __future__ import annotations - -import asyncio -from types import SimpleNamespace -from typing import Any, cast -from unittest.mock import AsyncMock, Mock - -import mcp.types as mcp_types -import pytest -from openai.types.responses import ResponseOutputText - -from hud.agents.openai.agent import EmptyShellCallError, OpenAIAgent, OpenAIRunState -from hud.agents.openai.tools.base import format_openai_result -from hud.agents.openai.tools.computer import OPENAI_COMPUTER_SPEC, OpenAIComputerTool -from hud.agents.types import OpenAIConfig -from hud.capabilities import Capability, RFBClient -from hud.clients.client import HudClient -from hud.eval.run import Run -from hud.types import MCPToolCall, MCPToolResult - - -class FakeResponses: - def __init__(self, response: Any) -> None: - self._response = response - self.calls: list[dict[str, Any]] = [] - - async def create(self, **kwargs: Any) -> Any: - self.calls.append(kwargs) - return self._response - - -class FakeOpenAI: - def __init__(self, response: Any) -> None: - self.responses = FakeResponses(response) - - -def _agent(response: Any) -> OpenAIAgent: - return OpenAIAgent(OpenAIConfig(model="gpt-test", model_client=FakeOpenAI(response))) - - -def test_format_message_shapes_user_text() -> None: - agent = _agent(SimpleNamespace(id="r", output=[])) - msg = cast("dict[str, Any]", agent._format_message("user", "hello")) - assert msg["role"] == "user" - - -def test_format_openai_result_empty_output_emits_one_text_item() -> None: - # The Responses API rejects a function_call_output with an empty output - # list, so a contentless tool result must still produce one input_text item. - call = MCPToolCall(id="call_1", name="noop") - formatted = cast("dict[str, Any]", format_openai_result(call, MCPToolResult(content=[]))) - - assert formatted["type"] == "function_call_output" - assert formatted["call_id"] == "call_1" - assert formatted["output"] == [{"type": "input_text", "text": ""}] - - -def test_format_computer_result_preserves_screenshot_mime_type() -> None: - agent = _agent(SimpleNamespace(id="r", output=[])) - tool = OpenAIComputerTool(spec=OPENAI_COMPUTER_SPEC, client=cast("Any", object())) - state = OpenAIRunState(tools={"computer": tool}) - result = MCPToolResult( - content=[ - mcp_types.ImageContent( - type="image", - data="d2VicA==", - mimeType="image/webp", - ), - ], - ) - - formatted = cast( - "dict[str, Any]", - agent._format_result(MCPToolCall(id="call_1", name="computer"), result, state), - ) - - assert formatted["output"]["image_url"] == "data:image/webp;base64,d2VicA==" - - -def _api_response( - id: str, output: list[Any], usage: Any = None, incomplete_details: Any = None -) -> Any: - """A fake Responses-API payload: output items plus the response envelope.""" - return SimpleNamespace( - id=id, - output=output, - model="gpt-test-v1", - usage=usage, - incomplete_details=incomplete_details, - ) - - -@pytest.fixture -def computer_client() -> Mock: - screen = Mock( - spec=RFBClient, - drain=AsyncMock(), - screenshot_png=AsyncMock(return_value=(b"screen", "image/png")), - ) - return Mock( - spec=HudClient, - manifest=SimpleNamespace( - bindings=[Capability.rfb(name="screen", url="rfb://localhost:5900")], - ), - open=AsyncMock(return_value=screen), - start_task=AsyncMock(return_value={"prompt": "Use the computer."}), - grade=AsyncMock(return_value={"score": 1.0}), - ) - - -@pytest.mark.parametrize( - ("actions", "error"), - [ - ([{"type": "screenshot"}, {"type": "response", "text": "all done"}], None), - ([], "actions list is empty"), - ([{"type": "keypress", "keys": ["UNKNOWN"]}, {"type": "type", "text": "later"}], "UNKNOWN"), - ], - ids=["response", "empty-call", "failed-keypress"], -) -async def test_computer_result_reaches_next_model_turn( - computer_client: Mock, actions: list[dict[str, Any]], error: str | None -) -> None: - screen = computer_client.open.return_value - screen.conn.keyboard.press.side_effect = KeyError("UNKNOWN") - call = SimpleNamespace( - type="computer_call", - call_id="call_1", - actions=[SimpleNamespace(to_dict=lambda action=action: action) for action in actions], - action=None, - pending_safety_checks=[], - ) - create = AsyncMock(side_effect=[_api_response("resp_1", [call]), _api_response("resp_2", [])]) - agent = OpenAIAgent( - OpenAIConfig(model="gpt-test", model_client=Mock(responses=Mock(create=create))), - ) - - async with Run(computer_client, "computer-task", {}) as run: - await agent(run) - - assert run.trace.status == "completed" - assert create.await_count == 2 - feedback = create.call_args_list[1].kwargs["input"] - assert feedback[0]["type"] == "computer_call_output" - assert feedback[0]["call_id"] == "call_1" - assert feedback[0]["output"]["image_url"].startswith("data:image/png;base64,") - if error: - assert error in feedback[1]["content"][0]["text"] - assert ( - "Remaining actions in this call were not executed" in feedback[1]["content"][0]["text"] - ) - screen.conn.keyboard.write.assert_not_called() - - -async def test_computer_error_preserves_failed_screenshot_context(computer_client: Mock) -> None: - computer_client.open.return_value.screenshot_png.side_effect = RuntimeError( - "display unavailable" - ) - call = SimpleNamespace( - type="computer_call", - call_id="call_1", - actions=[], - action=SimpleNamespace(to_dict=lambda: {"type": "frobnicate"}), - pending_safety_checks=[], - ) - agent = _agent(_api_response("resp_1", [call])) - - async with Run(computer_client, "computer-task", {}) as run: - await agent(run) - - assert run.trace.status == "error" - error = next(step.error for step in run.trace.steps if step.error) - assert "frobnicate" in error - assert "display unavailable" in error - - -async def test_computer_screenshot_timeout_preserves_action_context(computer_client: Mock) -> None: - timeout = TimeoutError("capture timed out") - screen = computer_client.open.return_value - screen.screenshot_png.side_effect = timeout - tool = OpenAIComputerTool(spec=OPENAI_COMPUTER_SPEC, client=screen) - - with pytest.raises(TimeoutError, match="capture timed out") as raised: - await tool.execute({"type": "frobnicate"}) - - assert raised.value is timeout - assert any("frobnicate" in note for note in raised.value.__notes__) - - -@pytest.mark.parametrize("error_type", [TimeoutError, asyncio.CancelledError]) -async def test_computer_action_interruption_stops_batch( - computer_client: Mock, error_type: type[BaseException] -) -> None: - screen = computer_client.open.return_value - error = error_type("input interrupted") - screen.drain = AsyncMock(side_effect=error) - tool = OpenAIComputerTool(spec=OPENAI_COMPUTER_SPEC, client=screen) - - with pytest.raises(error_type) as raised: - await tool.execute( - {"actions": [{"type": "move", "x": 3, "y": 4}, {"type": "type", "text": "later"}]}, - ) - - assert raised.value is error - screen.conn.keyboard.write.assert_not_called() - screen.screenshot_png.assert_not_awaited() - - -async def test_get_response_parses_text_and_function_call() -> None: - response = _api_response( - "resp_1", - [ - SimpleNamespace( - type="message", - content=[ResponseOutputText(type="output_text", text="hi", annotations=[])], - ), - SimpleNamespace( - type="function_call", - name="shell", - arguments='{"command": ["ls"]}', - call_id="call_1", - ), - ], - usage=SimpleNamespace( - input_tokens=9, - output_tokens=4, - input_tokens_details=SimpleNamespace(cached_tokens=2), - ), - ) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "go")]) - - result = await agent.get_response(state) - - assert result.content == "hi" - assert [tc.name for tc in result.tool_calls] == ["shell"] - assert result.tool_calls[0].arguments == {"command": ["ls"]} - assert result.done is False - assert state.last_response_id == "resp_1" - # Model and usage are normalized off the provider response. - assert result.model == "gpt-test-v1" - assert result.usage is not None - assert result.usage.prompt_tokens == 9 - assert result.usage.completion_tokens == 4 - assert result.usage.cached_tokens == 2 - - -async def test_get_response_done_when_no_tool_calls() -> None: - response = _api_response("resp_2", []) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "hi")]) - - result = await agent.get_response(state) - assert result.done is True - assert result.tool_calls == [] - assert result.usage is None # provider omitted usage - assert result.finish_reason is None - - -async def test_get_response_reuses_prompt_cache_key() -> None: - agent = _agent(_api_response("resp_cache", [])) - state = OpenAIRunState(messages=[agent._format_message("user", "first")]) - - await agent.get_response(state) - state.messages.append(agent._format_message("user", "second")) - await agent.get_response(state) - - calls = cast("Any", agent.openai_client.responses).calls - assert calls[0]["prompt_cache_key"] == agent.config.prompt_cache_key - assert calls[1]["prompt_cache_key"] == agent.config.prompt_cache_key - - -async def test_get_response_surfaces_token_cap_truncation() -> None: - # Responses has no finish_reason; incomplete_details.reason carries the cap. - response = _api_response( - "resp_3", [], incomplete_details=SimpleNamespace(reason="max_output_tokens") - ) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "hi")]) - - result = await agent.get_response(state) - assert result.finish_reason == "max_output_tokens" - - -async def test_get_response_short_circuits_on_consumed_messages() -> None: - agent = _agent(SimpleNamespace(id="unused", output=[])) - state = OpenAIRunState( - messages=[agent._format_message("user", "go")], - last_response_id="prev", - ) - state.message_cursor = len(state.messages) # nothing new to send - - result = await agent.get_response(state) - assert result.done is True - # No API call should have been made. - assert cast("Any", agent.openai_client.responses).calls == [] - - -async def test_get_response_parses_shell_call() -> None: - response = _api_response( - "resp_3", - [ - SimpleNamespace( - type="shell_call", - action=SimpleNamespace(to_dict=lambda: {"command": ["pwd"]}), - call_id="call_sh", - ), - ], - ) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "run")]) - - result = await agent.get_response(state) - assert [tc.name for tc in result.tool_calls] == ["shell"] - - -async def test_get_response_rejects_empty_shell_call() -> None: - # A shell_call with an empty commands array poisons the stored response - # chain (OpenAI 500s every continuation via previous_response_id), so the - # turn must fail fast and its response id must never be committed. - response = _api_response( - "resp_poisoned", - [ - SimpleNamespace( - type="shell_call", - action=SimpleNamespace(to_dict=lambda: {"commands": [], "timeout_ms": 10000}), - call_id="call_empty", - ), - ], - ) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "run")]) - - with pytest.raises(EmptyShellCallError, match="empty commands array"): - await agent.get_response(state) - - assert state.last_response_id is None diff --git a/hud/agents/tests/test_openai_compatible_agent.py b/hud/agents/tests/test_openai_compatible_agent.py deleted file mode 100644 index ea607d773..000000000 --- a/hud/agents/tests/test_openai_compatible_agent.py +++ /dev/null @@ -1,113 +0,0 @@ -"""``OpenAIChatAgent`` — chat.completions ``get_response`` parsing + error path.""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any, cast - -import pytest - -from hud.agents.openai_compatible.agent import OpenAIChatAgent, OpenAIChatRunState -from hud.agents.types import OpenAIChatConfig - - -class FakeCompletions: - def __init__(self, response: Any, error: Exception | None = None) -> None: - self._response = response - self._error = error - - async def create(self, **_kwargs: Any) -> Any: - if self._error is not None: - raise self._error - return self._response - - -class FakeOpenAI: - def __init__(self, response: Any, error: Exception | None = None) -> None: - self.chat = SimpleNamespace(completions=FakeCompletions(response, error)) - - -def _agent(response: Any, error: Exception | None = None) -> OpenAIChatAgent: - client = cast("Any", FakeOpenAI(response, error)) - return OpenAIChatAgent(OpenAIChatConfig(model="m", model_client=client)) - - -def _response(content: str, tool_calls: list[Any]) -> Any: - dump: dict[str, Any] = {"role": "assistant", "content": content} - if tool_calls: - dump["tool_calls"] = [ - { - "id": tc.id, - "type": "function", - "function": {"name": tc.function.name, "arguments": tc.function.arguments}, - } - for tc in tool_calls - ] - message = SimpleNamespace( - content=content, - tool_calls=tool_calls, - refusal=None, - model_dump=lambda exclude_none=True: dump, - ) - choice = SimpleNamespace(message=message, finish_reason="stop", logprobs=None) - return SimpleNamespace( - choices=[choice], - model="m-v1", - usage=SimpleNamespace(prompt_tokens=6, completion_tokens=2, prompt_tokens_details=None), - ) - - -def _state(agent: OpenAIChatAgent) -> Any: - return OpenAIChatRunState(messages=[agent._format_message("user", "go")]) - - -async def test_get_response_text_only() -> None: - agent = _agent(_response("hi", [])) - result = await agent.get_response(_state(agent)) - assert result.content == "hi" - assert result.done is True - assert result.tool_calls == [] - # Model and usage are normalized off the provider response. - assert result.model == "m-v1" - assert result.usage is not None - assert result.usage.prompt_tokens == 6 - assert result.usage.completion_tokens == 2 - - -async def test_get_response_with_tool_call() -> None: - tc = SimpleNamespace( - type="function", - id="c1", - function=SimpleNamespace(name="read", arguments='{"path": "x"}'), - ) - agent = _agent(_response("", [tc])) - result = await agent.get_response(_state(agent)) - assert [c.name for c in result.tool_calls] == ["read"] - assert result.tool_calls[0].arguments == {"path": "x"} - assert result.done is False - - -async def test_get_response_error_path() -> None: - agent = _agent(None, error=RuntimeError("boom")) - with pytest.raises(RuntimeError, match="boom"): - await agent.get_response(_state(agent)) - - -async def test_get_response_malformed_tool_args() -> None: - """Unparseable arguments (e.g. truncated at the token limit) yield a call carrying - the raw string, and the recorded message replays "{}" so history stays parseable.""" - tc = SimpleNamespace( - type="function", - id="c1", - function=SimpleNamespace(name="bash", arguments='{"command": "'), - ) - agent = _agent(_response("", [tc])) - state = _state(agent) - result = await agent.get_response(state) - assert result.error is None - assert result.done is False - (call,) = result.tool_calls - assert call.id == "c1" - assert call.arguments == '{"command": "' - recorded = state.messages[-1] - assert recorded["tool_calls"][0]["function"]["arguments"] == "{}" diff --git a/hud/agents/tests/test_provider_native_tools.py b/hud/agents/tests/test_provider_native_tools.py deleted file mode 100644 index 3ff92bbb8..000000000 --- a/hud/agents/tests/test_provider_native_tools.py +++ /dev/null @@ -1,700 +0,0 @@ -"""Provider native tool adapters: translate a provider tool call into SSH execution. - -Each provider exposes its own LLM-facing schema (``to_params``) but executes over a -shared ``SSHClient`` (``self.bash`` -> ``conn.run``). These tests inject a fake SSH -client and assert the command translation + result shape, fully offline. -""" - -from __future__ import annotations - -import base64 -import shlex -from io import BytesIO -from types import SimpleNamespace -from typing import Any, cast - -import asyncssh -import mcp.types as mcp_types -import pytest -from PIL import Image - -from hud.agents.claude.tools.coding import ClaudeBashTool, ClaudeTextEditorTool -from hud.agents.gemini.tools.coding import GeminiEditTool, GeminiShellTool -from hud.agents.openai.tools.coding import OpenAIShellTool -from hud.agents.openai_compatible.agent import OpenAIChatAgent -from hud.agents.openai_compatible.tools import BashTool, EditTool, ReadTool, WriteTool -from hud.agents.tool_agent import RunState -from hud.agents.tools.base import result_text -from hud.agents.tools.ssh import bound_shell_output -from hud.agents.types import OpenAIChatConfig -from hud.capabilities import Capability, SSHClient -from hud.types import MCPToolCall - - -class _Completed: - def __init__(self, *, stdout: str | bytes = "", stderr: str = "", exit_status: int = 0) -> None: - self.stdout = stdout - self.stderr = stderr - self.exit_status = exit_status - self.returncode = exit_status - - -class _Conn: - def __init__(self, completed: _Completed, store: dict[str, bytes]) -> None: - self._completed = completed - self._store = store - self.commands: list[str] = [] - - def is_closed(self) -> bool: - return False - - async def run( - self, - command: str, - *, - input: str | None = None, - check: bool = False, - encoding: str | None = "utf-8", - ) -> _Completed: - self.commands.append(command) - parts = shlex.split(command) - if len(parts) == 3 and parts[:2] == ["cat", "--"]: - if parts[2] not in self._store: - if check: - raise asyncssh.ProcessError( - env=None, - command=command, - subsystem=None, - exit_status=1, - exit_signal=None, - returncode=1, - stdout="", - stderr=f"cat: {parts[2]}: No such file or directory", - ) - return _Completed( - stderr=f"cat: {parts[2]}: No such file or directory", exit_status=1 - ) - data = self._store[parts[2]] - return _Completed(stdout=data if encoding is None else data.decode(encoding)) - if len(parts) == 3 and parts[:2] == ["cat", ">"]: - assert input is not None - self._store[parts[2]] = input.encode() - return _Completed() - if len(parts) == 4 and parts[:3] == ["ls", "-1A", "--"]: - prefix = parts[3].rstrip("/") - prefix = "/" if not prefix else prefix + "/" - names = { - rest.split("/", 1)[0] - for file_path in self._store - if file_path.startswith(prefix) and (rest := file_path[len(prefix) :]) - } - return _Completed(stdout="\n".join(sorted(names))) - if len(parts) == 3 and parts[:2] in (["test", "-d"], ["test", "-e"]): - path = parts[2] - exists = path in self._store or any( - file_path.startswith(path.rstrip("/") + "/") for file_path in self._store - ) - if parts[1] == "-d": - exists = any( - file_path.startswith(path.rstrip("/") + "/") for file_path in self._store - ) - return _Completed(exit_status=0 if exists else 1) - if len(parts) >= 3 and parts[:2] == ["mkdir", "-p"]: - return _Completed(exit_status=0) - return self._completed - - async def create_process(self, command: str, **kwargs: Any) -> _Process: - return _Process(await self.run(command, **kwargs)) - - -class _Process: - def __init__(self, completed: _Completed) -> None: - self.completed = completed - self.closed = False - - async def wait(self, *, check: bool, timeout: float) -> _Completed: # noqa: ASYNC109 - del timeout - if check and self.completed.exit_status: - raise asyncssh.ProcessError( - env=None, - command=None, - subsystem=None, - exit_status=self.completed.exit_status, - exit_signal=None, - returncode=self.completed.exit_status, - stdout=self.completed.stdout, - stderr=self.completed.stderr, - ) - return self.completed - - def close(self) -> None: - self.closed = True - - def terminate(self) -> None: - self.closed = True - - async def wait_closed(self) -> None: - pass - - -class _FakeSSH(SSHClient): - """SSH client with an in-memory exec-channel filesystem.""" - - def __init__( - self, - *, - stdout: str = "ok", - stderr: str = "", - exit_status: int = 0, - files: dict[str, bytes] | None = None, - cwd: str | None = None, - ) -> None: - self.files: dict[str, bytes] = files or {} - params = {"cwd": cwd} if cwd else {} - super().__init__( - Capability(name="shell", protocol="ssh/2", url="ssh://localhost:22", params=params), - cast( - "Any", - _Conn( - _Completed(stdout=stdout, stderr=stderr, exit_status=exit_status), - self.files, - ), - ), - ) - - -def _ssh(**kwargs: Any) -> SSHClient: - return cast("SSHClient", _FakeSSH(**kwargs)) - - -def _commands(tool: Any) -> list[str]: - return tool.client.conn.commands - - -class _OpenAIChatAgentForTest(OpenAIChatAgent): - async def build_tools_for_test(self, ssh: SSHClient) -> tuple[dict[str, Any], list[Any]]: - return await self._build_tools({"ssh": ssh}) - - -# ─── OpenAI shell ───────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - ("command", "timeout_ms", "expected"), - [ - ("pwd", 2500, "timeout 2.5s bash -lc pwd"), - ("pwd", 500, "timeout 0.5s bash -lc pwd"), - ( - "echo ready; sleep infinity", - 500, - "timeout 0.5s bash -lc 'echo ready; sleep infinity'", - ), - ], -) -async def test_openai_shell_applies_requested_timeout_to_entire_command( - command: str, - timeout_ms: int, - expected: str, -) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - - result = await tool.execute({"commands": [command], "timeout_ms": timeout_ms}) - - assert _commands(tool) == [expected] - assert result.isError is False - assert result.structuredContent is not None - assert result.structuredContent["provider_tool"] == "shell" - assert len(result.structuredContent["output"]) == 1 - - -async def test_openai_shell_runs_each_command_without_timeout() -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - - await tool.execute({"commands": ["echo a", "echo b"]}) - - assert _commands(tool) == ["echo a", "echo b"] - - -async def test_openai_shell_has_no_hidden_timeout_across_command_batch( - monkeypatch: pytest.MonkeyPatch, -) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - calls: list[dict[str, Any]] = [] - - async def run(*args: object, **kwargs: Any) -> _Completed: - del args - calls.append(kwargs) - return _Completed() - - monkeypatch.setattr(tool.client, "run", run) - agent = _OpenAIChatAgentForTest( - OpenAIChatConfig(model="test", model_client=cast("Any", object())) - ) - - await agent._dispatch_call( - MCPToolCall(name="shell", arguments={"commands": ["echo a", "echo b"]}), - RunState(tools={"shell": tool}), - ) - - assert len(calls) == 2 - assert all("timeout" not in kwargs for kwargs in calls) - - -async def test_openai_shell_applies_limit_independently_to_each_command() -> None: - limit = 80 - tool = OpenAIShellTool( - spec=OpenAIShellTool.default_spec("gpt-5.5"), - client=_ssh(stdout="stdout-start-" + "a" * 100, stderr="b" * 100 + "-stderr-end"), - ) - - result = await tool.execute( - {"commands": ["first-command", "second-command"], "max_output_length": limit} - ) - - assert result.structuredContent is not None - outputs = result.structuredContent["output"] - assert len(outputs) == 2 - assert all(len(output["stdout"]) + len(output["stderr"]) == limit for output in outputs) - assert all(output["stdout"].startswith("stdout-start-") for output in outputs) - assert all(output["stderr"].endswith("-stderr-end") for output in outputs) - assert sum(len(output["stdout"]) + len(output["stderr"]) for output in outputs) == 2 * limit - text_blocks = [ - block.text for block in result.content if isinstance(block, mcp_types.TextContent) - ] - assert len(text_blocks) == 2 - assert all(len(text) == limit for text in text_blocks) - assert text_blocks == [output["stdout"] + output["stderr"] for output in outputs] - - -@pytest.mark.parametrize(("limit", "expected"), [(1, "["), (10, "[truncated")]) -def test_shared_output_bound_handles_limits_smaller_than_marker( - limit: int, - expected: str, -) -> None: - assert bound_shell_output("x" * 100, "", limit) == (expected, "") - - -@pytest.mark.parametrize("max_output_length", [0, -1, "20000", 20_000.0, True]) -async def test_openai_shell_rejects_invalid_output_limits_without_running( - max_output_length: Any, -) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - - result = await tool.execute( - {"commands": ["echo should-not-run"], "max_output_length": max_output_length} - ) - - assert result.isError is True - assert _commands(tool) == [] - assert result.structuredContent is not None - assert "max_output_length" not in result.structuredContent - assert result_text(result) == "max_output_length must be a positive integer" - - -@pytest.mark.parametrize("max_output_length", [None, 20 * 1024 * 1024]) -async def test_openai_shell_uses_safe_effective_limit(max_output_length: int | None) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - arguments: dict[str, Any] = {"commands": ["echo ok"]} - if max_output_length is not None: - arguments["max_output_length"] = max_output_length - - result = await tool.execute(arguments) - - assert result.structuredContent is not None - assert result.structuredContent["max_output_length"] == 10 * 1024 * 1024 - - -async def test_openai_shell_rejects_non_list_commands_without_running() -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - - result = await tool.execute({"commands": 123}) - - assert result.isError is True - assert _commands(tool) == [] - - -def test_openai_shell_to_params_is_shell_type() -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - assert tool.to_params()["type"] == "shell" - - -# ─── OpenAI-compatible OpenCode workspace tools ─────────────────────── - - -async def test_openai_compatible_catalog_matches_opencode_workspace_tools() -> None: - agent = _OpenAIChatAgentForTest( - OpenAIChatConfig(model="qwen3.6-plus", model_client=cast("Any", object())) - ) - - tools, params = await agent.build_tools_for_test(_ssh()) - - assert list(tools) == ["bash", "read", "glob", "grep", "edit", "write"] - assert [param["function"]["name"] for param in params] == [ - "bash", - "read", - "glob", - "grep", - "edit", - "write", - ] - - -async def test_openai_compatible_bash_uses_workdir_and_timeout() -> None: - tool = BashTool(spec=BashTool.default_spec("qwen"), client=_ssh()) - - await tool.execute({"command": "echo hi", "workdir": "/tmp/my dir", "timeout": 2500}) - - assert _commands(tool) == ["cd '/tmp/my dir' && timeout 3s bash -lc 'echo hi'"] - - -async def test_shared_ssh_tool_has_no_hidden_command_timeout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - ssh = _ssh() - seen_kwargs: dict[str, Any] = {} - - async def run(*args: object, **kwargs: Any) -> _Completed: - del args - seen_kwargs.update(kwargs) - return _Completed() - - monkeypatch.setattr(ssh, "run", run) - tool = BashTool(spec=BashTool.default_spec("qwen"), client=ssh) - - result = await tool.execute({"command": "sleep forever"}) - - assert result.isError is False - assert "timeout" not in seen_kwargs - - -async def test_shared_ssh_tool_reports_a_signal_returncode( - monkeypatch: pytest.MonkeyPatch, -) -> None: - ssh = _ssh() - - async def signaled(*args: object, **kwargs: Any) -> SimpleNamespace: - del args, kwargs - return SimpleNamespace( - stdout="", - stderr="terminated", - exit_status=None, - returncode=-15, - ) - - monkeypatch.setattr(ssh, "run", signaled) - tool = BashTool(spec=BashTool.default_spec("qwen"), client=ssh) - - result = await tool.execute({"command": "sleep forever"}) - - assert result.isError is True - assert result_text(result).endswith("(exit -15)") - - -async def test_shared_ssh_tool_bounds_combined_output( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr("hud.agents.tools.ssh.MAX_SHELL_OUTPUT_LENGTH", 80) - tool = BashTool( - spec=BashTool.default_spec("qwen"), - client=_ssh( - stdout="stdout-start-" + "a" * 100, - stderr="b" * 100 + "-stderr-end", - ), - ) - - result = await tool.execute({"command": "noisy"}) - - text = result_text(result) - output = text.split("\n", 1)[1].rsplit("\n(exit 0)", 1)[0] - stdout, stderr = output.split("\nstderr:\n", 1) - assert len(stdout) + len(stderr) == 80 - assert stdout.startswith("stdout-start-") - assert stderr.endswith("-stderr-end") - assert "[truncated]" in output - - -async def test_openai_compatible_write_stores_file_via_ssh_exec() -> None: - ssh = _FakeSSH() - tool = WriteTool(spec=WriteTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - result = await tool.execute({"filePath": "/REPORT.md", "content": "done"}) - - assert result.isError is False - assert ssh.files["/REPORT.md"] == b"done" - - -async def test_paths_reach_the_session_verbatim() -> None: - """A path means what it says in the session's own namespace: file helpers - and shell commands must never disagree about what a path names, so - nothing is anchored or rewritten on the way through.""" - ssh = _FakeSSH(cwd="/app", files={"/app/f.txt": b"inside"}) - tool = WriteTool(spec=WriteTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - await tool.execute({"filePath": "/tmp/probe.txt", "content": "done"}) - - assert ssh.files["/tmp/probe.txt"] == b"done" - assert "/app/tmp/probe.txt" not in ssh.files - assert await cast("SSHClient", ssh).read_text("/app/f.txt") == "inside" - - -async def test_read_maps_the_directory_predicate_and_listing_together() -> None: - """`test -d`, listing, and reads must agree on the same path, or - workspace dirs are misclassified as files.""" - ssh = _FakeSSH(cwd="/workspace", files={"/workspace/pkg/mod.py": b"x = 1\n"}) - tool = ReadTool(spec=ReadTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - result = await tool.execute({"filePath": "/workspace/pkg"}) - - text = result_text(result) - assert "directory" in text - assert "mod.py" in text - assert "test -d /workspace/pkg" in cast("Any", ssh).conn.commands - - -async def test_openai_compatible_edit_rewrites_unique_match() -> None: - ssh = _FakeSSH(files={"/f.txt": b"hello old world"}) - tool = EditTool(spec=EditTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - result = await tool.execute( - {"filePath": "/f.txt", "oldString": "old", "newString": "new"}, - ) - - assert result.isError is False - assert ssh.files["/f.txt"] == b"hello new world" - - -async def test_openai_compatible_edit_rejects_ambiguous_match() -> None: - ssh = _FakeSSH(files={"/f.txt": b"a a a"}) - tool = EditTool(spec=EditTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - result = await tool.execute( - {"filePath": "/f.txt", "oldString": "a", "newString": "b"}, - ) - - assert result.isError is True - assert ssh.files["/f.txt"] == b"a a a" - - -async def test_openai_compatible_read_lists_directories() -> None: - tool = ReadTool( - spec=ReadTool.default_spec("qwen"), - client=_ssh(files={"/work/a.txt": b"a", "/work/nested/b.txt": b"b"}), - ) - - result = await tool.execute({"filePath": "/work"}) - - text = result_text(result) - assert "directory" in text - assert "a.txt" in text - assert "nested" in text - - -async def test_openai_compatible_read_accepts_zero_offset_for_first_page() -> None: - tool = ReadTool( - spec=ReadTool.default_spec("qwen"), - client=_ssh(files={"/f.txt": b"alpha\nbeta\n"}), - ) - - result = await tool.execute({"filePath": "/f.txt", "offset": 0, "limit": 1}) - - text = result_text(result) - assert "1: alpha" in text - assert "2: beta" not in text - - -# ─── Gemini shell ───────────────────────────────────────────────────── - - -async def test_gemini_shell_scopes_command_to_quoted_directory() -> None: - tool = GeminiShellTool(spec=GeminiShellTool.default_spec("gemini"), client=_ssh()) - - await tool.execute({"command": "ls -la", "dir_path": "/tmp/my dir"}) - - assert _commands(tool) == ["cd '/tmp/my dir' && ls -la"] - - -async def test_gemini_shell_runs_bare_command() -> None: - tool = GeminiShellTool(spec=GeminiShellTool.default_spec("gemini"), client=_ssh()) - - await tool.execute({"command": "ls"}) - - assert _commands(tool) == ["ls"] - - -async def test_gemini_shell_requires_command() -> None: - tool = GeminiShellTool(spec=GeminiShellTool.default_spec("gemini"), client=_ssh()) - - with pytest.raises(ValueError, match="command is required"): - await tool.execute({"command": ""}) - - -# ─── Claude bash ────────────────────────────────────────────────────── - - -async def test_claude_bash_runs_command() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude-sonnet-4-6"), client=_ssh()) - - await tool.execute({"command": "echo hi"}) - - assert _commands(tool) == ["echo hi"] - - -async def test_claude_bash_restart_is_a_noop() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude-sonnet-4-6"), client=_ssh()) - - result = await tool.execute({"restart": True}) - - assert result.isError is False - assert result_text(result) == ( - "restart is unnecessary; each command runs in a fresh shell session" - ) - assert _commands(tool) == [] # restart never touches the shell - - -async def test_claude_bash_requires_command() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude-sonnet-4-6"), client=_ssh()) - - result = await tool.execute({}) - - assert result.isError is True - assert _commands(tool) == [] - - -def test_claude_bash_to_params_carries_native_schema() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude-sonnet-4-6"), client=_ssh()) - params = tool.to_params() - assert params == {"type": "bash_20250124", "name": "bash"} - - -def test_claude_bash_generic_spec_for_non_anthropic_model() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("qwen/qwen3.8-max"), client=_ssh()) - params: Any = tool.to_params() - assert "type" not in params - assert params["name"] == "bash" - assert params["input_schema"]["required"] == ["command"] - - -def test_claude_editor_generic_spec_for_non_anthropic_model() -> None: - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("qwen/qwen3.8-max"), - client=_ssh(), - ) - params: Any = tool.to_params() - assert "type" not in params - assert params["name"] == "str_replace_based_edit_tool" - assert params["input_schema"]["required"] == ["command", "path"] - - -# ─── editor tools over SSH exec ─────────────────────────────────────── - - -async def test_claude_text_editor_creates_file() -> None: - ssh = _FakeSSH() - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), client=cast("SSHClient", ssh) - ) - - result = await tool.execute({"command": "create", "path": "/f.txt", "file_text": "hello"}) - - assert result.isError is False - assert ssh.files["/f.txt"] == b"hello" - - -async def test_claude_text_editor_str_replace_rewrites_file() -> None: - ssh = _FakeSSH(files={"/f.txt": b"hello old world"}) - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), client=cast("SSHClient", ssh) - ) - - result = await tool.execute( - {"command": "str_replace", "path": "/f.txt", "old_str": "old", "new_str": "new"}, - ) - - assert result.isError is False - assert ssh.files["/f.txt"] == b"hello new world" - - -async def test_claude_text_editor_str_replace_errors_when_not_unique() -> None: - ssh = _FakeSSH(files={"/f.txt": b"a a a"}) - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), client=cast("SSHClient", ssh) - ) - - result = await tool.execute( - {"command": "str_replace", "path": "/f.txt", "old_str": "a", "new_str": "b"}, - ) - - assert result.isError is True # ambiguous match must not write - assert ssh.files["/f.txt"] == b"a a a" - - -async def test_gemini_edit_creates_file_when_old_string_empty() -> None: - ssh = _FakeSSH() - tool = GeminiEditTool(spec=GeminiEditTool.default_spec("gemini"), client=cast("SSHClient", ssh)) - - await tool.execute({"file_path": "/n.txt", "old_string": "", "new_string": "fresh"}) - - assert ssh.files["/n.txt"] == b"fresh" - - -async def test_reading_a_missing_file_is_a_tool_error_not_a_raised_traceback() -> None: - """Reading before creating is the first thing an editor tool does; that failure - must come back as a tool result carrying the shell's message.""" - ssh = _FakeSSH() - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), client=cast("SSHClient", ssh) - ) - - result = await tool.execute({"command": "view", "path": "/nope.txt"}) - - assert result.isError is True - assert "No such file or directory" in result_text(result) - - -@pytest.mark.parametrize("image_format", ["PNG", "JPEG", "GIF", "WEBP"]) -@pytest.mark.parametrize("tool_class", [ClaudeTextEditorTool, ReadTool]) -async def test_file_view_returns_images_as_image_content( - image_format: str, tool_class: type[ClaudeTextEditorTool] | type[ReadTool] -) -> None: - buffer = BytesIO() - Image.new("RGB", (32, 24), "red").save(buffer, format=image_format) - data = buffer.getvalue() - ssh = _FakeSSH(files={"/image with no extension": data}) - tool = tool_class(spec=tool_class.default_spec("claude"), client=ssh) - - result = await tool.execute( - { - "command": "view", - "path": "/image with no extension", - "filePath": "/image with no extension", - } - ) - - assert not result.isError - assert len(result.content) == 1 - content = result.content[0] - assert isinstance(content, mcp_types.ImageContent) - assert content.mimeType == Image.MIME[image_format] - assert base64.b64decode(content.data) == data - - -@pytest.mark.parametrize("data", [b"\x00binary", b"\xff\xfeinvalid"]) -async def test_file_view_rejects_non_image_binary(data: bytes) -> None: - ssh = _FakeSSH(files={"/binary": data}) - tool = ClaudeTextEditorTool(spec=ClaudeTextEditorTool.default_spec("claude"), client=ssh) - - result = await tool.execute({"command": "view", "path": "/binary"}) - - assert result.isError - assert "expected UTF-8 text or an image" in result_text(result) - - -async def test_file_view_preserves_unicode_text() -> None: - text = "Hello, 世界\n" - ssh = _FakeSSH(files={"/text.png": text.encode()}) - tool = ClaudeTextEditorTool(spec=ClaudeTextEditorTool.default_spec("claude"), client=ssh) - - result = await tool.execute({"command": "view", "path": "/text.png"}) - - assert not result.isError - assert result_text(result) == text diff --git a/hud/agents/tests/test_robot_dataset_writer.py b/hud/agents/tests/test_robot_dataset_writer.py deleted file mode 100644 index 9829f6f51..000000000 --- a/hud/agents/tests/test_robot_dataset_writer.py +++ /dev/null @@ -1,162 +0,0 @@ -"""DatasetWriter shares by compatible schema/FPS, not a single process-global.""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock - -import numpy as np -import pytest - -from hud.agents.robot import dataset as dataset_mod -from hud.agents.robot.dataset import DatasetWriter - - -def _contract( - *, state_shape: tuple[int, ...], action_shape: tuple[int, ...], robot: str -) -> dict[str, Any]: - return { - "robot_type": robot, - "features": { - "state": {"role": "observation", "dtype": "float32", "shape": list(state_shape)}, - "action": {"role": "action", "dtype": "float32", "shape": list(action_shape)}, - }, - } - - -@pytest.fixture(autouse=True) -def reset_dataset_writer_globals(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: - DatasetWriter._datasets.clear() - DatasetWriter._open.clear() - DatasetWriter._atexit_registered = False - monkeypatch.setenv("RECORD_DIR", str(tmp_path)) - monkeypatch.delenv("HF_REPO", raising=False) - - -def _install_fake_lerobot(monkeypatch: pytest.MonkeyPatch) -> list[MagicMock]: - created: list[MagicMock] = [] - - def create(**kwargs: Any) -> MagicMock: - ds = MagicMock(name=f"ds-{len(created)}") - ds.create_kwargs = kwargs - created.append(ds) - return ds - - monkeypatch.setitem( - __import__("sys").modules, - "lerobot.datasets.lerobot_dataset", - SimpleNamespace(LeRobotDataset=SimpleNamespace(create=create)), - ) - - def _found(_name: str) -> object: - return object() - - monkeypatch.setattr(dataset_mod.importlib.util, "find_spec", _found) - return created - - -def test_matching_writers_share_one_dataset(monkeypatch: pytest.MonkeyPatch) -> None: - created = _install_fake_lerobot(monkeypatch) - contract = _contract(state_shape=(3,), action_shape=(2,), robot="arm") - obs, act = {"state": np.zeros(3, dtype=np.float32)}, np.zeros(2, dtype=np.float32) - DatasetWriter(contract, fps=10).add(obs, act, task="t") - DatasetWriter(contract, fps=10).add(obs, act, task="t") - assert len(created) == 1 - assert len(DatasetWriter._datasets) == 1 - - -def test_different_fps_or_features_get_separate_datasets( - monkeypatch: pytest.MonkeyPatch, -) -> None: - created = _install_fake_lerobot(monkeypatch) - c1 = _contract(state_shape=(3,), action_shape=(2,), robot="arm") - c2 = _contract(state_shape=(4,), action_shape=(2,), robot="arm") - act = np.zeros(2, dtype=np.float32) - DatasetWriter(c1, fps=10).add({"state": np.zeros(3, dtype=np.float32)}, act, task="t") - DatasetWriter(c1, fps=20).add({"state": np.zeros(3, dtype=np.float32)}, act, task="t") - DatasetWriter(c2, fps=10).add({"state": np.zeros(4, dtype=np.float32)}, act, task="t") - assert len(created) == 3 - assert created[0].create_kwargs["fps"] == 10 - assert created[1].create_kwargs["fps"] == 20 - assert tuple(created[2].create_kwargs["features"]["observation.state"]["shape"]) == (4,) - - -def test_different_feature_names_get_separate_datasets( - monkeypatch: pytest.MonkeyPatch, -) -> None: - created = _install_fake_lerobot(monkeypatch) - first = _contract(state_shape=(2,), action_shape=(1,), robot="arm") - second = _contract(state_shape=(2,), action_shape=(1,), robot="arm") - first["features"]["state"]["names"] = ["shoulder", "elbow"] - second["features"]["state"]["names"] = ["x", "y"] - obs = {"state": np.zeros(2, dtype=np.float32)} - action = np.zeros(1, dtype=np.float32) - - DatasetWriter(first, fps=10).add(obs, action, task="t") - DatasetWriter(second, fps=10).add(obs, action, task="t") - - assert len(created) == 2 - assert created[0].create_kwargs["features"]["observation.state"]["names"] == [ - "shoulder", - "elbow", - ] - assert created[1].create_kwargs["features"]["observation.state"]["names"] == ["x", "y"] - - -def test_finalize_clears_all_datasets(monkeypatch: pytest.MonkeyPatch) -> None: - created = _install_fake_lerobot(monkeypatch) - act = np.zeros(2, dtype=np.float32) - a = DatasetWriter(_contract(state_shape=(3,), action_shape=(2,), robot="a"), fps=10) - b = DatasetWriter(_contract(state_shape=(5,), action_shape=(2,), robot="b"), fps=10) - a.add({"state": np.zeros(3, dtype=np.float32)}, act, task="t") - b.add({"state": np.zeros(5, dtype=np.float32)}, act, task="t") - a.end_episode() - b.end_episode() - DatasetWriter.finalize() - assert DatasetWriter._datasets == {} - for ds in created: - ds.finalize.assert_called_once() - - -def test_nested_image_keys_stay_distinct(monkeypatch: pytest.MonkeyPatch) -> None: - created = _install_fake_lerobot(monkeypatch) - contract = { - "robot_type": "stereo", - "features": { - "left/image": {"role": "observation", "dtype": "image", "shape": [2, 2, 3]}, - "right/image": {"role": "observation", "dtype": "image", "shape": [2, 2, 3]}, - "state": {"role": "observation", "dtype": "float32", "shape": [1]}, - "action": {"role": "action", "dtype": "float32", "shape": [1]}, - }, - } - writer = DatasetWriter(contract, fps=10) - left = np.zeros((2, 2, 3), dtype=np.uint8) - right = np.full((2, 2, 3), 255, dtype=np.uint8) - writer.add( - {"left/image": left, "right/image": right, "state": np.zeros(1, dtype=np.float32)}, - np.zeros(1, dtype=np.float32), - task="t", - ) - writer.end_episode() - - features = created[0].create_kwargs["features"] - assert "observation.images.left_image" in features - assert "observation.images.right_image" in features - frame = created[0].add_frame.call_args.args[0] - assert frame["observation.images.left_image"] is left - assert frame["observation.images.right_image"] is right - - -def test_duplicate_mapped_keys_raise() -> None: - contract = { - "robot_type": "bad", - "features": { - # Both wire names become the same LeRobot key after path flattening. - "cam/rgb": {"role": "observation", "dtype": "image", "shape": [2, 2, 3]}, - "cam_rgb": {"role": "observation", "dtype": "image", "shape": [2, 2, 3]}, - "action": {"role": "action", "dtype": "float32", "shape": [1]}, - }, - } - with pytest.raises(ValueError, match="both map to LeRobot key"): - DatasetWriter(contract, fps=10) diff --git a/hud/agents/tests/test_tool_agent.py b/hud/agents/tests/test_tool_agent.py deleted file mode 100644 index 19ff303b8..000000000 --- a/hud/agents/tests/test_tool_agent.py +++ /dev/null @@ -1,826 +0,0 @@ -"""``ToolAgent`` plumbing: catalog→clients, message formatting, dispatch + loop. - -The provider-specific bits are abstract; this drives a tiny concrete subclass with a -scripted ``get_response`` so the loop, dispatch, and message formatting run offline. -""" - -from __future__ import annotations - -import asyncio -from contextlib import AsyncExitStack -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, Literal, cast -from unittest.mock import AsyncMock, Mock - -import fastmcp -import mcp.types as mcp_types -import pytest -from fastmcp.client.transports import SSETransport, StreamableHttpTransport - -from hud.agents.claude.agent import ClaudeAgent -from hud.agents.claude.tools.coding import ClaudeBashTool, ClaudeTextEditorTool -from hud.agents.openai.tools.coding import OpenAIShellTool -from hud.agents.openai.tools.mcp_proxy import OpenAIMCPProxyTool -from hud.agents.tool_agent import DegenerateTurnError, RunState, ToolAgent -from hud.agents.tools.base import AgentToolSpec, result_text -from hud.agents.tools.rfb import RFBTool -from hud.agents.tools.ssh import SSHInfrastructureErrorResult -from hud.agents.types import ( - AgentStep, - ClaudeCLIConfig, - ClaudeConfig, - ToolAgentConfig, - ToolStep, -) -from hud.capabilities import ( - Capability, - CapabilityClient, - MCPClient, - RFBClient, - SSHClient, -) -from hud.capabilities.mcp import get_mcp_trace_id -from hud.capabilities.rfb import PngScreenshotEncoding, WebPScreenshotEncoding -from hud.capabilities.ssh import SSHConnectionError -from hud.telemetry.context import set_trace_context -from hud.types import MCPToolCall, MCPToolResult, Step, Trace - -if TYPE_CHECKING: - from hud.eval.run import Run - -_Msg = dict[str, Any] - - -class _FakeRun: - """Offline stand-in for ``Run``: records steps onto a local trace only.""" - - def __init__(self) -> None: - self.trace = Trace() - - def record(self, step: Step) -> None: - self.trace.record(step) - - -class DictAgent(ToolAgent[_Msg, ToolAgentConfig]): - """Minimal concrete ToolAgent over plain-dict messages.""" - - config_cls = ToolAgentConfig - - def __init__(self, turns: list[AgentStep], **config: Any) -> None: - super().__init__(ToolAgentConfig(model="test-model", **config)) - self._turns = list(turns) - - async def _initialize_state(self, *, prompt: Any) -> RunState[_Msg]: - return RunState(messages=self._initial_messages(prompt)) - - async def get_response( - self, state: RunState[_Msg], *, system_prompt: Any = None, citations_enabled: bool = False - ) -> AgentStep: - return self._turns.pop(0) - - def _format_message(self, role: str, text: str) -> _Msg: - return {"role": role, "content": text} - - def _format_result( - self, call: MCPToolCall, result: MCPToolResult, state: RunState[_Msg] - ) -> _Msg: - return {"role": "tool", "name": call.name, "isError": result.isError} - - -# ─── catalog → clients derivation ───────────────────────────────────── - - -def test_init_subclass_derives_clients_from_catalog() -> None: - class WithCatalog(DictAgent): - tool_catalog = (OpenAIShellTool,) - - assert WithCatalog.clients == (SSHClient,) - - -def test_claude_defaults_to_configurable_webp_screenshots() -> None: - assert ToolAgentConfig().screenshot_encoding == PngScreenshotEncoding() - assert ClaudeConfig().screenshot_encoding == WebPScreenshotEncoding() - assert ClaudeCLIConfig().screenshot_encoding == WebPScreenshotEncoding() - - configured = ClaudeConfig.model_validate( - {"screenshot_encoding": {"mime_type": "image/webp", "quality": 42}}, - ) - assert configured.screenshot_encoding == WebPScreenshotEncoding(quality=42) - - with pytest.raises(ValueError): - ClaudeConfig.model_validate( - {"screenshot_encoding": {"mime_type": "image/webp", "quality": 101}}, - ) - - -def test_only_claude_provider_has_a_default_tool_timeout() -> None: - config = ClaudeConfig(timeout_seconds=600) - - assert config.timeout_seconds == 600 - assert config.tool_timeout_seconds == 120 - assert ClaudeConfig(tool_timeout_seconds=None).tool_timeout_seconds is None - assert ToolAgentConfig().tool_timeout_seconds is None - - -async def test_agent_passes_screenshot_encoding_to_rfb_tools() -> None: - class ScreenTool(RFBTool): - name = "screen" - - @classmethod - def default_spec(cls, model: str) -> AgentToolSpec: - del model - return AgentToolSpec(api_type="screen", api_name="screen") - - async def execute(self, arguments: dict[str, Any]) -> MCPToolResult: - del arguments - return MCPToolResult(content=[]) - - def to_params(self) -> dict[str, str]: - return {"name": self.name} - - class ScreenAgent(DictAgent): - tool_catalog = (ScreenTool,) - - encoding = {"mime_type": "image/webp", "quality": 42} - agent = ScreenAgent([AgentStep(content="done", done=True)], screenshot_encoding=encoding) - rfb = object.__new__(RFBClient) - - tools, _ = await agent._build_tools({"screen": rfb}) - - tool = tools["screen"] - assert isinstance(tool, ScreenTool) - assert tool.screenshot_encoding == WebPScreenshotEncoding(quality=42) - - -async def test_agent_opens_every_mcp_capability_by_name() -> None: - capabilities = [ - Capability.mcp(name="database", url="http://database:8000/mcp"), - Capability.mcp(name="search", url="http://search:8000/mcp"), - ] - opened: list[str] = [] - - class Client: - manifest = SimpleNamespace(bindings=capabilities) - - async def open(self, ref: str) -> CapabilityClient: - opened.append(ref) - return cast("CapabilityClient", object()) - - class MultiMCPAgent(DictAgent): - clients = (MCPClient,) - - class LiveRun(_FakeRun): - def __init__(self) -> None: - super().__init__() - self.client = Client() - self.prompt_messages: list[Any] = [] - - await MultiMCPAgent([AgentStep(content="done", done=True)])(cast("Any", LiveRun())) - - assert opened == ["database", "search"] - - -async def test_agent_opens_only_one_non_mcp_capability_per_protocol() -> None: - capabilities = [ - Capability.rfb(name="screen-0", url="rfb://display-0", display=0), - Capability.rfb(name="screen-1", url="rfb://display-1", display=1), - ] - opened: list[str] = [] - - class Client: - manifest = SimpleNamespace(bindings=capabilities) - - async def open(self, ref: str) -> CapabilityClient: - opened.append(ref) - return cast("CapabilityClient", object()) - - class ComputerAgent(DictAgent): - clients = (RFBClient,) - - class LiveRun(_FakeRun): - def __init__(self) -> None: - super().__init__() - self.client = Client() - self.prompt_messages: list[Any] = [] - - await ComputerAgent([AgentStep(content="done", done=True)])(cast("Any", LiveRun())) - - assert opened == ["screen-0"] - - -async def test_mcp_capability_names_do_not_collide_with_protocol_keys() -> None: - capabilities = [ - Capability.mcp(name="ssh/2", url="http://database:8000/mcp"), - Capability.ssh(name="shell", url="ssh://workspace", host_pubkey="key"), - ] - opened: list[str] = [] - - class Client: - manifest = SimpleNamespace(bindings=capabilities) - - async def open(self, ref: str) -> CapabilityClient: - opened.append(ref) - return cast("CapabilityClient", object()) - - class MCPAndShellAgent(DictAgent): - clients = (MCPClient, SSHClient) - - class LiveRun(_FakeRun): - def __init__(self) -> None: - super().__init__() - self.client = Client() - self.prompt_messages: list[Any] = [] - - await MCPAndShellAgent([AgentStep(content="done", done=True)])(cast("Any", LiveRun())) - - assert opened == ["ssh/2", "shell"] - - -@pytest.mark.parametrize( - ("transport", "expected_type"), - [("sse", SSETransport), ("streamable-http", StreamableHttpTransport)], -) -async def test_mcp_client_uses_the_declared_http_transport( - monkeypatch: pytest.MonkeyPatch, - transport: Literal["sse", "streamable-http"], - expected_type: type[SSETransport] | type[StreamableHttpTransport], -) -> None: - transports: list[Any] = [] - - class Client: - def __init__(self, selected: Any, **_kwargs: Any) -> None: - transports.append(selected) - - async def __aenter__(self) -> Client: - return self - - async def __aexit__(self, *_exc: object) -> None: - return None - - monkeypatch.setattr(fastmcp, "Client", Client) - capability = Capability.mcp( - url="https://tools.example/events", - transport=transport, - ) - - client = await MCPClient.connect(capability) - await client.close() - - assert isinstance(transports[0], expected_type) - - -async def test_mcp_client_propagates_active_trace_context() -> None: - calls: list[dict[str, Any]] = [] - - class Client: - async def call_tool_mcp(self, **kwargs: Any) -> mcp_types.CallToolResult: - calls.append(kwargs) - return mcp_types.CallToolResult(content=[]) - - capability = Capability.mcp(url="https://tools.example/mcp") - client = MCPClient(capability, cast("Any", Client()), AsyncExitStack()) - - with set_trace_context("child-trace"): - await client.call_tool("verify", {}) - - assert calls == [ - { - "name": "verify", - "arguments": {}, - "meta": {"hud/trace-id": "child-trace"}, - } - ] - - -def test_mcp_server_reads_propagated_trace_context(monkeypatch: pytest.MonkeyPatch) -> None: - request_context = SimpleNamespace( - meta=SimpleNamespace(model_extra={"hud/trace-id": "child-trace"}) - ) - monkeypatch.setattr( - "fastmcp.server.dependencies.get_context", - lambda: SimpleNamespace(request_context=request_context), - ) - - assert get_mcp_trace_id() == "child-trace" - - -def test_mcp_server_reads_propagated_trace_header(monkeypatch: pytest.MonkeyPatch) -> None: - request_context = SimpleNamespace(meta=SimpleNamespace(model_extra={})) - monkeypatch.setattr( - "fastmcp.server.dependencies.get_context", - lambda: SimpleNamespace(request_context=request_context), - ) - monkeypatch.setattr( - "fastmcp.server.dependencies.get_http_headers", - lambda: { - "trace-id": "child-trace", - }, - ) - - assert get_mcp_trace_id() == "child-trace" - - -async def test_multiple_mcp_capabilities_qualify_tool_names() -> None: - class Client(MCPClient): - def __init__(self, *names: str) -> None: - self.tools = [ - mcp_types.Tool( - name=name, - description=f"Run {name}", - inputSchema={"type": "object", "properties": {}}, - ) - for name in names - ] - self.calls: list[str] = [] - - async def list_tools(self) -> list[mcp_types.Tool]: - return self.tools - - async def call_tool(self, name: str, arguments: dict[str, Any]) -> MCPToolResult: - self.calls.append(name) - return MCPToolResult(content=[]) - - class MultiMCPAgent(DictAgent): - tool_catalog = (OpenAIMCPProxyTool,) - - database = Client("lookup", "write") - search = Client("lookup", "find") - tools, params = await MultiMCPAgent([])._build_tools({"database": database, "search": search}) - - assert list(tools) == [ - "database__lookup", - "database__write", - "search__lookup", - "search__find", - ] - assert [param["name"] for param in params] == list(tools) - - await tools["database__lookup"].execute({}) - assert database.calls == ["lookup"] - assert search.calls == [] - - single, _ = await MultiMCPAgent([])._build_tools({"database": Client("lookup")}) - assert list(single) == ["lookup"] - - -async def test_multiple_mcp_capabilities_reject_qualified_name_collisions() -> None: - class Client(MCPClient): - def __init__(self, name: str) -> None: - self.tool = mcp_types.Tool( - name=name, - description=f"Run {name}", - inputSchema={"type": "object", "properties": {}}, - ) - - async def list_tools(self) -> list[mcp_types.Tool]: - return [self.tool] - - async def call_tool(self, name: str, arguments: dict[str, Any]) -> MCPToolResult: - raise AssertionError("colliding tools must not be callable") - - class MultiMCPAgent(DictAgent): - tool_catalog = (OpenAIMCPProxyTool,) - - with pytest.raises(ValueError, match=r"MCP tool name collision.*a__b__c"): - await MultiMCPAgent([])._build_tools({"a": Client("b__c"), "a__b": Client("c")}) - - -async def test_qualified_mcp_names_are_valid_provider_tool_names() -> None: - class Client(MCPClient): - def __init__(self, name: str) -> None: - self.tool = mcp_types.Tool( - name=name, - description=f"Run {name}", - inputSchema={"type": "object", "properties": {}}, - ) - - async def list_tools(self) -> list[mcp_types.Tool]: - return [self.tool] - - async def call_tool(self, name: str, arguments: dict[str, Any]) -> MCPToolResult: - return MCPToolResult(content=[]) - - class MultiMCPAgent(DictAgent): - tool_catalog = (OpenAIMCPProxyTool,) - - long_name = "lookup_" * 10 - tools, params = await MultiMCPAgent([])._build_tools( - { - "company.tools": Client(long_name + "a"), - "company-tools": Client(long_name + "b"), - } - ) - - assert len(tools) == 2 - assert set(tools) == {param["name"] for param in params} - assert all( - len(name) <= 64 and name.replace("_", "").replace("-", "").isalnum() for name in tools - ) - - -# ─── initial messages / user text formatting ────────────────────────── - - -def test_initial_messages_formats_each_turn() -> None: - agent = DictAgent([]) - turn = mcp_types.PromptMessage( - role="user", content=mcp_types.TextContent(type="text", text="a") - ) - assert agent._initial_messages([turn]) == [{"role": "user", "content": "a"}] - assert agent._format_user_text("hey") == {"role": "user", "content": "hey"} - - -# ─── dispatch + loop ────────────────────────────────────────────────── - - -async def test_dispatch_unknown_tool_returns_error_result() -> None: - agent = DictAgent([]) - result = await agent._dispatch_call(MCPToolCall(name="ghost"), RunState()) - assert result.isError is True - - -async def test_dispatch_unparsed_arguments_returns_error_result() -> None: - # A call whose provider arguments never parsed is answered, not executed. - agent = DictAgent([]) - call = MCPToolCall(name="bash", arguments='{"command": "') - result = await agent._dispatch_call(call, RunState()) - assert result.isError is True - content = result.content[0] - assert isinstance(content, mcp_types.TextContent) - assert "not executed" in content.text - assert '{"command": "' in content.text # the raw prefix re-anchors the model - - -async def test_dispatch_marks_exhausted_ssh_reconnect_as_infrastructure_error() -> None: - agent = DictAgent([]) - tool = SimpleNamespace( - execute=AsyncMock(side_effect=SSHConnectionError("SSH reconnect failed after 3 attempts")) - ) - state: RunState[_Msg] = RunState(tools={"bash": cast("Any", tool)}) - - result = await agent._dispatch_call(MCPToolCall(name="bash"), state) - - assert result.isError is True - assert isinstance(result, SSHInfrastructureErrorResult) - - -async def test_dispatch_reraises_tool_timeout_before_deadline() -> None: - ssh = SimpleNamespace(run=AsyncMock(side_effect=TimeoutError("remote timed out"))) - tool = ClaudeBashTool( - spec=ClaudeBashTool.default_spec("claude"), - client=cast("Any", ssh), - ) - agent = DictAgent([], tool_timeout_seconds=120) - state: RunState[_Msg] = RunState(tools={"bash": tool}) - - with pytest.raises(TimeoutError, match="remote timed out"): - await agent._dispatch_call( - MCPToolCall(name="bash", arguments={"command": "sleep forever"}), - state, - ) - - -async def test_claude_bash_timeout_terminates_process_and_continues_loop( - monkeypatch: pytest.MonkeyPatch, -) -> None: - started = asyncio.Event() - - async def wait(*, check: bool, timeout: None) -> None: # noqa: ASYNC109 - del check, timeout - started.set() - await asyncio.Event().wait() - - process = SimpleNamespace( - wait=AsyncMock(side_effect=wait), - terminate=Mock(), - close=Mock(side_effect=AssertionError("graceful termination should succeed")), - wait_closed=AsyncMock(), - ) - connection = SimpleNamespace( - is_closed=Mock(return_value=False), - create_process=AsyncMock(return_value=process), - ) - ssh = SSHClient( - Capability.ssh(url="ssh://workspace", host_pubkey="key"), - cast("Any", connection), - ) - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude"), client=ssh) - agent = ClaudeAgent( - ClaudeConfig( - model="claude-test", - model_client=cast("Any", object()), - tool_timeout_seconds=0.01, - ) - ) - responses = AsyncMock( - side_effect=[ - AgentStep( - content="", - done=False, - tool_calls=[ - MCPToolCall(id="tool-1", name="bash", arguments={"command": "sleep forever"}) - ], - ), - AgentStep(content="recovered", done=True), - ] - ) - monkeypatch.setattr(agent, "get_response", responses) - state = RunState(messages=[], tools={"bash": tool}) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, state) - - assert started.is_set() - process.terminate.assert_called_once_with() - process.wait_closed.assert_awaited_once_with() - assert responses.await_count == 2 - assert run.trace.status == "completed" - assert run.trace.content == "recovered" - tool_result = cast("list[Any]", state.messages[0]["content"])[0] - error_block = cast("list[Any]", tool_result["content"])[0] - assert error_block["text"] == ( - "Error: bash timed out after 0.01s; retry with a shorter command" - ) - - -async def test_composite_editor_uses_one_tool_deadline( - monkeypatch: pytest.MonkeyPatch, -) -> None: - deadline = Mock(wraps=asyncio.timeout) - monkeypatch.setattr(asyncio, "timeout", deadline) - ssh = SimpleNamespace( - read_text=AsyncMock(return_value="old"), - write_text=AsyncMock(), - ) - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), - client=cast("Any", ssh), - ) - agent = DictAgent([], tool_timeout_seconds=120) - state: RunState[_Msg] = RunState(tools={tool.provider_name: tool}) - - result = await agent._dispatch_call( - MCPToolCall( - name=tool.provider_name, - arguments={ - "command": "str_replace", - "path": "/file.txt", - "old_str": "old", - "new_str": "new", - }, - ), - state, - ) - - assert result.isError is False - assert result_text(result) == "wrote 3 bytes to /file.txt" - deadline.assert_called_once_with(120) - ssh.read_text.assert_awaited_once_with("/file.txt") - ssh.write_text.assert_awaited_once_with("/file.txt", "new") - - -async def test_loop_finishes_on_done_response() -> None: - agent = DictAgent([AgentStep(content="final answer", done=True)]) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.status == "completed" - assert run.trace.content == "final answer" - assert run.trace.is_error is False - assert run.trace.stop_reason == "done" - assert run.trace.is_truncated is False - # The agent turn was recorded directly, with loop-stamped fallbacks. - (step,) = run.trace.steps - assert isinstance(step, AgentStep) - assert step.source == "agent" - assert step.content == "final answer" - assert step.model == "test-model" - assert step.started_at is not None - - -class _DegenerateDictAgent(DictAgent): - """DictAgent whose scripted turns may be ``DegenerateTurnError`` instances.""" - - def __init__(self, turns: list[AgentStep | DegenerateTurnError], **config: Any) -> None: - super().__init__(cast("list[AgentStep]", turns), **config) - - async def get_response( - self, state: RunState[_Msg], *, system_prompt: Any = None, citations_enabled: bool = False - ) -> AgentStep: - turn = await super().get_response( - state, system_prompt=system_prompt, citations_enabled=citations_enabled - ) - if isinstance(turn, DegenerateTurnError): - raise turn - return turn - - -async def test_loop_discards_degenerate_turn_and_resamples() -> None: - agent = _DegenerateDictAgent( - [ - DegenerateTurnError("empty shell_call"), - AgentStep(content="recovered", done=True), - ] - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - # The degenerate turn is discarded (consuming a step) and never recorded. - assert run.trace.status == "completed" - assert run.trace.content == "recovered" - assert [step.source for step in run.trace.steps] == ["agent"] - - -async def test_loop_fails_when_degenerate_turn_exhausts_steps() -> None: - agent = _DegenerateDictAgent([DegenerateTurnError("empty shell_call")] * 2) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 2 - await agent._loop(run, RunState()) - - assert run.trace.status == "error" - assert run.trace.error == "empty shell_call" - - -async def test_loop_dispatches_tool_calls_then_finishes() -> None: - agent = DictAgent( - [ - AgentStep(content="", done=False, tool_calls=[MCPToolCall(name="ghost")]), - AgentStep(content="done now", done=True), - ] - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.content == "done now" - assert [step.source for step in run.trace.steps] == ["agent", "tool", "agent"] - # the (unknown) tool call produced an observed tool step in the trajectory - tool_step = run.trace.steps[1] - assert isinstance(tool_step, ToolStep) - assert tool_step.call is not None - assert tool_step.call.name == "ghost" - assert tool_step.result is not None - assert tool_step.result.isError is True # unknown tool → error result - - -async def test_loop_resets_infrastructure_error_count_after_other_result( - monkeypatch: pytest.MonkeyPatch, -) -> None: - turns = [ - AgentStep(content="", done=False, tool_calls=[MCPToolCall(name="bash")]) for _ in range(5) - ] - agent = DictAgent(turns) - infrastructure_error = SSHInfrastructureErrorResult(content=[], isError=True) - ordinary_error = MCPToolResult(content=[], isError=True) - dispatch = AsyncMock( - side_effect=[ - infrastructure_error, - ordinary_error, - infrastructure_error, - infrastructure_error, - infrastructure_error, - ] - ) - monkeypatch.setattr(agent, "_dispatch_call", dispatch) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 10 - await agent._loop(run, RunState()) - - assert dispatch.await_count == 5 - assert run.trace.status == "error" - assert run.trace.stop_reason is None - assert run.trace.error == ("SSH tool failure limit reached after 3 consecutive errors") - - -async def test_loop_max_steps_is_normal_termination() -> None: - # Always returns a tool call → never "done" → hits max_steps. Exhausting the - # configured budget is a stop reason, not an agent error (the platform must - # not paint the rollout or its last tool call as failed). - never_done = [ - AgentStep(content="", done=False, tool_calls=[MCPToolCall(name="ghost")]) for _ in range(5) - ] - agent = DictAgent(never_done) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 2 - await agent._loop(run, RunState()) - - assert run.trace.is_error is False - assert run.trace.status == "completed" - assert run.trace.stop_reason == "max_steps" - assert run.trace.is_truncated is True - # No synthetic error step — the trajectory ends on the real agent/tool steps. - assert all(step.source != "system" for step in run.trace.steps) - - -async def test_loop_marks_length_finish_as_truncated() -> None: - # A final turn cut off at the provider token cap (e.g. mid-tool-call) ends the - # rollout normally but is a truncation, not a natural finish — across every - # provider's finish-reason vocabulary. - for finish_reason in ("length", "max_output_tokens", "max_tokens", "MAX_TOKENS"): - agent = DictAgent([AgentStep(content="partial", done=True, finish_reason=finish_reason)]) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.status == "completed" - assert run.trace.stop_reason == "length" - assert run.trace.is_truncated is True - - -async def test_loop_answers_malformed_call_by_default() -> None: - # Default "retry": the malformed call gets an error result and the loop continues. - agent = DictAgent( - [ - AgentStep( - content="", - done=False, - tool_calls=[MCPToolCall(name="bash", arguments='{"command": "')], - ), - AgentStep(content="recovered", done=True), - ] - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.content == "recovered" - tool_step = run.trace.steps[1] - assert isinstance(tool_step, ToolStep) - assert tool_step.result is not None - assert tool_step.result.isError is True - - -async def test_loop_stops_on_malformed_call_when_configured() -> None: - # The rollout ends at the malformed-call turn with nothing dispatched, and the - # fired condition is the recorded stop reason. - agent = DictAgent( - [ - AgentStep( - content="", - done=False, - tool_calls=[MCPToolCall(name="bash", arguments='{"command": "')], - ) - ], - stop_on={"malformed_tool_call"}, - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.status == "completed" - assert run.trace.stop_reason == "malformed_tool_call" - assert run.trace.is_truncated is True - assert all(not isinstance(step, ToolStep) for step in run.trace.steps) - - -async def test_loop_stops_on_length_when_configured() -> None: - # A token-capped turn ends the rollout even when its tool calls parsed. - agent = DictAgent( - [ - AgentStep( - content="", - done=False, - finish_reason="length", - tool_calls=[MCPToolCall(name="bash", arguments={"command": "ls"})], - ) - ], - stop_on={"length", "malformed_tool_call"}, - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.stop_reason == "length" - assert run.trace.is_truncated is True - assert all(not isinstance(step, ToolStep) for step in run.trace.steps) - - -@pytest.mark.parametrize("auto_respond", [False, True]) -@pytest.mark.parametrize("reason", [None, "malformed_tool_call"]) -async def test_provider_error_stops_without_auto_response( - auto_respond: bool, reason: Literal["malformed_tool_call"] | None -) -> None: - agent = DictAgent( - [AgentStep(error="provider failure", stop_reason=reason)], - auto_respond=auto_respond, - max_steps=3, - ) - run = cast("Run", _FakeRun()) - await agent._loop(run, RunState()) - assert run.trace.status == "error" - assert run.trace.stop_reason == reason - assert len(run.trace.steps) == 1 - assert run.trace.steps[0].error == "provider failure" diff --git a/hud/agents/tests/test_trace.py b/hud/agents/tests/test_trace.py deleted file mode 100644 index 43e5246ea..000000000 --- a/hud/agents/tests/test_trace.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Tool-agent family layer tests: the flat ``Step`` subclasses.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any -from unittest.mock import patch - -from mcp import types as mcp_types - -from hud.agents.types import ( - AgentStep, - Citation, - Sample, - SubagentStep, - ToolStep, - Usage, -) -from hud.telemetry.context import set_trace_context -from hud.types import MCPToolCall, MCPToolResult, Step, Trace - - -def test_agent_step_raw_serializes_safely(): - """AgentStep captures raw provider payloads in JSON-safe dumps.""" - - @dataclass - class RawResponse: - raw_data: str - - step = AgentStep(raw=RawResponse(raw_data="value")) - data = step.model_dump(mode="json") - - assert step.raw == RawResponse(raw_data="value") - assert data["raw"] == {"raw_data": "value"} - - -def test_agent_step_dump_uses_canonical_field_names(): - """AgentStep dumps use the normalized SDK field names, flat on the step.""" - step = AgentStep(raw={"raw_data": "value"}) - step.reasoning = "because" - step.citations = [Citation(source="https://example.com")] - - data = step.model_dump(exclude_none=True, mode="json") - - assert data["source"] == "agent" - assert data["reasoning"] == "because" - assert data["citations"] == [{"type": "citation", "text": "", "source": "https://example.com"}] - assert data["raw"] == {"raw_data": "value"} - - -def test_agent_step_citations_roundtrip(): - """Citations survive serialize/deserialize as typed Citations.""" - cit = Citation(type="url_citation", source="https://example.com", title="Example") - step = AgentStep(content="hello", citations=[cit]) - data = step.model_dump(mode="json") - restored = AgentStep(**data) - assert restored.citations == [cit] - assert restored.citations[0].source == "https://example.com" - - -def test_citation_defaults(): - """Citation defaults to a generic, empty-span citation.""" - c = Citation() - assert c.type == "citation" - assert c.text == "" - assert c.start_index is None - - -def test_final_query_reads_the_final_agent_turns_citations(): - """The chat-surface read: trace.final() with family vocabulary at the call site.""" - - def reply_citations(step: Step) -> list[Citation] | None: - return step.citations if isinstance(step, AgentStep) else None - - cited = Citation(source="https://example.com") - trace = Trace() - trace.record(AgentStep(content="draft", citations=[cited])) - trace.record(ToolStep()) - assert trace.final(reply_citations) == [cited] - - trace.record(AgentStep(content="final", done=True)) - assert trace.final(reply_citations) == [] - - -def test_agent_step_timing_and_error_live_on_the_skeleton(): - """The turn uses the skeleton's channels: record() stamps ended_at, error is error.""" - trace = Trace() - step = AgentStep(error="rate limited", done=True) - trace.record(step) - - assert step.ended_at is not None - assert trace.error == "rate limited" - - -def test_trace_dump_keeps_family_payloads(): - """Family subclass fields survive a whole-trace dump (SerializeAsAny).""" - trace = Trace() - trace.record( - AgentStep( - content="answer", - usage=Usage(prompt_tokens=10, completion_tokens=3), - sample=Sample(prompt_token_ids=[1], output_token_ids=[2], output_logprobs=[-0.5]), - ) - ) - trace.record(SubagentStep(subagent=Trace(content="inner"))) - - dump = trace.model_dump(mode="json", exclude_none=True) - assert dump["steps"][0]["content"] == "answer" - assert dump["steps"][0]["usage"]["prompt_tokens"] == 10 - assert dump["steps"][0]["sample"]["output_token_ids"] == [2] - assert dump["steps"][1]["subagent"]["content"] == "inner" - - -def test_step_emit_carries_tool_call_and_result(): - captured: list[dict[str, Any]] = [] - call = MCPToolCall(name="bash", arguments={"command": "ls"}) - result = MCPToolResult( - content=[mcp_types.TextContent(type="text", text="file.txt")], - isError=False, - ) - - with ( - patch("hud.types.queue_span", side_effect=captured.append), - set_trace_context("run-1"), - ): - ToolStep(call=call, result=result).emit() - - (span,) = captured - assert span["name"] == "step.tool" - assert span["attributes"]["hud.schema"] == "hud.step.v1" - payload = span["attributes"]["hud.payload"] - assert payload["call"]["name"] == "bash" - assert payload["result"]["content"][0]["text"] == "file.txt" - assert payload["result"]["isError"] is False diff --git a/hud/agents/tool_agent.py b/hud/agents/tool_agent.py index 198776e0a..4ffb6d680 100644 --- a/hud/agents/tool_agent.py +++ b/hud/agents/tool_agent.py @@ -17,7 +17,7 @@ from hud.agents.tools.rfb import RFBTool from hud.agents.tools.ssh import SSHInfrastructureErrorResult, SSHTool from hud.agents.types import AgentStep, ToolStep -from hud.capabilities import MCPClient, RFBClient +from hud.capabilities import MCPClient from hud.capabilities.ssh import SSHConnectionError from hud.types import MCPToolCall, MCPToolResult, Step, StopCondition from hud.utils.time import now_iso @@ -147,15 +147,7 @@ async def _build_tools( tools[tool.provider_name] = tool params.append(tool.to_params()) else: - if issubclass(tool_cls, RFBTool): - assert isinstance(client, RFBClient) - tool = tool_cls( - spec=spec, - client=client, - screenshot_encoding=self.config.screenshot_encoding, - ) - else: - tool = tool_cls(spec=spec, client=client) + tool = tool_cls(spec=spec, client=client, **self._tool_options(tool_cls)) tools[tool.provider_name] = tool params.append(tool.to_params()) @@ -167,6 +159,12 @@ async def _build_tools( return tools, params + def _tool_options(self, tool_cls: type[AgentTool[Any]]) -> dict[str, Any]: + """Constructor options this agent's config sets on a tool beyond its spec and client.""" + if issubclass(tool_cls, RFBTool): + return {"screenshot_encoding": self.config.screenshot_encoding} + return {} + async def _loop( self, run: Run, diff --git a/hud/agents/types.py b/hud/agents/types.py index 5be380bf8..69b8cdde3 100644 --- a/hud/agents/types.py +++ b/hud/agents/types.py @@ -252,25 +252,6 @@ class CodexCLIConfig(AgentConfig): reasoning_effort: CodexReasoningEffort | None = None -# ----------------------------------------------------------------------------- -# Browser Use -# ----------------------------------------------------------------------------- - - -class BrowserUseConfig(AgentConfig): - """Configuration for BrowserUseAgent. - - Lives here (not in the agent module) so it can be imported and serialized - without the optional ``browser-use`` dependency installed. - """ - - model_name: str = "Browser Use" - model: str = Field(default="claude-sonnet-4-5", validation_alias=_model_alias) - api_key: str | None = Field(default=None, exclude=True) - base_url: str | None = Field(default=None, exclude=True) - max_steps: int = 25 - - # ----------------------------------------------------------------------------- # Trajectory (tool-agent family step payloads) # ----------------------------------------------------------------------------- diff --git a/hud/capabilities/cdp.py b/hud/capabilities/cdp.py index 592e17b60..1a665cdfe 100644 --- a/hud/capabilities/cdp.py +++ b/hud/capabilities/cdp.py @@ -113,9 +113,12 @@ async def send(self, method: str, params: dict[str, Any] | None = None) -> dict[ self._pending[msg_id] = future await self._ws.send(json.dumps({"id": msg_id, "method": method, "params": params or {}})) try: - return await future + reply = await future finally: self._pending.pop(msg_id, None) + if "error" in reply: + raise CDPError(method, reply["error"]) + return reply.get("result", {}) async def _read_loop(self) -> None: try: @@ -125,10 +128,7 @@ async def _read_loop(self) -> None: future = self._pending.get(msg_id) if msg_id is not None else None if future is None or future.done(): continue # protocol event (no waiter) — ignored for now - if "error" in msg: - future.set_exception(CDPError(str(msg.get("method", "")), msg["error"])) - else: - future.set_result(msg.get("result", {})) + future.set_result(msg) except (ConnectionClosed, OSError) as exc: LOGGER.debug("CDP read loop ended: %s", exc) finally: diff --git a/hud/capabilities/tests/test_rfb.py b/hud/capabilities/tests/test_rfb.py deleted file mode 100644 index 72ff77017..000000000 --- a/hud/capabilities/tests/test_rfb.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any -from unittest.mock import ANY, AsyncMock, Mock, patch - -import mcp.types as mcp_types -from PIL import Image - -from hud.agents.tools.rfb import RFBTool -from hud.capabilities.rfb import RFBClient, WebPScreenshotEncoding - - -class RecordingRFBTool(RFBTool): - name = "rfb-test" - client: Any - - def __init__(self) -> None: - self.screenshot_encoding = WebPScreenshotEncoding(quality=42) - self.client = SimpleNamespace( - screenshot_png=AsyncMock(return_value=(b"webp", "image/webp")), - ) - - async def execute(self, arguments: dict[str, Any]) -> Any: - del arguments - raise NotImplementedError - - def to_params(self) -> Any: - raise NotImplementedError - - -async def test_screenshot_uses_requested_webp_mime_type() -> None: - client = object.__new__(RFBClient) - object.__setattr__( - client, - "_conn", - SimpleNamespace( - screenshot=AsyncMock(return_value=object()), - ), - ) - - save = Mock(side_effect=lambda buffer, **_kwargs: buffer.write(b"webp")) - with patch("hud.capabilities.rfb.Image.fromarray", return_value=SimpleNamespace(save=save)): - data, mime_type = await client.screenshot_png(WebPScreenshotEncoding(quality=42)) - - assert mime_type == "image/webp" - assert data == b"webp" - save.assert_called_once_with(ANY, format="WEBP", quality=42) - - -async def test_screenshot_uses_requested_png_mime_type() -> None: - client = object.__new__(RFBClient) - object.__setattr__( - client, - "_conn", - SimpleNamespace( - screenshot=AsyncMock(return_value=object()), - ), - ) - - with patch("hud.capabilities.rfb.Image.fromarray", return_value=Image.new("RGB", (8, 8))): - data, mime_type = await client.screenshot_png("image/png") - - assert mime_type == "image/png" - assert data.startswith(b"\x89PNG") - - -async def test_screenshot_reports_encoded_mime_type() -> None: - tool = RecordingRFBTool() - - result = await tool.screenshot() - - image = result.content[0] - assert isinstance(image, mcp_types.ImageContent) - assert image.mimeType == "image/webp" - tool.client.screenshot_png.assert_awaited_once_with(tool.screenshot_encoding) diff --git a/hud/capabilities/tests/test_ssh.py b/hud/capabilities/tests/test_ssh.py deleted file mode 100644 index 0c619dc28..000000000 --- a/hud/capabilities/tests/test_ssh.py +++ /dev/null @@ -1,437 +0,0 @@ -from __future__ import annotations - -import asyncio -from typing import TYPE_CHECKING, Any, cast -from unittest.mock import AsyncMock - -import asyncssh -import pytest - -from hud.capabilities.base import Capability -from hud.capabilities.connection import Connection -from hud.capabilities.ssh import PROCESS_CONNECTIONS_REQUEST, SSHClient, SSHConnectionError - -if TYPE_CHECKING: - from collections.abc import Callable - - -class _Completed: - stdout = "ok" - stderr = "" - exit_status = 0 - returncode: int | None = 0 - - -class _Process: - def __init__( - self, - *, - completed: _Completed | None = None, - wait_error: BaseException | None = None, - block: bool = False, - ) -> None: - self.completed = completed or _Completed() - self.wait_error = wait_error - self.block = block - self.on_wait: Callable[[], None] | None = None - self.started = asyncio.Event() - self.closed = False - self.terminated = False - self.waited_closed = False - - async def wait(self, *, check: bool, **kwargs: Any) -> _Completed: - assert kwargs == {"timeout": None} - del check - self.started.set() - if self.on_wait is not None: - self.on_wait() - if self.block: - await asyncio.Event().wait() - if self.wait_error is not None: - raise self.wait_error - return self.completed - - def close(self) -> None: - self.closed = True - - def terminate(self) -> None: - self.terminated = True - - async def wait_closed(self) -> None: - self.waited_closed = True - - -class _Connection: - def __init__( - self, - *, - closed: bool = False, - run_error: Exception | None = None, - process: _Process | None = None, - stall_open: bool = False, - open_error: Exception | None = None, - ) -> None: - self.closed = closed - self.run_error = run_error - self.process = process or _Process() - self.stall_open = stall_open - self.open_error = open_error - self.open_cancelled = False - self.commands: list[str] = [] - self.process_kwargs: dict[str, Any] = {} - - def is_closed(self) -> bool: - return self.closed - - def close(self) -> None: - self.closed = True - - async def wait_closed(self) -> None: - pass - - async def run(self, command: str, **kwargs: Any) -> _Completed: - del kwargs - self.commands.append(command) - if self.run_error is not None: - if isinstance(self.run_error, asyncssh.ConnectionLost): - self.closed = True - raise self.run_error - return _Completed() - - async def create_process(self, *args: object, **kwargs: Any) -> _Process: - self.process_kwargs = kwargs - self.commands.append(str(args[0])) - if self.stall_open: - try: - await asyncio.Event().wait() - except asyncio.CancelledError: - self.open_cancelled = True - raise - if self.open_error is not None: - raise self.open_error - if self.run_error is not None: - if isinstance(self.run_error, asyncssh.ConnectionLost): - self.closed = True - raise self.run_error - return self.process - - -def _capability() -> Capability: - return Capability( - name="shell", - protocol="ssh/2", - url="ssh://workspace.example:2222", - params={"user": "agent", "client_key_path": "/tmp/key"}, - ) - - -def _client(connection: object) -> SSHClient: - return SSHClient(_capability(), cast("asyncssh.SSHClientConnection", connection)) - - -async def test_connect_keeps_tunneled_connection_active(monkeypatch: pytest.MonkeyPatch) -> None: - connection = cast("asyncssh.SSHClientConnection", object()) - connect = AsyncMock(return_value=connection) - monkeypatch.setattr(asyncssh, "connect", connect) - - client = await SSHClient.connect( - Capability(name="shell", protocol="ssh/2", url="ssh://sandbox.example:8765") - ) - - assert client.conn is connection - connect.assert_awaited_once_with( - host="sandbox.example", - port=8765, - username="agent", - client_keys=None, - known_hosts=None, - errors="replace", - keepalive_interval=15, - keepalive_count_max=4, - ) - - -async def test_connect_classifies_a_lost_handshake_as_retryable( - monkeypatch: pytest.MonkeyPatch, -) -> None: - connect = AsyncMock(side_effect=asyncssh.ConnectionLost("dropped")) - monkeypatch.setattr(SSHClient, "_connect", connect) - - with pytest.raises(SSHConnectionError, match="failed during handshake"): - await SSHClient.connect(_capability()) - - -async def test_run_does_not_replay_a_command_lost_in_flight( - monkeypatch: pytest.MonkeyPatch, -) -> None: - dropped = _Connection(run_error=asyncssh.ConnectionLost("dropped")) - replacement = _Connection() - client = _client(dropped) - reconnect = AsyncMock(return_value=replacement) - monkeypatch.setattr(client, "_connect", reconnect) - - with pytest.raises(SSHConnectionError, match="lost during operation"): - await client.run("apply-side-effect") - - reconnect.assert_not_awaited() - assert dropped.commands == ["apply-side-effect"] - - await client.run("next-command") - reconnect.assert_awaited_once_with(client.capability) - assert replacement.commands == ["next-command"] - - -async def test_create_process_reconnects_before_opening_channel( - monkeypatch: pytest.MonkeyPatch, -) -> None: - dropped = _Connection(closed=True) - replacement = _Connection() - client = _client(dropped) - reconnect = AsyncMock(return_value=replacement) - monkeypatch.setattr(client, "_connect", reconnect) - - process = await client.create_process("bridge") - - assert process is replacement.process - reconnect.assert_awaited_once_with(client.capability) - assert dropped.commands == [] - assert replacement.commands == ["bridge"] - - -async def test_create_process_sends_only_connection_names() -> None: - transport = _Connection() - capability = _capability() - capability.params["process_connections"] = True - client = SSHClient(capability, cast("Any", transport)) - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-secret"}, - ) - - await client.create_process("agent", connections=(connection,)) - - assert transport.process_kwargs["env"] == {PROCESS_CONNECTIONS_REQUEST: '["inference"]'} - assert "scoped-secret" not in str(transport.process_kwargs) - - -async def test_create_process_rejects_connections_the_server_did_not_advertise() -> None: - client = _client(_Connection()) - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-secret"}, - ) - - with pytest.raises(ValueError, match="does not support"): - await client.create_process("agent", connections=(connection,)) - - -async def test_create_process_rejects_connections_for_another_capability() -> None: - transport = _Connection() - capability = _capability() - capability.params["process_connections"] = True - client = SSHClient(capability, cast("Any", transport)) - connection = Connection( - name="inference", - capability="other-workspace", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-secret"}, - ) - - with pytest.raises(ValueError, match="do not belong"): - await client.create_process("agent", connections=(connection,)) - - -async def test_create_process_preserves_reconnect_failure( - monkeypatch: pytest.MonkeyPatch, -) -> None: - client = _client(_Connection(closed=True)) - reconnect = AsyncMock(side_effect=OSError("unreachable")) - monkeypatch.setattr(client, "_connect", reconnect) - monkeypatch.setattr("hud.capabilities.ssh.asyncio.sleep", AsyncMock()) - - with pytest.raises(SSHConnectionError, match="reconnect failed after 3 attempts"): - await client.create_process("bridge") - - -async def test_run_classifies_rejected_session_as_connection_error() -> None: - connection = _Connection( - open_error=asyncssh.ChannelOpenError(asyncssh.OPEN_RESOURCE_SHORTAGE, "busy") - ) - client = _client(connection) - - with pytest.raises(SSHConnectionError, match="rejected the session"): - await client.run("echo never", timeout=1) - - assert connection.closed is False - - -async def test_run_timeout_includes_opening_the_process() -> None: - connection = _Connection(stall_open=True) - client = _client(connection) - - with pytest.raises(TimeoutError): - await client.run("echo never", timeout=0.01) - - assert connection.open_cancelled is True - - -async def test_run_timeout_includes_reconnecting( - monkeypatch: pytest.MonkeyPatch, -) -> None: - connection = _Connection(closed=True) - client = _client(connection) - cancelled = False - - async def reconnect(capability: Capability) -> asyncssh.SSHClientConnection: - nonlocal cancelled - assert capability is client.capability - try: - await asyncio.Event().wait() - except asyncio.CancelledError: - cancelled = True - raise - raise AssertionError("reconnect unexpectedly completed") - - monkeypatch.setattr(client, "_connect", reconnect) - - with pytest.raises(TimeoutError): - await client.run("echo never", timeout=0.01) - - assert cancelled is True - - -async def test_run_preserves_timeout_when_the_connection_closes() -> None: - process = _Process(wait_error=TimeoutError()) - connection = _Connection(process=process) - process.on_wait = lambda: setattr(connection, "closed", True) - client = _client(connection) - - with pytest.raises(TimeoutError): - await client.run("echo never", timeout=1) - - -@pytest.mark.parametrize("command_timeout", [None, 300]) -async def test_run_cancellation_terminates_the_remote_process( - command_timeout: float | None, -) -> None: - connection = _Connection(process=_Process(block=True)) - client = _client(connection) - run = asyncio.create_task(client.run("echo never", timeout=command_timeout)) - await connection.process.started.wait() - - run.cancel() - - with pytest.raises(asyncio.CancelledError): - await run - assert connection.process.terminated is True - assert connection.process.closed is False - assert connection.process.waited_closed is True - - -async def test_run_rejects_a_completed_process_without_a_returncode() -> None: - completed = _Completed() - completed.returncode = None - connection = _Connection(process=_Process(completed=completed)) - client = _client(connection) - - with pytest.raises(SSHConnectionError, match="without an exit status"): - await client.run("echo incomplete", timeout=1) - - assert connection.closed is True - - -async def test_windows_write_uses_one_timeout_budget( - monkeypatch: pytest.MonkeyPatch, -) -> None: - client = SSHClient( - Capability( - name="shell", - protocol="ssh/2", - url="ssh://workspace.example:2222", - params={"shell": "powershell"}, - ), - cast("asyncssh.SSHClientConnection", _Connection()), - ) - timeouts: list[float] = [] - - async def run(*args: object, **kwargs: Any) -> _Completed: - del args - timeout = kwargs["timeout"] - assert isinstance(timeout, float) - timeouts.append(timeout) - await asyncio.sleep(0.01) - return _Completed() - - monkeypatch.setattr(client, "run", run) - - await client.write_text("C:\\file.txt", "x" * 7000, timeout_s=1) - - assert len(timeouts) == 3 - assert timeouts[0] > timeouts[1] > timeouts[2] - - -async def test_posix_empty_write_supplies_eof(monkeypatch: pytest.MonkeyPatch) -> None: - client = _client(_Connection()) - run = AsyncMock(return_value=_Completed()) - monkeypatch.setattr(client, "run", run) - - await client.write_text("empty.txt", "") - - run.assert_awaited_once_with( - "cat > empty.txt", - check=True, - timeout=None, - input="", - stdin=asyncssh.DEVNULL, - ) - - -async def test_close_during_reconnect_discards_the_replacement( - monkeypatch: pytest.MonkeyPatch, -) -> None: - dropped = _Connection(closed=True) - replacement = _Connection() - reconnect_started = asyncio.Event() - allow_reconnect = asyncio.Event() - - async def reconnect(capability: Capability) -> asyncssh.SSHClientConnection: - assert capability is client.capability - reconnect_started.set() - await allow_reconnect.wait() - return cast("asyncssh.SSHClientConnection", replacement) - - client = _client(dropped) - monkeypatch.setattr(client, "_connect", reconnect) - - run = asyncio.create_task(client.run("echo never")) - await reconnect_started.wait() - close = asyncio.create_task(client.close()) - await asyncio.sleep(0) - allow_reconnect.set() - - with pytest.raises(SSHConnectionError, match="client is closed"): - await run - await close - - assert replacement.closed is True - assert replacement.commands == [] - - -async def test_reconnect_exhaustion_raises_connection_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - client = _client(_Connection(closed=True)) - reconnect = AsyncMock(side_effect=OSError("unreachable")) - sleep = AsyncMock() - monkeypatch.setattr(client, "_connect", reconnect) - monkeypatch.setattr("hud.capabilities.ssh.asyncio.sleep", sleep) - - with pytest.raises(SSHConnectionError, match="failed after 3 attempts"): - await client.run("echo never") - - assert reconnect.await_count == 3 - assert [call.args for call in sleep.await_args_list] == [(0.25,), (0.5,)] diff --git a/hud/cli/__main__.py b/hud/cli/__main__.py index 8151fcd8b..af92d8b09 100644 --- a/hud/cli/__main__.py +++ b/hud/cli/__main__.py @@ -120,6 +120,19 @@ def root_command( raise typer.Exit(2) +def underlying_error(error: BaseException) -> BaseException | None: + """The exception a command failed with, beneath the exits that report it. + + Typer turns every failure into ``sys.exit(code)``; the original exception + survives only on the chain: ``raise Exit(code) from exc`` sets the cause, and + exiting inside an ``except`` block sets the context. + """ + current: BaseException | None = error + while isinstance(current, (SystemExit, typer.Exit)): + current = current.__cause__ or current.__context__ + return current + + @contextlib.contextmanager def recorded_invocation(argv: list[str], cli: typer.Typer) -> Iterator[None]: """Record one CLI invocation around the wrapped block, then re-raise as-is.""" @@ -129,17 +142,12 @@ def recorded_invocation(argv: list[str], cli: typer.Typer) -> Iterator[None]: try: yield except BaseException as error: - if isinstance(error, KeyboardInterrupt): - exit_code, error_class = 130, "KeyboardInterrupt" + if isinstance(error, SystemExit): + exit_code = error.code if isinstance(error.code, int) else 1 else: - exit_code = getattr(error, "exit_code", None) - if exit_code is None and isinstance(error, SystemExit): - exit_code = error.code if isinstance(error.code, int) else 1 - if isinstance(exit_code, int): - cause = error.__cause__ - error_class = type(cause).__name__ if cause is not None else None - else: - exit_code, error_class = 1, type(error).__name__ + exit_code = 130 if isinstance(error, KeyboardInterrupt) else 1 + underlying = underlying_error(error) + error_class = type(underlying).__name__ if underlying is not None else None raise finally: with contextlib.suppress(Exception): diff --git a/hud/cli/deploy.py b/hud/cli/deploy.py index 2ae0b8642..bf3664ba0 100644 --- a/hud/cli/deploy.py +++ b/hud/cli/deploy.py @@ -25,6 +25,7 @@ from websockets.exceptions import ConnectionClosed, WebSocketException from hud.cli import ( + CLI, AuthScope, CliError, DirectoryLink, @@ -186,6 +187,12 @@ async def deploy_command( dry_run: bool = typer.Option( False, "--dry-run", help="Print the planned action without making changes." ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompts (required in non-interactive terminals).", + ), ) -> Any: """Deploy HUD environment to the platform. @@ -200,6 +207,7 @@ async def deploy_command( [not dim]Examples: hud deploy hud deploy --name judge + hud deploy --yes --json hud deploy --dry-run --json[/not dim] """ platform = PlatformClient.from_settings() @@ -380,6 +388,7 @@ async def deploy_command( ) ) + CLI.confirm_or_abort(f"Deploy {name}?", yes=yes, default=True) hud_console.progress_message("Creating build context tarball...") dockerignore = env_dir / ".dockerignore" rules: list[tuple[re.Pattern[str], bool, bool]] = [] diff --git a/hud/cli/eval.py b/hud/cli/eval.py index 758c91f90..09bb10969 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -21,7 +21,7 @@ from rich import box from rich.table import Table -from hud.cli import CLI, CliError, parse_key_value +from hud.cli import CLI, CliError, Result, parse_key_value from hud.eval import ( DaytonaRuntime, DockerRuntime, @@ -196,7 +196,7 @@ def eval_command( "--remote", help="Run the whole rollout on the HUD platform (same as --runtime hosted)", ), -) -> dict[str, Any]: +) -> dict[str, Any] | Result: """Run evaluation on datasets or individual tasks with agents. A tasks file (tasks.py, a directory, or JSON/JSONL beside its env source) runs @@ -299,7 +299,7 @@ def eval_command( "action": "eval", "source": cfg.source, "agent": agent_type.value, - "model": cfg.model, + "model": cfg.model or cfg.agent_config.get(agent_type.value, {}).get("model"), "runtime": planned, "remote": planned is Placement.HOSTED, "all": cfg.all, @@ -525,8 +525,8 @@ def spawn(task: Task) -> AbstractAsyncContextManager[Runtime]: if job.runs and settings.telemetry_enabled and settings.api_key: hud_console.info(f"{settings.hud_web_url}/jobs/{UUID(job.id)}") + errors = set(map(id, job.errors)) if job.runs: - errors = set(map(id, job.errors)) hud_console.print(f"\n[bold]'{cfg.source}' Results[/bold]") hud_console.print(f" [dim]Runs:[/dim] {len(job.runs)}") hud_console.print(f" [dim]Time:[/dim] {elapsed:.1f}s") @@ -552,21 +552,23 @@ def spawn(task: Task) -> AbstractAsyncContextManager[Runtime]: hud_console.print(details) hud_console.print() - return { + result = { "job_id": job.id, "source": cfg.source, "run_count": len(job.runs), "mean_reward": job.reward, - "error_count": len(job.errors), + "error_count": len(errors), "elapsed_seconds": elapsed, "runs": [ { "task_id": run.task_id, "slug": run.slug, "reward": run.reward, - "is_error": run.trace.is_error, + "is_error": id(run) in errors, "trace_id": run.trace_id, } for run in job.runs ], } + # A low reward is a result; a run without a valid grade is a failure. + return Result(result) if errors else result diff --git a/hud/cli/harbor.py b/hud/cli/harbor.py index db91bbe7b..8239ba68c 100644 --- a/hud/cli/harbor.py +++ b/hud/cli/harbor.py @@ -263,6 +263,9 @@ def cancel_command( "-y", help="Skip confirmation prompts (required in non-interactive terminals).", ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Print the planned action without making changes." + ), ) -> Any: """Cancel a Harbor import. @@ -271,13 +274,17 @@ def cancel_command( Examples: hud harbor cancel - hud harbor cancel --yes --json[/not dim] + hud harbor cancel --yes --json + hud harbor cancel --dry-run --json[/not dim] """ platform = PlatformClient.from_settings() imported = _get_import(platform, import_id) if imported.ended: hud_console.info(f"The import already ended: {imported.status}") return imported.model_dump(mode="json") + if dry_run: + hud_console.info(f"--dry-run: would cancel import {imported.id}") + return {"dry_run": True, "action": "cancel_import", "import_id": imported.id} CLI.confirm_or_abort(f"Cancel import {imported.id}?", yes=yes, default=False) imported = imports.cancel_import(platform, imported.id) hud_console.success("Cancellation requested") diff --git a/hud/cli/jobs.py b/hud/cli/jobs.py index aa36cf80f..4b4b90179 100644 --- a/hud/cli/jobs.py +++ b/hud/cli/jobs.py @@ -182,7 +182,7 @@ def cancel_job_command( [not dim]Examples: hud jobs cancel - hud jobs cancel --trace-id --json + hud jobs cancel --trace-id --yes --json hud jobs cancel --all --yes hud jobs cancel --dry-run --json[/not dim] """ @@ -234,6 +234,9 @@ def cancel_job_command( for job in result.get("job_details", []): hud_console.info(f" • {job['job_id']}: {job['cancelled']} tasks cancelled") elif trace_id: + CLI.confirm_or_abort( + f"Cancel trace {trace_id} in job {job_id}?", yes=yes, default=False + ) hud_console.info(f"Cancelling trace {trace_id} in job {job_id}...") result = platform.post( "/rollouts/cancel", json={"job_id": job_id, "trace_id": trace_id} diff --git a/hud/cli/models.py b/hud/cli/models.py index f635da50b..df9c2a8de 100644 --- a/hud/cli/models.py +++ b/hud/cli/models.py @@ -95,6 +95,12 @@ def fork_model( "--if-not-exists", help="If a model with this name already exists, print it and exit 0.", ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompts (required in non-interactive terminals).", + ), ) -> Any: """Create a team-owned trainable model derived from an existing one. @@ -104,7 +110,7 @@ def fork_model( [not dim]Examples: hud models fork claude-sonnet-4-6 --name my-sonnet - hud models fork claude-sonnet-4-6 --name my-sonnet --json + hud models fork claude-sonnet-4-6 --name my-sonnet --yes --json hud models fork claude-sonnet-4-6 --name my-sonnet --if-not-exists hud models fork claude-sonnet-4-6 --name my-sonnet --dry-run --json[/not dim] """ @@ -118,6 +124,7 @@ def fork_model( "if_not_exists": if_not_exists, } + CLI.confirm_or_abort(f"Fork {source} as {name}?", yes=yes, default=True) source_id = resolve_gateway_model(source).id try: model = PlatformClient.from_settings().post( @@ -213,6 +220,12 @@ def show_head( dry_run: bool = typer.Option( False, "--dry-run", help="Print the planned action without making changes." ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompts (required in non-interactive terminals).", + ), ) -> Any: """Show — or with ``--set``, change — the model's active checkpoint (the weights the gateway serves now). @@ -220,6 +233,7 @@ def show_head( [not dim]Examples: hud models head hud models head --json + hud models head --set --yes hud models head --set --dry-run --json[/not dim] """ model_id = resolve_gateway_model(model).id @@ -236,6 +250,7 @@ def show_head( "model_id": model_id, "checkpoint_id": set_to, } + CLI.confirm_or_abort(f"Serve checkpoint {set_to} as the head of {model}?", yes=yes) asyncio.run(client.set_head(set_to)) hud_console.stdout.print(f"[green]Head set to[/green] [cyan]{set_to}[/cyan]") hud_console.stdout.print(f"[dim]View: {view}[/dim]") diff --git a/hud/cli/project.py b/hud/cli/project.py index 30b8136bb..0600956ee 100644 --- a/hud/cli/project.py +++ b/hud/cli/project.py @@ -173,6 +173,12 @@ def create_command( dry_run: bool = typer.Option( False, "--dry-run", help="Print the planned action without making changes." ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompts (required in non-interactive terminals).", + ), ) -> Any: """Create a Project and link this directory unless --no-use is passed.""" platform = PlatformClient.from_settings() @@ -191,6 +197,7 @@ def create_command( ) if state is not None: state.load() # a config written under other credentials fails before we create anything + CLI.confirm_or_abort(f"Create Project {name}?", yes=yes, default=True) created = Project.from_record(platform.post("/projects", json=payload)) if state is not None: state.update(DirectoryLink(project_id=created.id)) diff --git a/hud/cli/qa.py b/hud/cli/qa.py index a3f0b6a07..ca4bb7b41 100644 --- a/hud/cli/qa.py +++ b/hud/cli/qa.py @@ -517,6 +517,12 @@ def run_checks( dry_run: bool = typer.Option( False, "--dry-run", help="Print the planned action without making changes." ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompts (required in non-interactive terminals).", + ), ) -> Any: """Run QA checks on completed evaluation traces or on tasks as one run. @@ -527,6 +533,7 @@ def run_checks( hud qa run failure_analysis hud qa run reward_hacking,false_positive --no-wait --json hud qa run task_reward_exploit,task_failure_analysis + hud qa run failure_analysis --yes --json hud qa run failure_analysis --dry-run --json[/not dim] """ check_keys = list(dict.fromkeys(key.strip() for key in checks.split(",") if key.strip())) @@ -553,6 +560,9 @@ def run_checks( "overwrite": overwrite, "wait": wait, } + CLI.confirm_or_abort( + f"Run {len(check_keys)} check(s) on {len(subject_ids)} {level}(s)?", yes=yes, default=True + ) try: response = cast( diff --git a/hud/cli/serve.py b/hud/cli/serve.py index 34aa5972d..ea30c2c8a 100644 --- a/hud/cli/serve.py +++ b/hud/cli/serve.py @@ -14,7 +14,7 @@ from hud.cli import parse_key_value from hud.environment import load_environment -from hud.environment.server import serve +from hud.environment.server import serve_until_terminated from hud.utils.hud_console import HUDConsole hud_console = HUDConsole() @@ -69,6 +69,6 @@ def serve_command( ) hud_console.hint("Press Ctrl+C to stop.") try: - asyncio.run(serve(env, host, port)) + asyncio.run(serve_until_terminated(env, host, port)) except KeyboardInterrupt: hud_console.info("Stopped.") diff --git a/hud/cli/sync.py b/hud/cli/sync.py index 64787f268..d76ccb7c5 100644 --- a/hud/cli/sync.py +++ b/hud/cli/sync.py @@ -414,21 +414,40 @@ def sync_env_command( @sync_app.callback(invoke_without_command=True) -def sync_callback(ctx: typer.Context) -> Any: +def sync_callback( + ctx: typer.Context, + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompts (required in non-interactive terminals).", + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Print the planned action without making changes." + ), +) -> Any: """Sync tasks and environments to the HUD platform. [not dim]Without a subcommand, syncs tasks using stored config. Examples: hud sync # sync tasks using .hud/config.json + hud sync --yes # without the confirmation prompt hud sync tasks my-taskset # sync tasks to specific taskset hud sync env my-env # link to a deployed environment[/not dim] """ if ctx.invoked_subcommand is not None: + if yes or dry_run: + raise CliError( + error="usage", + message="Pass --yes and --dry-run after the subcommand.", + suggestion=f"hud sync {ctx.invoked_subcommand} --yes", + ) return assert isinstance(ctx.command, TyperGroup) command = ctx.command.get_command(ctx, "tasks") assert command is not None - with command.make_context("tasks", [], parent=ctx) as task_context: + forwarded = [flag for flag, on in (("--yes", yes), ("--dry-run", dry_run)) if on] + with command.make_context("tasks", forwarded, parent=ctx) as task_context: return command.invoke(task_context) diff --git a/hud/cli/tests/test_cli.py b/hud/cli/tests/test_cli.py deleted file mode 100644 index 38cce0a2c..000000000 --- a/hud/cli/tests/test_cli.py +++ /dev/null @@ -1,876 +0,0 @@ -"""Tests for ``hud.cli`` (infrastructure) and ``hud.cli.__main__`` (the assembled command).""" - -from __future__ import annotations - -import json -import re -import sys -import uuid -import warnings -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock, patch -from urllib.parse import urlsplit -from uuid import UUID - -import httpx -import pytest -import typer -from dotenv import dotenv_values -from typer.core import TyperGroup -from typer.main import get_command -from typer.testing import CliRunner - -from hud.cli import ( - CLI, - AuthScope, - CliError, - DirectoryLink, - DirectoryState, - ExitCode, - set_env_values, -) -from hud.cli.__main__ import ( - app, - main, - notify_if_outdated, - recorded_invocation, - render_hud_warnings, - version, -) -from hud.utils.exceptions import HudDeprecationWarning, HudException, HudRequestError -from hud.utils.gateway import list_gateway_models -from hud.utils.platform import PlatformClient - -runner = CliRunner() -_ANSI = re.compile(r"\x1b\[[0-9;]*m") - - -def _plain(text: str) -> str: - return _ANSI.sub("", text) - - -def _stdout(result: Any) -> str: - return result.stdout if getattr(result, "stdout", None) is not None else result.output - - -def test_set_env_values_merges_and_preserves_quotes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - secret = "key # with 'quotes' and \\slashes\nsecond line" - path = set_env_values({"HUD_API_KEY": secret, "C": '"quoted"'}) - assert path == tmp_path / ".hud" / ".env" - set_env_values({"HUD_DEFAULT_PROJECT": "example"}) - loaded = dotenv_values(path, interpolate=False) - assert loaded["HUD_API_KEY"] == secret - assert loaded["C"] == '"quoted"' - assert loaded["HUD_DEFAULT_PROJECT"] == "example" - - -SCOPE = { - "origin": "https://api.example", - "user_id": "11111111-1111-4111-8111-111111111111", - "team_id": "22222222-2222-4222-8222-222222222222", -} - - -def test_scoped_links_do_not_cross_origins_users_teams_or_directories(tmp_path: Path) -> None: - scope = AuthScope.model_validate(SCOPE) - directory = tmp_path / "environment" - state = DirectoryState(scope, directory) - registry = UUID(int=10) - assert state.update(DirectoryLink(registry_id=registry)) is True - before = state.path.read_text() - assert state.update(DirectoryLink(registry_id=registry)) is False - assert state.path.read_text() == before - assert state.load().registry_id == registry - assert DirectoryState(scope, tmp_path / "worktree").load().registry_id is None - for field, value in [ - ("origin", "https://other.example"), - ("user_id", str(UUID(int=30))), - ("team_id", str(UUID(int=40))), - ]: - other = AuthScope.model_validate({**SCOPE, field: value}) - with pytest.raises(CliError, match=r"different HUD credentials|was linked against"): - DirectoryState(other, directory).load() - assert state.path == directory / ".hud" / "config.json" - - -def test_released_camelcase_config_reads_and_migrates_on_write(tmp_path: Path) -> None: - """A config written by hud 0.6.x (camelCase ids, cached name, .env preference, no - scope) still links the directory, and the next write rewrites it in this schema.""" - directory = tmp_path / "environment" - path = directory / ".hud" / "config.json" - path.parent.mkdir(parents=True) - registry, taskset = UUID(int=10), UUID(int=11) - path.write_text( - json.dumps( - { - "registryId": str(registry), - "registryName": "browser-env", - "tasksetId": str(taskset), - "syncEnv": True, - } - ) - ) - state = DirectoryState(AuthScope.model_validate(SCOPE), directory) - - link = state.load() - assert (link.registry_id, link.taskset_id, link.scope) == (registry, taskset, None) - - assert state.update(DirectoryLink(project_id=UUID(int=12))) is True - written = json.loads(path.read_text()) - assert written["scope"] == SCOPE - assert written["registry_id"] == str(registry) - assert written["taskset_id"] == str(taskset) - assert written["project_id"] == str(UUID(int=12)) - assert not {"registryId", "registryName", "tasksetId", "syncEnv"} & written.keys() - with pytest.raises(CliError, match="different HUD credentials"): - DirectoryState( - AuthScope.model_validate({**SCOPE, "user_id": str(UUID(int=3))}), directory - ).load() - - -def test_read_leaves_legacy_files_and_home_untouched(tmp_path: Path) -> None: - directory = tmp_path / "environment" - legacy = directory / ".hud" / "deploy.json" - legacy.parent.mkdir(parents=True) - legacy.write_text('{"registryId":"old"}') - state = DirectoryState(AuthScope.model_validate(SCOPE), directory) - assert state.load().registry_id is None - assert legacy.read_text() == '{"registryId":"old"}' - assert not (legacy.parent / "config.json").exists() - - -def test_corrupt_config_is_not_overwritten(tmp_path: Path) -> None: - state = DirectoryState(AuthScope.model_validate(SCOPE), tmp_path) - path = state.path - path.parent.mkdir(parents=True) - path.write_text('{"broken":') - with pytest.raises(CliError, match="not a valid HUD workspace config"): - state.load() - with pytest.raises(CliError, match="not a valid HUD workspace config"): - state.update(DirectoryLink()) - assert path.read_text() == '{"broken":' - - -def test_api_scope_uses_authoritative_identity_not_credentials(monkeypatch) -> None: - def request(method, url, **kwargs): - assert method == "GET" - assert url.endswith("/v2/auth/me") - return SCOPE - - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) - first = AuthScope.resolve(PlatformClient("https://API.EXAMPLE:443/", "first-secret")) - second = AuthScope.resolve(PlatformClient("https://api.example", "rotated-secret")) - assert first == second == AuthScope.model_validate(SCOPE) - assert "secret" not in first.model_dump_json() - - -def test_json_object_names_the_flag() -> None: - assert CLI.json_object('{"image": "x"}', option="--payload") == {"image": "x"} - with pytest.raises(CliError) as exc_info: - CLI.json_object("[]", option="--payload") - assert exc_info.value.message == "--payload must be a JSON object" - assert exc_info.value.input == {"payload": "[]"} - - -def test_output_mode_does_not_leak_between_invocations() -> None: - from typer.testing import CliRunner - - from hud.cli.__main__ import app - - runner = CliRunner() - first = runner.invoke(app, ["version", "--json"]) - second = runner.invoke(app, ["version"]) - flag = runner.invoke(app, ["--version"]) - assert json.loads(first.stdout)["name"] == "hud" - assert "HUD CLI version:" not in first.stdout - assert second.stdout.startswith("HUD CLI version:") - assert flag.exit_code == 0 - assert "HUD CLI version:" in flag.stdout - unexpected = runner.invoke(app, ["--json", "version"]) - assert unexpected.exit_code != 0 - - -def test_map_request_error_status_codes() -> None: - not_found = CliError.from_http(HudRequestError("x", status_code=404)) - assert not_found.error == "not_found" - assert not_found.exit_code == ExitCode.FAILURE - permission = CliError.from_http(HudRequestError("x", status_code=403)) - assert permission.error == "permission_denied" - assert permission.exit_code == ExitCode.FAILURE - conflict = CliError.from_http(HudRequestError("x", status_code=409)) - assert conflict.error == "conflict" - assert conflict.exit_code == ExitCode.FAILURE - mapped = CliError.from_http(HudRequestError("x", status_code=429)) - assert mapped.error == "rate_limited" - - -def test_failure_text_writes_only_stderr() -> None: - from typer.testing import CliRunner - - from hud.cli.__main__ import app - - result = CliRunner().invoke(app, ["set", "NOT_A_PAIR"]) - assert result.exit_code == ExitCode.USAGE - assert result.stdout == "" - assert "Error:" in result.stderr - assert "Hint:" in result.stderr - - -def test_confirm_or_abort_noninteractive_requires_yes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(sys.stdin, "isatty", lambda: False) - with pytest.raises(CliError) as exc_info: - CLI.confirm_or_abort("Proceed?") - assert exc_info.value.exit_code == ExitCode.USAGE - - -def test_confirm_or_abort_yes_skips_prompt(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(sys.stdin, "isatty", lambda: False) - CLI.confirm_or_abort("Proceed?", yes=True) - - -def test_read_text_stdin_and_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - target = tmp_path / "answer.txt" - target.write_text("hello", encoding="utf-8") - assert CLI.read_text(str(target)) == "hello" - - monkeypatch.setattr("hud.cli.sys.stdin.read", lambda: "from-stdin") - assert CLI.read_text("-") == "from-stdin" - - -def test_read_text_missing_file_is_not_found() -> None: - with pytest.raises(CliError) as exc_info: - CLI.read_text("/definitely/missing/hud-cli-file.txt") - assert exc_info.value.error == "not_found" - assert exc_info.value.exit_code == ExitCode.FAILURE - - -def _pypi(version: str) -> httpx.Response: - return httpx.Response(200, json={"info": {"version": version}}) - - -def _fail_fetch(*_a: object, **_k: object) -> httpx.Response: - raise AssertionError("fetch") - - -def test_cli_prints_hud_deprecation_warnings_through_the_console( - capsys: pytest.CaptureFixture[str], -) -> None: - with warnings.catch_warnings(record=True) as passed_through: - warnings.simplefilter("always") - render_hud_warnings() - warnings.warn("GET /v2/old is deprecated", HudDeprecationWarning, stacklevel=1) - warnings.warn("other", UserWarning, stacklevel=1) - - assert capsys.readouterr().err == "\u26a0 GET /v2/old is deprecated\n" - assert [str(warning.message) for warning in passed_through] == ["other"] - - -class TestVersionCheck: - @pytest.fixture(autouse=True) - def _isolated(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.delenv("CI", raising=False) - monkeypatch.delenv("HUD_SKIP_VERSION_CHECK", raising=False) - monkeypatch.delenv("VIRTUAL_ENV", raising=False) - monkeypatch.setattr("hud.cli.__main__.__version__", "1.0.0") - - def test_outdated_prints_banner_and_reuses_cache( - self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], tmp_path: Path - ) -> None: - fetches = {"n": 0} - - def get(*_a: object, **_k: object) -> httpx.Response: - fetches["n"] += 1 - return _pypi("2.0.0") - - monkeypatch.setattr(httpx, "get", get) - monkeypatch.setattr(sys, "prefix", "/usr") - monkeypatch.setattr(sys, "base_prefix", "/usr") - notify_if_outdated(["hud", "eval"]) - first = capsys.readouterr().err - assert "2.0.0" in first - assert "current: 1.0.0" in first - assert "uv tool upgrade hud" in first - notify_if_outdated(["hud", "eval"]) - assert capsys.readouterr().err.count("2.0.0") == 1 - assert fetches["n"] == 1 - cached = json.loads((tmp_path / ".hud" / ".cache" / "version_check.json").read_text()) - assert cached["latest"] == "2.0.0" - - def test_current_version_is_silent( - self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] - ) -> None: - monkeypatch.setattr(httpx, "get", lambda *_a, **_k: _pypi("1.0.0")) - notify_if_outdated(["hud", "eval"]) - assert capsys.readouterr().err == "" - - def test_project_venv_suggests_uv_sync( - self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] - ) -> None: - monkeypatch.setattr(httpx, "get", lambda *_a, **_k: _pypi("2.0.0")) - monkeypatch.setattr(sys, "prefix", "/proj/.venv") - monkeypatch.setattr(sys, "base_prefix", "/usr") - notify_if_outdated(["hud", "eval"]) - assert "uv sync --upgrade-package hud" in capsys.readouterr().err - - def test_uv_tool_install_is_not_a_project_venv( - self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] - ) -> None: - monkeypatch.setattr(httpx, "get", lambda *_a, **_k: _pypi("2.0.0")) - monkeypatch.setattr(sys, "prefix", "/Users/x/.local/share/uv/tools/hud") - monkeypatch.setattr(sys, "base_prefix", "/usr") - notify_if_outdated(["hud", "eval"]) - assert "uv tool upgrade hud" in capsys.readouterr().err - - @pytest.mark.parametrize( - "argv", - [["hud"], ["hud", "--version"], ["hud", "--help"], ["hud", "--version", "--json"]], - ) - def test_help_and_version_do_not_fetch( - self, argv: list[str], monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(httpx, "get", _fail_fetch) - notify_if_outdated(argv) - - def test_ci_and_opt_out_do_not_fetch(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(httpx, "get", _fail_fetch) - monkeypatch.setenv("CI", "1") - notify_if_outdated(["hud", "eval"]) - monkeypatch.delenv("CI") - monkeypatch.setenv("HUD_SKIP_VERSION_CHECK", "1") - notify_if_outdated(["hud", "eval"]) - - def test_expired_cache_refetches( - self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] - ) -> None: - cache = tmp_path / ".hud" / ".cache" / "version_check.json" - cache.parent.mkdir(parents=True) - cache.write_text(json.dumps({"latest": "1.5.0", "checked_at": 0.0})) - monkeypatch.setattr(httpx, "get", lambda *_a, **_k: _pypi("2.0.0")) - notify_if_outdated(["hud", "eval"]) - assert "2.0.0" in capsys.readouterr().err - - def test_fetch_failure_does_not_raise( - self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] - ) -> None: - def down(*_a: object, **_k: object) -> httpx.Response: - raise httpx.ConnectError("down") - - monkeypatch.setattr(httpx, "get", down) - notify_if_outdated(["hud", "eval"]) - assert capsys.readouterr().err == "" - - -def _event( - argv: list[str], - monkeypatch: pytest.MonkeyPatch, - *, - error: BaseException | None = None, -) -> dict[str, Any] | None: - sent: list[dict[str, Any]] = [] - monkeypatch.setattr( - httpx, "post", lambda _url, json, **_k: sent.append(json) or httpx.Response(204) - ) - try: - with recorded_invocation(argv, app): - if error is not None: - raise error - except BaseException as exc: - if exc is not error and type(exc) is not type(error): - raise - if not sent: - return None - (payload,) = sent - (event,) = payload["events"] - return event - - -class TestUsage: - @pytest.fixture(autouse=True) - def _analytics_on(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "1") - monkeypatch.setenv("HUD_TELEMETRY_URL", "https://telemetry.example.test/v3/api") - - def test_arguments_are_never_captured(self, monkeypatch: pytest.MonkeyPatch) -> None: - event = _event(["hud", "eval", "tasks.py", "claude"], monkeypatch) - assert event is not None - assert event["command"] == "eval" - assert event["subcommand"] is None - - def test_registered_subcommands_are_captured(self, monkeypatch: pytest.MonkeyPatch) -> None: - event = _event(["hud", "models", "list"], monkeypatch) - assert event is not None - assert (event["command"], event["subcommand"]) == ("models", "list") - - def test_callback_group_positionals_are_never_captured( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - trace = _event(["hud", "trace", "8b1f2c3d4e5f"], monkeypatch) - jobs = _event(["hud", "jobs", "0f9e8d7c"], monkeypatch) - assert trace is not None and jobs is not None - assert (trace["command"], trace["subcommand"]) == ("trace", None) - assert (jobs["command"], jobs["subcommand"]) == ("jobs", None) - - def test_jobs_verbs_are_captured(self, monkeypatch: pytest.MonkeyPatch) -> None: - for argv, verb in [ - (["hud", "jobs", "list"], "list"), - (["hud", "jobs", "cancel"], "cancel"), - (["hud", "trace", "get"], "get"), - (["hud", "qa", "list"], "list"), - ]: - event = _event(argv, monkeypatch) - assert event is not None and event["subcommand"] == verb - - def test_unregistered_command_is_other(self, monkeypatch: pytest.MonkeyPatch) -> None: - event = _event(["hud", "secret-name"], monkeypatch) - assert event is not None - assert event["command"] == "other" - assert "secret-name" not in event.values() - - def test_bare_invocation_is_help(self, monkeypatch: pytest.MonkeyPatch) -> None: - event = _event(["hud"], monkeypatch) - assert event is not None - assert event["command"] == "help" - - def test_version_flag_is_not_an_event(self, monkeypatch: pytest.MonkeyPatch) -> None: - assert _event(["hud", "--version"], monkeypatch) is None - assert _event(["hud", "--version", "--json"], monkeypatch) is None - - def test_flags_are_skipped(self, monkeypatch: pytest.MonkeyPatch) -> None: - event = _event(["hud", "--verbose", "serve"], monkeypatch) - assert event is not None - assert event["command"] == "serve" - - def test_typer_exit_from_hud_exception_names_the_cause( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - try: - raise HudException("boom") - except HudException as exc: - converted = typer.Exit(1) - converted.__cause__ = exc - event = _event(["hud", "eval"], monkeypatch, error=converted) - assert event is not None - assert (event["exit_code"], event["error_class"]) == (1, "HudException") - - def test_plain_exit_has_no_error_class(self, monkeypatch: pytest.MonkeyPatch) -> None: - event = _event(["hud", "eval"], monkeypatch, error=typer.Exit(2)) - assert event is not None - assert (event["exit_code"], event["error_class"]) == (2, None) - - def test_keyboard_interrupt(self, monkeypatch: pytest.MonkeyPatch) -> None: - event = _event(["hud", "eval"], monkeypatch, error=KeyboardInterrupt()) - assert event is not None - assert (event["exit_code"], event["error_class"]) == (130, "KeyboardInterrupt") - - def test_unexpected_exception(self, monkeypatch: pytest.MonkeyPatch) -> None: - event = _event(["hud", "eval"], monkeypatch, error=ValueError("x")) - assert event is not None - assert (event["exit_code"], event["error_class"]) == (1, "ValueError") - - def test_install_id_created_once( - self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] - ) -> None: - first = _event(["hud", "eval"], monkeypatch) - second = _event(["hud", "eval"], monkeypatch) - assert first is not None and second is not None - assert first["install_id"] == second["install_id"] - assert uuid.UUID(first["install_id"]) - assert capsys.readouterr().err.count("anonymous CLI usage") == 1 - - def test_opt_out_applies_immediately(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "0") - assert _event(["hud", "eval"], monkeypatch) is None - - def test_command_error_propagates_when_opted_out(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "0") - with ( - pytest.raises(ValueError, match="boom"), - recorded_invocation(["hud", "eval"], app), - ): - raise ValueError("boom") - - def test_payload_is_the_allowlist(self, monkeypatch: pytest.MonkeyPatch) -> None: - sent: list[tuple[str, dict[str, Any]]] = [] - monkeypatch.setattr( - httpx, - "post", - lambda url, json, **_k: sent.append((url, json)) or httpx.Response(204), - ) - with recorded_invocation(["hud", "serve", "my_env.py"], app): - pass - (url, payload) = sent[0] - assert url == "https://telemetry.example.test/v3/api/sdk-events/cli" - (event,) = payload["events"] - assert event["command"] == "serve" - assert event["subcommand"] is None - assert "my_env.py" not in str(payload) - assert set(event) == { - "command", - "subcommand", - "exit_code", - "error_class", - "duration_ms", - "cli_version", - "python_version", - "os", - "is_ci", - "install_id", - } - - -class TestCLICommands: - """Test CLI command handling.""" - - def test_main_shows_help_when_no_args(self) -> None: - """Test that main() shows help when no arguments provided.""" - result = runner.invoke(app) - assert result.exit_code == 2 - assert "Usage:" in result.output - - def test_version_command(self) -> None: - with patch("hud.cli.__main__.__version__", "1.2.3"): - result = runner.invoke(app, ["version"]) - assert result.exit_code == 0 - assert "1.2.3" in _plain(result.output) - - def test_mcp_command(self) -> None: - """Test mcp server command.""" - result = runner.invoke(app, ["mcp"]) - assert result.exit_code == 2 - - def test_help_command(self) -> None: - """Test help command lists v6 commands.""" - result = runner.invoke(app, ["--help"]) - assert result.exit_code == 0 - assert "eval" in result.output - assert "deploy" in result.output - - -class TestMainFunction: - """Test the main() function specifically.""" - - def test_main_with_help_flag(self) -> None: - original_argv = sys.argv - try: - sys.argv = ["hud", "--help"] - with ( - patch("hud.cli.__main__.notify_if_outdated"), - patch("hud.cli.__main__.app") as mock_app, - ): - main() - mock_app.assert_called() - finally: - sys.argv = original_argv - - def test_main_with_no_args(self) -> None: - original_argv = sys.argv - try: - sys.argv = ["hud"] - with pytest.raises(SystemExit) as exc_info: - main() - assert exc_info.value.code == 2 - finally: - sys.argv = original_argv - - -def test_version_does_not_crash() -> None: - version() - - -def test_model_commands_share_platform_transport(monkeypatch): - from hud.settings import settings - - model_id = "00000000-0000-4000-a000-000000000001" - requests = [] - - def request(method, url, **kwargs): - requests.append((method, url, kwargs)) - assert url.startswith("https://api.example/v2/") - assert kwargs["api_key"] == "test-key" - if urlsplit(url).path == "/v2/models": - return {"items": [{"id": model_id, "name": "owner/model + version"}], "total": 1} - return [] if method == "GET" else {"id": model_id, "model_name": "forked"} - - async def async_request(method, url, **kwargs): - return request(method, url, **kwargs) - - monkeypatch.setattr(settings, "api_key", "test-key") - monkeypatch.setattr(settings, "hud_api_url", "https://api.example/") - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) # catalog, fork - monkeypatch.setattr("hud.train.base.make_request", async_request) # TrainingClient - for command, extra, method, endpoint in [ - ("checkpoints", [], "GET", f"/models/{model_id}/checkpoints"), - ("head", [], "GET", f"/models/{model_id}/checkpoints"), - ("head", ["--set", "checkpoint"], "PUT", f"/models/{model_id}/head"), - ("fork", ["--name", "forked"], "POST", "/models/fork"), - ]: - requests.clear() - list_gateway_models.cache_clear() - result = runner.invoke(app, ["models", command, "owner/model + version", *extra, "--json"]) - assert result.exit_code == 0, result.output - assert len(requests) == 2 - assert requests[-1][:2] == (method, f"https://api.example/v2{endpoint}") - if command == "head" and not extra: - # No active checkpoint is a valid state and still a JSON payload. - assert json.loads(result.stdout) == {"model_id": model_id, "head": None} - - -def test_task_rejects_non_object_args_before_connecting(): - result = runner.invoke(app, ["task", "grade", "solve", "--args", "[]", "--json"]) - assert result.exit_code == ExitCode.USAGE - assert json.loads(result.stdout)["message"] == "--args must be a JSON object" - - -def test_root_help_lists_nouns() -> None: - result = runner.invoke(app, ["--help"]) - assert result.exit_code == 0 - text = _plain(result.output) - assert "Usage: hud COMMAND" in text - assert "[OPTIONS] COMMAND [ARGS]" not in text - assert "jobs" in text - assert "models" in text - assert "task" in text - assert re.search(r"^ {2}auth\b", text, re.M) is None - assert re.search(r"^ {2}client\b", text, re.M) is None - assert re.search(r"^ {2}cancel\b", text, re.M) is None - assert "--help" in text - assert "--version" in text - assert "--json" not in text - assert text.index("--help") < text.index("--version") - assert "Show help." in text - assert "Show this message and exit." not in text - group = get_command(app) - assert isinstance(group, TyperGroup) - assert group.list_commands(typer.Context(group)) == [ - "init", - "serve", - "deploy", - "eval", - "task", - "project", - "sync", - "harbor", - "qa", - "jobs", - "cancel", - "trace", - "models", - "set", - "version", - ] - - -def test_plan_flags_use_shared_help() -> None: - for args in ( - ["eval", "--help"], - ["deploy", "--help"], - ["sync", "tasks", "--help"], - ["sync", "env", "--help"], - ["jobs", "cancel", "--help"], - ["harbor", "import", "--help"], - ): - result = runner.invoke(app, args) - assert result.exit_code == 0, result.output - text = " ".join(_plain(result.output).replace("│", " ").split()) - assert "planned action" in text - if args[0] != "deploy": - assert "non-interactive terminals" in text - - -def test_jobs_list_help_documents_json_and_examples() -> None: - result = runner.invoke(app, ["jobs", "list", "--help"]) - assert result.exit_code == 0 - text = _plain(result.output) - assert "--json" in text - assert "hud jobs list --json" in text - assert "--quiet" in text - - -def test_jobs_list_json_and_quiet() -> None: - client = MagicMock() - client.get.return_value = { - "items": [ - {"id": "job-1", "name": "eval", "status": "done", "created_at": "2026-01-01T00:00:00Z"} - ] - } - with ( - patch("hud.utils.platform.PlatformClient.from_settings", return_value=client), - ): - json_result = runner.invoke(app, ["jobs", "list", "--json"]) - quiet_result = runner.invoke(app, ["jobs", "list", "--quiet"]) - - assert json_result.exit_code == 0 - payload = json.loads(_stdout(json_result)) - assert payload[0]["id"] == "job-1" - assert quiet_result.exit_code == 0 - assert quiet_result.stdout.strip() == "job-1" or "job-1" in quiet_result.output - - -def test_jobs_get_not_found_exit_code() -> None: - client = MagicMock() - client.get.side_effect = HudRequestError("missing", status_code=404) - with ( - patch("hud.utils.platform.PlatformClient.from_settings", return_value=client), - ): - result = runner.invoke( - app, ["jobs", "get", "00000000-0000-0000-0000-000000000001", "--json"] - ) - - assert result.exit_code == ExitCode.FAILURE - payload = json.loads(_stdout(result)) - assert payload["error"] == "not_found" - assert "Error:" not in (result.stderr or "") - assert payload["input"]["job_id"] == "00000000-0000-0000-0000-000000000001" - - -def test_jobs_malformed_bare_id_is_a_usage_error() -> None: - result = runner.invoke(app, ["jobs", "00000000-0000-0000-0000-invalid", "--json"]) - assert result.exit_code == ExitCode.USAGE - - -def test_cancel_usage_error_and_dry_run() -> None: - missing = runner.invoke(app, ["cancel", "--json"]) - assert missing.exit_code == ExitCode.USAGE - assert json.loads(_stdout(missing))["error"] == "usage" - - dry = runner.invoke(app, ["jobs", "cancel", "job-1", "--dry-run", "--json", "--yes"]) - assert dry.exit_code == 0 - payload = json.loads(_stdout(dry)) - assert payload["dry_run"] is True - assert payload["action"] == "cancel_job" - assert payload["job_id"] == "job-1" - - -def test_cancel_dry_run_json_skips_confirmation() -> None: - result = runner.invoke(app, ["cancel", "job-1", "--dry-run", "--json"]) - assert result.exit_code == 0 - payload = json.loads(_stdout(result)) - assert payload == { - "dry_run": True, - "action": "cancel_job", - "job_id": "job-1", - "trace_id": None, - "all": False, - } - - -def test_cancel_alias_is_hidden_and_deprecated() -> None: - result = runner.invoke(app, ["cancel", "--help"]) - assert result.exit_code == 0 - text = _plain(result.output) - assert "--json" in text - assert "--dry-run" in text - assert "--yes" in text - assert "deprecated" in text.lower() - root = _plain(runner.invoke(app, ["--help"]).output) - assert re.search(r"^ {2}cancel\b", root, re.M) is None - - -def test_trace_get_help_and_json() -> None: - get_help = runner.invoke(app, ["trace", "get", "--help"]) - assert get_help.exit_code == 0 - assert "--json" in _plain(get_help.output) - - with ( - patch( - "hud.utils.platform.PlatformClient.get", - return_value={"events": [{"kind": "agent_message", "text": "hi"}]}, - ), - patch("hud.settings.settings.api_key", "test-key"), - patch("hud.settings.settings.telemetry_local_dir", None), - ): - result = runner.invoke(app, ["trace", "get", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "--json"]) - assert result.exit_code == 0 - assert json.loads(_stdout(result))[0]["kind"] == "agent_message" - - -def test_version_json() -> None: - result = runner.invoke(app, ["version", "--json"]) - assert result.exit_code == 0 - payload = json.loads(_stdout(result)) - assert payload["name"] == "hud" - assert isinstance(payload["version"], str) - - -def test_set_invalid_assignment_is_usage() -> None: - result = runner.invoke(app, ["set", "NOT_A_PAIR", "--json"]) - assert result.exit_code == ExitCode.USAGE - assert json.loads(_stdout(result))["error"] == "usage" - - -def test_auth_noun_group_is_removed() -> None: - result = runner.invoke(app, ["auth", "--help"]) - assert result.exit_code != 0 - assert "No such command" in _plain(result.output) or "Usage:" in _plain(result.output) - - -def test_models_list_help_has_examples() -> None: - result = runner.invoke(app, ["models", "list", "--help"]) - assert result.exit_code == 0 - text = _plain(result.output) - assert "--json" in text - assert "hud models list --json" in text - - -def test_missing_api_key_is_permission(monkeypatch: pytest.MonkeyPatch) -> None: - from hud.settings import settings - - monkeypatch.setattr(settings, "api_key", "") - result = runner.invoke(app, ["jobs", "list", "--json"]) - assert result.exit_code == ExitCode.FAILURE - payload = json.loads(_stdout(result)) - assert payload["error"] == "permission_denied" - - -def test_init_conflict_exit_code(tmp_path: Any) -> None: - target = tmp_path / "taken" - target.mkdir() - (target / "keep.txt").write_text("x") - result = runner.invoke( - app, ["init", "taken", "--dir", str(tmp_path), "--preset", "blank", "--json"] - ) - assert result.exit_code == ExitCode.FAILURE - payload = json.loads(_stdout(result)) - assert payload["error"] == "conflict" - - -def test_init_dry_run_json(tmp_path: Any) -> None: - result = runner.invoke( - app, - ["init", "fresh", "--dir", str(tmp_path), "--preset", "blank", "--dry-run", "--json"], - ) - assert result.exit_code == 0 - payload = json.loads(_stdout(result)) - assert payload["dry_run"] is True - assert payload["action"] == "init" - assert not (tmp_path / "fresh" / "env.py").exists() - - -def test_task_list_json_and_quiet(tmp_path: Any) -> None: - (tmp_path / "tasks.json").write_text( - json.dumps([{"id": "solve", "prompt": "hi", "env": "demo"}]), - encoding="utf-8", - ) - # Taskset.from_file on a JSON list may not match this repo's schema; invoke help instead - # if collection fails. The contract under test is flags + help. - help_result = runner.invoke(app, ["task", "list", "--help"]) - assert help_result.exit_code == 0 - text = _plain(help_result.output) - assert "--json" in text - assert "--quiet" in text - - -def test_qa_help_documents_json() -> None: - result = runner.invoke(app, ["qa", "run", "--help"]) - assert result.exit_code == 0 - text = _plain(result.output) - assert "--json" in text - assert "--dry-run" in text diff --git a/hud/cli/tests/test_deploy.py b/hud/cli/tests/test_deploy.py deleted file mode 100644 index 21b486283..000000000 --- a/hud/cli/tests/test_deploy.py +++ /dev/null @@ -1,883 +0,0 @@ -"""Tests for CLI deploy command.""" - -from __future__ import annotations - -import io -import json -import sys -import tarfile -from contextlib import contextmanager -from pathlib import Path -from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from typer.testing import CliRunner - -from hud.cli import AuthScope, DirectoryState -from hud.cli.__main__ import app -from hud.utils.hud_console import HUDConsole -from hud.utils.platform import PlatformClient - -if TYPE_CHECKING: - from collections.abc import Iterator - -_PROJECT_ID = "44444444-4444-4444-8444-444444444444" -_REGISTRY_ID = "55555555-5555-4555-8555-555555555555" - - -class _FakeHttpxClient: - uploaded: bytes = b"" - - def __init__(self, *args: object, **kwargs: object) -> None: - pass - - async def __aenter__(self) -> _FakeHttpxClient: - return self - - async def __aexit__(self, *args: object) -> None: - return None - - async def put(self, *args: object, **kwargs: object) -> MagicMock: - content = kwargs["content"] - assert isinstance(content, bytes) - type(self).uploaded = content - response = MagicMock() - response.raise_for_status = MagicMock() - return response - - -class _TtyCliRunner(CliRunner): - """CliRunner replaces stdin with a pipe; this one still reports a TTY.""" - - @contextmanager - def isolation(self, *args: Any, **kwargs: Any) -> Iterator[Any]: - with ( - super().isolation(*args, **kwargs) as streams, - patch.object(sys.stdin, "isatty", return_value=True), - ): - yield streams - - -def _write_env(tmp_path: Path, name: str = "e") -> None: - (tmp_path / "Dockerfile").write_text("FROM python:3.12\n", encoding="utf-8") - (tmp_path / "env.py").write_text(f'env = Environment("{name}")\n', encoding="utf-8") - - -def _dry_run(tmp_path: Path, *args: str) -> dict[str, Any]: - result = CliRunner().invoke( - app, ["deploy", str(tmp_path), "--dry-run", "--json", "--no-env", *args] - ) - assert result.exit_code == 0, result.output - return json.loads(result.stdout) - - -def _stub_remote_build(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: - captured: dict[str, Any] = {} - - async def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - if url.endswith("/upload-url"): - return {"upload_url": "https://upload.example", "build_id": "build-1"} - if url.endswith("/trigger"): - captured.clear() - captured.update(kwargs["json"]) - return {"id": "build-1", "registry_id": _REGISTRY_ID} - if url.endswith("/status"): - return {"status": "SUCCEEDED"} - raise AssertionError(url) - - websocket = MagicMock() - websocket.__aiter__.return_value = [ - json.dumps({"type": "complete", "final_status": "SUCCEEDED", "message": "Build SUCCEEDED"}) - ] - connection = MagicMock() - connection.__aenter__.return_value = websocket - monkeypatch.setattr("hud.utils.platform.make_request", request) - monkeypatch.setattr("hud.cli.deploy.httpx.AsyncClient", _FakeHttpxClient) - monkeypatch.setattr("hud.cli.deploy.websockets.connect", lambda *a, **k: connection) - monkeypatch.setattr("hud.cli.deploy.asyncio.sleep", AsyncMock()) - return captured - - -def _archive(source: Path, monkeypatch: pytest.MonkeyPatch) -> set[str]: - directory = source.parent if source.is_file() else source - if not any( - "Environment(" in path.read_text(encoding="utf-8") - for path in directory.glob("*.py") - if path.is_file() - ): - (directory / "env.py").write_text('env = Environment("e")\n', encoding="utf-8") - _FakeHttpxClient.uploaded = b"" - _stub_remote_build(monkeypatch) - result = CliRunner().invoke(app, ["deploy", str(source), "--json", "--no-env"]) - assert result.exit_code == 0, result.output - with tarfile.open(fileobj=io.BytesIO(_FakeHttpxClient.uploaded), mode="r:gz") as tar: - return set(tar.getnames()) - - -def test_runtime_flag_is_case_insensitive(tmp_path: Path) -> None: - _write_env(tmp_path) - - assert _dry_run(tmp_path, "--runtime", "HUD")["runtime"] == "hud" - - -def test_unknown_runtime_is_a_usage_error_before_upload(tmp_path: Path) -> None: - _write_env(tmp_path) - - message = _deploy_name_error(tmp_path, "--runtime", "moddal") - assert "Unknown runtime 'moddal'" in message - assert "hud, modal" in message - - -def _deploy_name_error(tmp_path: Path, *args: str) -> str: - result = CliRunner().invoke( - app, ["deploy", str(tmp_path), "--dry-run", "--json", "--no-env", *args] - ) - assert result.exit_code == 2, result.output - return json.loads(result.stdout)["message"] - - -class TestResolveEnvironmentName: - """Tests for code-authoritative environment name resolution.""" - - def test_single_declared_name_wins(self, tmp_path: Path) -> None: - (tmp_path / "env.py").write_text('env = Environment("my-env")\n', encoding="utf-8") - - assert _dry_run(tmp_path)["name"] == "my-env" - - def test_repeated_same_name_is_fine(self, tmp_path: Path) -> None: - (tmp_path / "a.py").write_text('a = Environment("same")\n', encoding="utf-8") - (tmp_path / "b.py").write_text('b = Environment(name="same")\n', encoding="utf-8") - (tmp_path / "c.py").write_text('c = hud.Environment("same")\n', encoding="utf-8") - - assert _dry_run(tmp_path)["name"] == "same" - - def test_nested_source_counts_but_junk_directories_do_not(self, tmp_path: Path) -> None: - nested = tmp_path / "server" / "pkg" - nested.mkdir(parents=True) - (nested / "env.py").write_text('env = Environment("nested")\n', encoding="utf-8") - for junk in (".venv", "node_modules", "__pycache__"): - (tmp_path / junk).mkdir() - (tmp_path / junk / "env.py").write_text( - 'env = Environment("excluded")\n', encoding="utf-8" - ) - (tmp_path / "broken.py").write_text("def broken(:\n", encoding="utf-8") - - assert _dry_run(tmp_path)["name"] == "nested" - - def test_explicit_file_disambiguates_nested_environment(self, tmp_path: Path) -> None: - selected = tmp_path / "env.py" - selected.write_text('env = Environment("adapter")\n', encoding="utf-8") - nested = tmp_path / "source" - nested.mkdir() - (nested / "env.py").write_text('env = Environment("original")\n', encoding="utf-8") - - assert _dry_run(selected)["name"] == "adapter" - assert "Pass --name" in _deploy_name_error(tmp_path) - - def test_explicit_file_without_declaration_does_not_select_another_file( - self, tmp_path: Path - ) -> None: - selected = tmp_path / "selected.py" - selected.write_text("x = 1\n", encoding="utf-8") - (tmp_path / "env.py").write_text('env = Environment("other")\n', encoding="utf-8") - - assert "No environment found" in _deploy_name_error(selected) - - def test_ignores_calls_without_a_literal(self, tmp_path: Path) -> None: - (tmp_path / "env.py").write_text( - 'env = Environment("named")\nother = Environment(name=NAME)\n', - encoding="utf-8", - ) - - assert _dry_run(tmp_path)["name"] == "named" - - def test_name_flag_selects_among_literals(self, tmp_path: Path) -> None: - (tmp_path / "env.py").write_text( - 'actor = Environment("workspace")\nverifier = Environment("judge")\n', - encoding="utf-8", - ) - assert _dry_run(tmp_path, "--name", "judge")["name"] == "judge" - assert "Pass --name" in _deploy_name_error(tmp_path) - - def test_serve_line_does_not_break_ties(self, tmp_path: Path) -> None: - (tmp_path / "Dockerfile").write_text('CMD ["hud", "serve", "env:env"]\n', encoding="utf-8") - (tmp_path / "env.py").write_text('env = Environment("trace-explorer")\n', encoding="utf-8") - (tmp_path / "verify.py").write_text( - 'verify_env = Environment("qa-verifier")\n', encoding="utf-8" - ) - - assert "Pass --name" in _deploy_name_error(tmp_path) - assert _dry_run(tmp_path, "--name", "trace-explorer")["name"] == "trace-explorer" - assert "No environment named 'missing'" in _deploy_name_error(tmp_path, "--name", "missing") - - @pytest.mark.parametrize( - "source", ["x = 1\n", "env = Environment()\n", "env = Environment(name=NAME)\n"] - ) - def test_requires_a_literal(self, tmp_path: Path, source: str) -> None: - (tmp_path / "env.py").write_text(source, encoding="utf-8") - - assert "No environment found" in _deploy_name_error(tmp_path) - - def test_registry_id_does_not_replace_missing_declaration(self, tmp_path: Path) -> None: - (tmp_path / "server.py").write_text("x = 1\n", encoding="utf-8") - - assert "No environment found" in _deploy_name_error(tmp_path, "--registry-id", "r-1") - - def test_registry_id_must_name_the_declared_environment( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - (tmp_path / "env.py").write_text('env = Environment("My Env")\n', encoding="utf-8") - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - - def get(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - assert url.endswith(f"/registry/{_REGISTRY_ID}") - return {"id": _REGISTRY_ID, "name": "other-env"} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", get) - message = _deploy_name_error(tmp_path, "--registry-id", _REGISTRY_ID) - assert "Environment('My Env')" in message - assert "targets 'other-env'" in message - - -class TestBuildContext: - def test_excludes_secrets_and_keeps_source( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - (tmp_path / "main.py").write_text("print('hi')", encoding="utf-8") - (tmp_path / ".env").write_text("SECRET=1", encoding="utf-8") - (tmp_path / ".git").mkdir() - (tmp_path / ".git" / "config").write_text("x", encoding="utf-8") - - names = _archive(tmp_path, monkeypatch) - assert "main.py" in names - assert "env.py" in names - assert ".env" not in names - assert not any(name == ".git" or name.startswith(".git/") for name in names) - - def test_explicit_file_uploads_its_parent_tree( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - selected = tmp_path / "env.py" - selected.write_text('env = Environment("selected")\n', encoding="utf-8") - nested = tmp_path / "source" - nested.mkdir() - (nested / "env.py").write_text('env = Environment("original")\n', encoding="utf-8") - (tmp_path / "compose.yaml").write_text("services: {}\n", encoding="utf-8") - - names = _archive(selected, monkeypatch) - assert {"env.py", "compose.yaml", "source/env.py"} <= names - - def test_preserves_empty_directories( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - (tmp_path / "packages").mkdir() - (tmp_path / "ignored").mkdir() - (tmp_path / ".dockerignore").write_text("ignored/\n", encoding="utf-8") - - names = _archive(tmp_path, monkeypatch) - assert "packages" in names - assert "ignored" not in names - - def test_gitignore_does_not_scope_the_context( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - (tmp_path / ".gitignore").write_text("data/\nbundle.bin\n", encoding="utf-8") - (tmp_path / "bundle.bin").write_text("weights", encoding="utf-8") - (tmp_path / "data").mkdir() - (tmp_path / "data" / "fixture.json").write_text("{}", encoding="utf-8") - - assert {"bundle.bin", "data/fixture.json"} <= _archive(tmp_path, monkeypatch) - - def test_dockerignore_can_reinclude_default_junk( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - (tmp_path / "dist").mkdir() - (tmp_path / "dist" / "app.whl").write_text("wheel", encoding="utf-8") - (tmp_path / "node_modules").mkdir() - (tmp_path / "node_modules" / "dep.js").write_text("x", encoding="utf-8") - (tmp_path / ".dockerignore").write_text("!node_modules\n", encoding="utf-8") - - names = _archive(tmp_path, monkeypatch) - assert "dist/app.whl" in names - assert "node_modules/dep.js" in names - - def test_dockerignore_negations_cannot_reinclude_secrets( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - for name in [".env", ".env.prod", "service.env", ".git/config", "nested/.env"]: - path = tmp_path / name - path.parent.mkdir(exist_ok=True) - path.write_text("secret", encoding="utf-8") - (tmp_path / "keep.pyc").write_text("cache", encoding="utf-8") - (tmp_path / ".dockerignore").write_text("!*\n", encoding="utf-8") - - assert _archive(tmp_path, monkeypatch) == {"nested", "keep.pyc", ".dockerignore", "env.py"} - - def test_dockerignore_skips_comments_and_honors_last_match( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - (tmp_path / "keep.pyc").write_text("x", encoding="utf-8") - (tmp_path / "drop.pyc").write_text("x", encoding="utf-8") - (tmp_path / ".dockerignore").write_text("# comment\n\n*.pyc\n!keep.pyc\n", encoding="utf-8") - - names = _archive(tmp_path, monkeypatch) - assert "keep.pyc" in names - assert "drop.pyc" not in names - - def test_dockerignore_globs_and_directory_patterns( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - (tmp_path / "a.py").write_text("x", encoding="utf-8") - (tmp_path / "a.pyc").write_text("x", encoding="utf-8") - (tmp_path / "pkg").mkdir() - (tmp_path / "pkg" / "a.pyc").write_text("x", encoding="utf-8") - (tmp_path / "node_modules").mkdir() - (tmp_path / "node_modules" / "index.js").write_text("x", encoding="utf-8") - (tmp_path / ".dockerignore").write_text("*.pyc\nnode_modules/\n", encoding="utf-8") - - names = _archive(tmp_path, monkeypatch) - assert "a.py" in names - assert "a.pyc" not in names - assert "pkg/a.pyc" not in names - assert "node_modules/index.js" not in names - - def test_dockerignore_double_star_and_anchored_patterns( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - (tmp_path / "a" / "b" / "c").mkdir(parents=True) - (tmp_path / "a" / "b" / "c" / "cache.tmp").write_text("x", encoding="utf-8") - (tmp_path / "src" / "build").mkdir(parents=True) - (tmp_path / "src" / "build" / "out.o").write_text("x", encoding="utf-8") - (tmp_path / "build").mkdir() - (tmp_path / "build" / "out.o").write_text("x", encoding="utf-8") - (tmp_path / "foo" / "a" / "b").mkdir(parents=True) - (tmp_path / "foo" / "a" / "b" / "bar").write_text("x", encoding="utf-8") - (tmp_path / "foo" / "a" / "bar").write_text("x", encoding="utf-8") - (tmp_path / "foo" / "bar").write_text("x", encoding="utf-8") - (tmp_path / ".dockerignore").write_text("**/*.tmp\n/build\nfoo/**/bar\n", encoding="utf-8") - - names = _archive(tmp_path, monkeypatch) - assert "a/b/c/cache.tmp" not in names - assert "build/out.o" not in names - assert "src/build/out.o" in names - assert "foo/a/b/bar" not in names - assert "foo/bar" not in names - assert "foo/a/bar" not in names - - def test_dockerignore_single_star_does_not_span_directories( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - (tmp_path / "foo" / "a" / "b").mkdir(parents=True) - (tmp_path / "foo" / "a" / "b" / "bar").write_text("x", encoding="utf-8") - (tmp_path / "foo" / "a" / "bar").write_text("x", encoding="utf-8") - (tmp_path / ".dockerignore").write_text("foo/*/bar\n", encoding="utf-8") - - names = _archive(tmp_path, monkeypatch) - assert "foo/a/bar" not in names - assert "foo/a/b/bar" in names - - -class TestDeployEnvironmentVariables: - def test_dry_run_has_no_env_keys_without_sources(self, tmp_path: Path) -> None: - _write_env(tmp_path) - assert _dry_run(tmp_path)["env_var_keys"] == [] - - def test_env_file_keys_appear_in_dry_run(self, tmp_path: Path) -> None: - _write_env(tmp_path) - env_file = tmp_path / "vars.env" - env_file.write_text("# comment\nKEY1=value1\nEMPTY=\nNOEQ\nKEY2=value2\n") - - assert _dry_run(tmp_path, "--env-file", str(env_file))["env_var_keys"] == [ - "EMPTY", - "KEY1", - "KEY2", - ] - - def test_missing_env_file_fails(self, tmp_path: Path) -> None: - _write_env(tmp_path) - result = CliRunner().invoke( - app, - [ - "deploy", - str(tmp_path), - "--dry-run", - "--json", - "--env-file", - str(tmp_path / "missing"), - ], - ) - assert result.exit_code == 1 - assert "Env file not found" in json.loads(result.stdout)["message"] - - def test_env_flags_override_file_on_trigger( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - _write_env(tmp_path) - env_file = tmp_path / "vars.env" - env_file.write_text("KEY1=file_value\n") - captured = _stub_remote_build(monkeypatch) - - result = CliRunner().invoke( - app, - [ - "deploy", - str(tmp_path), - "--json", - "--env-file", - str(env_file), - "--env", - "KEY1=flag_value", - "--env", - "KEY2=new_value", - ], - ) - assert result.exit_code == 0, result.output - assert captured["environment_variables"] == {"KEY1": "flag_value", "KEY2": "new_value"} - - def test_env_flag_invalid_format(self, tmp_path: Path) -> None: - _write_env(tmp_path) - result = CliRunner().invoke( - app, ["deploy", str(tmp_path), "--dry-run", "--json", "--no-env", "--env", "INVALID"] - ) - assert result.exit_code == 2 - assert "Invalid --env format" in json.loads(result.stdout)["message"] - - def test_build_secret_from_env_reaches_trigger( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - _write_env(tmp_path) - monkeypatch.setenv("GITHUB_TOKEN", "tok") - captured = _stub_remote_build(monkeypatch) - - result = CliRunner().invoke( - app, - [ - "deploy", - str(tmp_path), - "--json", - "--no-env", - "--secret", - "id=GITHUB_TOKEN,env=GITHUB_TOKEN", - ], - ) - assert result.exit_code == 0, result.output - assert captured["build_secrets"] == {"GITHUB_TOKEN": "tok"} - - -class TestRuntimeConfigFile: - def test_compose_file_is_not_inferred_as_runtime_config( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - _write_env(tmp_path, "compose-env") - (tmp_path / "compose.yaml").write_text( - "services:\n main:\n image: alpine\n", - encoding="utf-8", - ) - captured = _stub_remote_build(monkeypatch) - - result = CliRunner().invoke(app, ["deploy", str(tmp_path), "--json", "--no-env"]) - assert result.exit_code == 0, result.output - assert "runtime_config" not in captured - - def test_runtime_config_flag_reaches_trigger( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - _write_env(tmp_path) - config_path = tmp_path / "runtime.json" - config_path.write_text( - json.dumps({"limits": {"startup_timeout_s": 300}}), - encoding="utf-8", - ) - captured = _stub_remote_build(monkeypatch) - - result = CliRunner().invoke( - app, - [ - "deploy", - str(tmp_path), - "--json", - "--no-env", - "--runtime-config", - str(config_path), - ], - ) - assert result.exit_code == 0, result.output - assert captured["runtime_config"] == {"limits": {"startup_timeout_s": 300}} - - def test_runtime_config_uses_sdk_shape( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - _write_env(tmp_path) - config_path = tmp_path / "runtime.json" - config_path.write_text( - json.dumps( - { - "resources": {"gpu": {"type": "A10G", "count": 2}}, - "limits": {"startup_timeout_s": 300}, - } - ), - encoding="utf-8", - ) - captured = _stub_remote_build(monkeypatch) - - result = CliRunner().invoke( - app, - ["deploy", str(tmp_path), "--json", "--no-env", "--runtime-config", str(config_path)], - ) - assert result.exit_code == 0, result.output - assert captured["runtime_config"] == { - "resources": {"gpu": {"type": "A10G", "count": 2}}, - "limits": {"startup_timeout_s": 300}, - } - - def test_runtime_config_preserves_null_override( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - _write_env(tmp_path) - config_path = tmp_path / "runtime.json" - config_path.write_text(json.dumps({"resources": None}), encoding="utf-8") - captured = _stub_remote_build(monkeypatch) - - result = CliRunner().invoke( - app, - ["deploy", str(tmp_path), "--json", "--no-env", "--runtime-config", str(config_path)], - ) - assert result.exit_code == 0, result.output - assert captured["runtime_config"] == {"resources": None} - - def test_runtime_config_resolves_compose_project_from_config_directory( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - _write_env(tmp_path) - project = tmp_path / "project" - project.mkdir() - (project / "compose.json").write_text('{"services":{"main":{"image":"postgres:16"}}}') - config_path = tmp_path / "runtime.json" - config_path.write_text('{"compose":{"document":"project/compose.json","root":"."}}') - captured = _stub_remote_build(monkeypatch) - - result = CliRunner().invoke( - app, - ["deploy", str(tmp_path), "--json", "--no-env", "--runtime-config", str(config_path)], - ) - assert result.exit_code == 0, result.output - assert captured["runtime_config"]["compose"]["root"] == { - "compose_path": "project/compose.json" - } - - def test_runtime_config_rejects_empty_object(self, tmp_path: Path) -> None: - _write_env(tmp_path) - config_path = tmp_path / "runtime.json" - config_path.write_text("{}", encoding="utf-8") - - result = CliRunner().invoke( - app, - [ - "deploy", - str(tmp_path), - "--dry-run", - "--json", - "--no-env", - "--runtime-config", - str(config_path), - ], - ) - assert result.exit_code == 2 - assert "at least one field" in json.loads(result.stdout)["message"] - - def test_runtime_config_rejects_unknown_fields(self, tmp_path: Path) -> None: - _write_env(tmp_path) - config_path = tmp_path / "runtime.json" - config_path.write_text(json.dumps({"provider_config": {}}), encoding="utf-8") - - result = CliRunner().invoke( - app, - [ - "deploy", - str(tmp_path), - "--dry-run", - "--json", - "--no-env", - "--runtime-config", - str(config_path), - ], - ) - assert result.exit_code == 2 - assert "provider_config" in json.loads(result.stdout)["message"] - - -class TestPlanDeploy: - def test_no_api_key_error(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Test error when no API key is set.""" - from hud.settings import settings - - (tmp_path / "Dockerfile.hud").write_text("FROM python:3.12") - monkeypatch.setattr(settings, "api_key", None) - - result = CliRunner().invoke(app, ["deploy", str(tmp_path), "--json"]) - assert result.exit_code == 1 - assert json.loads(result.stdout)["error"] == "permission_denied" - - def test_empty_tree_fails_on_missing_name(self, tmp_path: Path) -> None: - result = CliRunner().invoke(app, ["deploy", str(tmp_path), "--json"]) - assert result.exit_code == 2 - payload = json.loads(result.stdout) - assert "No environment found" in payload["message"] - assert payload["suggestion"] - - def test_missing_license_does_not_block_dry_run(self, tmp_path: Path) -> None: - _write_env(tmp_path) - (tmp_path / "pyproject.toml").write_text('[project]\nlicense = {file = "LICENSE"}\n') - - assert _dry_run(tmp_path)["success"] is True - - -def test_hud_row_is_polled_only_while_pending( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _write_env(tmp_path) - _stub_remote_build(monkeypatch) - statuses = ["MIGRATING", "SUCCEEDED"] - - async def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - if url.endswith("/upload-url"): - return {"upload_url": "https://upload.example", "build_id": "build-1"} - if url.endswith("/trigger"): - return {"id": "build-1", "registry_id": _REGISTRY_ID} - if url.endswith("/status"): - return {"status": statuses.pop(0)} - raise AssertionError(url) - - slept = AsyncMock() - monkeypatch.setattr("hud.utils.platform.make_request", request) - monkeypatch.setattr("hud.cli.deploy.asyncio.sleep", slept) - - result = CliRunner().invoke(app, ["deploy", str(tmp_path), "--json", "--no-env"]) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["status"] == "SUCCEEDED" - assert statuses == [] - slept.assert_awaited() - - -class TestDeployAsync: - """Tests for the remote upload/trigger path.""" - - def test_upload_url_failure(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Test handling of upload URL failure.""" - from hud.utils.exceptions import HudRequestError - - _write_env(tmp_path) - monkeypatch.setattr( - "hud.utils.platform.make_request", - AsyncMock(side_effect=HudRequestError("Unauthorized", status_code=401)), - ) - - result = CliRunner().invoke(app, ["deploy", str(tmp_path), "--json", "--no-env"]) - assert result.exit_code == 1 - payload = json.loads(result.stdout) - assert payload["error"] == "permission_denied" - assert payload["message"] == "Unauthorized" - - def test_upload_url_network_error( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """Test handling of network error during upload URL fetch.""" - _write_env(tmp_path) - monkeypatch.setattr( - "hud.utils.platform.make_request", - AsyncMock(side_effect=Exception("Network error")), - ) - - result = CliRunner().invoke(app, ["deploy", str(tmp_path), "--json", "--no-env"]) - assert result.exit_code == 1 - assert json.loads(result.stdout)["message"] == "Network error" - - def test_trigger_build_sends_resolved_project( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """A resolved placement reaches the platform as project_id.""" - _write_env(tmp_path, "test-env") - captured = _stub_remote_build(monkeypatch) - - result = CliRunner().invoke( - app, ["deploy", str(tmp_path), "--json", "--no-env", "--project", _PROJECT_ID] - ) - assert result.exit_code == 0, result.output - assert captured["project_id"] == _PROJECT_ID - - def test_trigger_build_omits_project_for_the_team_default( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """The zero-config deploy stays byte-identical to before projects existed.""" - _write_env(tmp_path, "test-env") - captured = _stub_remote_build(monkeypatch) - - result = CliRunner().invoke(app, ["deploy", str(tmp_path), "--json", "--no-env"]) - assert result.exit_code == 0, result.output - assert "project_id" not in captured - - -class TestDeployCommand: - """Tests for deploy_command typer function.""" - - def test_command_exists(self) -> None: - """Test deploy_command function exists and is callable.""" - from hud.cli.deploy import deploy_command - - assert callable(deploy_command) - - def test_command_docstring(self) -> None: - """Test deploy_command has proper docstring.""" - from hud.cli.deploy import deploy_command - - assert deploy_command.__doc__ is not None - assert "Deploy" in deploy_command.__doc__ - - -@pytest.fixture(autouse=True) -def authenticated_scope(monkeypatch: pytest.MonkeyPatch) -> None: - def get(method: str, url: str, **kwargs: object) -> dict[str, Any]: - if url.endswith("/auth/me"): - return { - "user_id": "11111111-1111-4111-8111-111111111111", - "team_id": "22222222-2222-4222-8222-222222222222", - } - if "/projects/" in url: - return { - "id": url.rsplit("/", 1)[-1], - "name": "chosen", - "capabilities": {"create": True}, - } - raise AssertionError(url) - - monkeypatch.setattr("hud.utils.platform.make_request_sync", get) - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - monkeypatch.setattr("hud.settings.settings.default_project", None) - - -@pytest.mark.parametrize( - "override", - [ - [], - ["--registry-id", "33333333-3333-4333-8333-333333333333"], - ["--project", "44444444-4444-4444-8444-444444444444"], - ], -) -@pytest.mark.parametrize("stream_status", ["SUCCEEDED", "UNKNOWN", None]) -def test_deploy_lifecycle_preserves_links( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, override: list[str], stream_status: str | None -) -> None: - env = tmp_path / "environment" - env.mkdir() - (env / "env.py").write_text('env = Environment("example")') - (env / "Dockerfile").write_text("FROM python:3.12") - (env / "pyproject.toml").write_text('[project]\nname="example"\nversion="0"') - (env / ".env").write_text("SECRET=test-secret-value") - identity = { - "user_id": "11111111-1111-4111-8111-111111111111", - "team_id": "22222222-2222-4222-8222-222222222222", - } - registry_id = "55555555-5555-4555-8555-555555555555" - requests: list[dict[str, Any]] = [] - - def get(method: str, url: str, **kwargs): - if url.endswith("/auth/me"): - return identity - if "/registry/" in url: - return {"id": url.rsplit("/", 1)[-1], "name": "example"} - if "/projects/" in url: - return { - "id": url.rsplit("/", 1)[-1], - "name": "chosen", - "capabilities": {"create": True}, - } - raise AssertionError(url) - - status_reads = 0 - - async def request(method: str, url: str, **kwargs): - nonlocal status_reads - if url.endswith("/upload-url"): - return {"upload_url": "https://upload.example", "build_id": "build-1"} - if url.endswith("/trigger"): - requests.append(kwargs["json"]) - return {"id": "build-1", "registry_id": kwargs["json"].get("registry_id", registry_id)} - if url.endswith("/status"): - status_reads += 1 - return {"status": "SUCCEEDED"} - raise AssertionError(url) - - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - monkeypatch.setattr("hud.utils.platform.make_request_sync", get) - monkeypatch.setattr("hud.utils.platform.make_request", request) - monkeypatch.setattr("hud.cli.deploy.httpx.AsyncClient", _FakeHttpxClient) - websocket = MagicMock() - websocket.__aiter__.return_value = ( - [ - json.dumps( - { - "type": "complete", - "final_status": stream_status, - "message": f"Build {stream_status}", - } - ) - ] - if stream_status - else [] - ) - connection = MagicMock() - connection.__aenter__.return_value = websocket - monkeypatch.setattr("hud.cli.deploy.websockets.connect", lambda *a, **k: connection) - monkeypatch.setattr("hud.cli.deploy.asyncio.sleep", AsyncMock()) - prompts = MagicMock(return_value=True) - monkeypatch.setattr(HUDConsole, "confirm", prompts) - first = _TtyCliRunner().invoke(app, ["deploy", str(env), "--json"]) - assert first.exit_code == 0, first.output - assert json.loads(first.stdout)["registry_id"] == registry_id - assert "test-secret-value" not in first.stdout - state = DirectoryState(AuthScope.resolve(PlatformClient.from_settings()), env) - before = state.load() - assert before.registry_id is not None - second = _TtyCliRunner().invoke(app, ["deploy", str(env), "--json", *override]) - assert second.exit_code == 0, second.output - assert state.load() == before - assert status_reads == 2 - assert (env / ".hud" / "config.json").exists() - assert requests[0]["environment_variables"] == {"SECRET": "test-secret-value"} - if not override: - assert prompts.call_count == 1 - assert "registry_id" not in requests[-1] - assert "environment_variables" not in requests[-1] - elif override[:1] == ["--registry-id"]: - assert prompts.call_count == 1 - assert "environment_variables" not in requests[-1] - - -def test_deploy_dry_run_has_no_prompt_or_writes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from typer.testing import CliRunner - - from hud.cli.__main__ import app - - env = tmp_path / "environment" - env.mkdir() - (env / "env.py").write_text('env = Environment("example")') - (env / "Dockerfile").write_text("FROM python:3.12") - (env / ".env").write_text("SECRET=hidden") - legacy = env / ".hud" / "deploy.json" - legacy.parent.mkdir() - legacy.write_text('{"registryId":"old","syncEnv":true}') - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - monkeypatch.setattr(HUDConsole, "confirm", lambda *a, **k: pytest.fail("dry-run prompted")) - result = CliRunner().invoke(app, ["deploy", str(env), "--dry-run", "--json"]) - assert result.exit_code == 0, result.output - plan = json.loads(result.stdout) - assert plan["dotenv_pending"] is True - assert plan["env_var_keys"] == [] - assert plan["dry_run"] is True - assert plan["name"] == "example" - assert legacy.read_text() == '{"registryId":"old","syncEnv":true}' - assert not (legacy.parent / "config.json").exists() - assert not (Path.home() / ".hud").exists() diff --git a/hud/cli/tests/test_eval.py b/hud/cli/tests/test_eval.py deleted file mode 100644 index e3fb48862..000000000 --- a/hud/cli/tests/test_eval.py +++ /dev/null @@ -1,954 +0,0 @@ -"""Tests for ``hud eval``.""" - -from __future__ import annotations - -import json -import sys -from dataclasses import dataclass, field -from pathlib import Path -from typing import TYPE_CHECKING, Any -from unittest.mock import MagicMock - -import pytest -from typer.testing import CliRunner - -from hud.agents import OpenAIAgent -from hud.agents.types import ClaudeConfig -from hud.cli import eval as eval_mod -from hud.cli.__main__ import app -from hud.cli.eval import EvalConfig -from hud.eval import ( - DaytonaRuntime, - DockerRuntime, - Grade, - HostedRuntime, - HUDRuntime, - Job, - ModalRuntime, - Run, - Runtime, - RuntimeConfig, - Task, - Taskset, -) -from hud.settings import settings -from hud.utils.gateway import GatewayModelInfo -from hud.utils.hud_console import HUDConsole - -if TYPE_CHECKING: - from collections.abc import Callable, Iterator - -_TASKS_PY = """\ -from hud import Environment - -env = Environment("demo") - - -@env.template(id="solve") -async def solve(n: int = 0): - yield f"solve {n}" - yield 1.0 - - -tasks = [solve(n=0), solve(n=1)] -""" - -_BEDROCK_ARN = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/my-profile" -_CONTAINER_ROW = '{"env": "demo", "id": "solve", "runtime_config": {"image": "example:latest"}}' - - -@dataclass -class _EvalCli: - """``hud eval`` in a scratch project; ``Taskset.run`` is captured, not executed.""" - - taskset: Taskset | None = None - agent: Any = None - kwargs: dict[str, Any] = field(default_factory=dict) - job: Job = field(default_factory=lambda: Job(id="job-1", name="demo")) - - def invoke(self, *args: str, exit_code: int = 0) -> dict[str, Any]: - result = CliRunner().invoke(app, ["eval", *args, "--json"]) - assert result.exit_code == exit_code, result.output - return json.loads(result.stdout) - - -@pytest.fixture -def eval_cli(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> _EvalCli: - monkeypatch.chdir(tmp_path) - (tmp_path / "tasks.py").write_text(_TASKS_PY, encoding="utf-8") - monkeypatch.setattr(settings, "api_key", "sk-hud-test") - for key in ("anthropic_api_key", "openai_api_key", "gemini_api_key"): - monkeypatch.setattr(settings, key, None) - cli = _EvalCli() - - async def fake_run(self: Taskset, agent: Any, **kwargs: Any) -> Job: - cli.taskset, cli.agent, cli.kwargs = self, agent, kwargs - return cli.job - - monkeypatch.setattr(Taskset, "run", fake_run) - return cli - - -def _catalog(*rows: tuple[str, str, str, str]) -> list[GatewayModelInfo]: - """(sdk_agent_type, name, model_name, created_at) rows, as the gateway catalog returns them.""" - return [ - GatewayModelInfo( - id=model_name, - name=name, - model_name=model_name, - sdk_agent_type=agent, - created_at=created, - ) - for agent, name, model_name, created in rows - ] - - -def _picker(agent: str, pick_model: Callable[[list[Any]], Any]) -> Any: - """A ``HUDConsole.select`` that answers the agent prompt with ``agent`` and the model - prompt via ``pick_model(choices)``.""" - - def select(self: HUDConsole, message: str, choices: Any, **_: Any) -> Any: - if message == "Select an agent:": - return agent - assert message == "Select a model:" - return pick_model(list(choices)) - - return select - - -# ─── config file contract ─────────────────────────────────────────────── - - -def test_load_missing_returns_defaults_without_writing(tmp_path: Path) -> None: - path = tmp_path / ".hud_eval.toml" - assert EvalConfig.load(path) == EvalConfig() - assert not path.exists() - - -def test_load_parses_eval_and_agent_sections(tmp_path: Path) -> None: - path = tmp_path / ".hud_eval.toml" - path.write_text( - '[eval]\nagent = "openai"\nmax_steps = 5\nruntime = "hosted"\n\n' - '[openai]\nmodel = "gpt-4o"\n', - encoding="utf-8", - ) - cfg = EvalConfig.load(path) - assert cfg.agent_type is not None and cfg.agent_type.value == "openai" - assert cfg.max_steps == 5 - assert cfg.runtime == "hosted" - assert cfg.agent_config == {"openai": {"model": "gpt-4o"}} - - -@pytest.mark.parametrize( - "contents, source, flags, expected", - [ - ("remote = true", "tasks.py", [], "hosted"), - ("remote = false", "tasks.py", [], "local"), - ("remote = false", "My Tasks", [], "hosted"), - ('remote = true\nruntime = "local"', "tasks.py", [], "local"), - ('remote = false\nruntime = "hud"', "tasks.py", [], "hud"), - ("remote = true", "tasks.py", ["--runtime", "local"], "local"), - ("remote = false", "tasks.py", ["--remote"], "hosted"), - ], -) -def test_legacy_remote_config_resolves_placement( - eval_cli: _EvalCli, - tmp_path: Path, - contents: str, - source: str, - flags: list[str], - expected: str, -) -> None: - path = tmp_path / ".hud_eval.toml" - path.write_text(f"[eval]\n{contents}\n") - result = eval_cli.invoke(source, "openai", *flags, "--dry-run") - assert result["runtime"] == expected - assert "remote" not in EvalConfig.load(path).model_dump() - - -def test_legacy_remote_config_uses_hosted_runtime(eval_cli: _EvalCli, tmp_path: Path) -> None: - (tmp_path / ".hud_eval.toml").write_text("[eval]\nremote = true\n") - eval_cli.invoke("tasks.py", "openai", "--yes") - assert isinstance(eval_cli.kwargs["runtime"], HostedRuntime) - - -def test_load_resolves_env_var_placeholders( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("MY_EVAL_MODEL", "gpt-4o") - path = tmp_path / ".hud_eval.toml" - path.write_text('[openai]\nmodel = "${MY_EVAL_MODEL}"\n', encoding="utf-8") - assert EvalConfig.load(path).agent_config["openai"]["model"] == "gpt-4o" - - -def test_load_rejects_unset_env_var(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("HUD_EVAL_TEST_UNSET", raising=False) - path = tmp_path / ".hud_eval.toml" - path.write_text('[openai]\nmodel = "${HUD_EVAL_TEST_UNSET}"\n', encoding="utf-8") - with pytest.raises(ValueError, match=r"\$\{HUD_EVAL_TEST_UNSET\} is not set"): - EvalConfig.load(path) - - -def test_load_treats_unset_settings_as_unset( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(settings, "openai_api_key", None) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - path = tmp_path / ".hud_eval.toml" - path.write_text('[openai_compatible]\napi_key = "${openai_api_key}"\n', encoding="utf-8") - with pytest.raises(ValueError, match=r"\$\{openai_api_key\} is not set"): - EvalConfig.load(path) - - -@pytest.mark.parametrize( - "contents, match", - [ - ('[eval]\nmodle = "x"\n', "modle"), - ('[eval]\nremote = "false"\n', "remote must be a boolean"), - ('[eval]\nagent = "not-an-agent"\n', "claude"), - ('[eval]\nruntime = "cloud"\n', "'local', 'hud', 'hosted', 'docker', 'modal' or 'daytona'"), - ("[other]\nx = 1\n", "unknown sections: other"), - ], -) -def test_load_rejects_invalid_configuration(tmp_path: Path, contents: str, match: str) -> None: - path = tmp_path / ".hud_eval.toml" - path.write_text(contents, encoding="utf-8") - with pytest.raises(ValueError, match=match): - EvalConfig.load(path) - - -def test_placement_defaults_from_source(eval_cli: _EvalCli) -> None: - assert eval_cli.invoke("tasks.py", "openai", "--dry-run")["runtime"] == "local" - assert eval_cli.invoke("My Tasks", "openai", "--dry-run")["runtime"] == "hosted" - assert ( - eval_cli.invoke("My Tasks", "openai", "--runtime", "hud", "--dry-run")["runtime"] == "hud" - ) - - -def test_local_against_a_platform_taskset_is_refused(eval_cli: _EvalCli, monkeypatch) -> None: - monkeypatch.setattr( - Taskset, - "from_api", - classmethod(lambda cls, name: Taskset(name, [Task(env="demo", id="a")])), - ) - payload = eval_cli.invoke("My Tasks", "openai", "--runtime", "local", "--yes", exit_code=2) - assert "My Tasks is a platform taskset" in payload["message"] - assert "--remote" in payload["message"] - assert eval_cli.taskset is None - - -@pytest.mark.parametrize("flags", [["--runtime", "hud"], ["--remote"], ["--gateway"]]) -def test_platform_features_require_hud_key(eval_cli: _EvalCli, monkeypatch, flags) -> None: - monkeypatch.setattr(settings, "api_key", None) - payload = eval_cli.invoke("tasks.py", "gemini", *flags, "--yes", exit_code=1) - assert payload["error"] == "permission_denied" - assert eval_cli.taskset is None - - -def test_openai_compatible_requires_a_model(eval_cli: _EvalCli) -> None: - payload = eval_cli.invoke("tasks.py", "openai_compatible", "--yes", exit_code=2) - assert "Model name is required" in payload["message"] - - -def test_cli_model_wins_over_config_section(eval_cli: _EvalCli, tmp_path: Path) -> None: - """``--model`` and the TOML section are passed to the agent verbatim; the positional - agent argument is what resolves short names through the catalog.""" - (tmp_path / ".hud_eval.toml").write_text( - '[openai_compatible]\nmodel = "moonshotai/kimi-k2.6"\n' - "completion_kwargs = { temperature = 0.5 }\n", - encoding="utf-8", - ) - eval_cli.invoke("tasks.py", "openai_compatible", "--yes") - assert eval_cli.agent.config.model == "moonshotai/kimi-k2.6" - assert eval_cli.agent.config.completion_kwargs == {"temperature": 0.5} - - eval_cli.invoke( - "tasks.py", "openai_compatible", "-m", "z-ai/glm-5.2", "--max-steps", "7", "--yes" - ) - assert eval_cli.agent.config.model == "z-ai/glm-5.2" - assert eval_cli.agent.config.max_steps == 7 - assert eval_cli.agent.config.completion_kwargs == {"temperature": 0.5} - - -# ─── command: planning and validation ────────────────────────────────── - - -@pytest.mark.parametrize("args", [[], ["tasks.json", "claude"]]) -def test_dry_run_does_not_prompt_or_write( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, args: list[str] -) -> None: - monkeypatch.chdir(tmp_path) - (tmp_path / "tasks.json").write_text("[]") - monkeypatch.setattr(HUDConsole, "select", lambda *a, **k: pytest.fail("dry-run prompted")) - result = CliRunner().invoke(app, ["eval", *args, "--dry-run", "--json"]) - assert result.exit_code == (0 if args else 2), result.output - payload = json.loads(result.stdout) - if args: - assert payload["runtime"] == "local" - assert payload["remote"] is False - assert payload["agent"] == "claude" - else: - assert payload["error"] == "usage" - assert not (tmp_path / ".hud_eval.toml").exists() - - -@pytest.mark.parametrize("contents", ["[eval", '[eval]\nagent="invalid"\n']) -def test_invalid_configuration_is_a_structured_error(tmp_path, monkeypatch, contents) -> None: - monkeypatch.chdir(tmp_path) - config = tmp_path / ".hud_eval.toml" - config.write_text(contents) - result = CliRunner().invoke(app, ["eval", "tasks.json", "openai", "--dry-run", "--json"]) - assert result.exit_code == 2, result.output - assert json.loads(result.stdout)["error"] == "usage" - assert config.read_text() == contents - - -def test_full_expands_to_all_auto_respond_and_100_steps(eval_cli: _EvalCli) -> None: - payload = eval_cli.invoke("tasks.py", "openai", "--full", "--dry-run") - assert payload["all"] is True - assert payload["max_steps"] == 100 - eval_cli.invoke("tasks.py", "openai", "--full", "--yes") - assert eval_cli.agent.config.auto_respond is True - assert eval_cli.agent.config.max_steps == 100 - - -def test_runtime_and_remote_flags_conflict(eval_cli: _EvalCli) -> None: - payload = eval_cli.invoke("tasks.py", "openai", "--runtime", "hud", "--remote", exit_code=2) - assert payload["error"] == "usage" - assert "mutually exclusive" in payload["message"] - - -def test_config_flag_lands_in_agent_config(eval_cli: _EvalCli) -> None: - eval_cli.invoke("tasks.py", "claude", "--config", "max_tokens=100", "--yes") - assert eval_cli.agent.config.max_tokens == 100 - eval_cli.invoke("tasks.py", "claude", "-c", "claude.max_tokens=200", "--yes") - assert eval_cli.agent.config.max_tokens == 200 - - -@pytest.mark.parametrize( - "args, match", - [ - (["tasks.py", "claude", "--config", "max_tokens"], "key=value"), - (["tasks.py", "--config", "max_tokens=1"], "needs an agent"), - (["tasks.py", "claude", "--config", "robot.max_tokens=1"], "not a valid AgentType"), - ], -) -def test_malformed_config_flag_is_a_usage_error( - eval_cli: _EvalCli, args: list[str], match: str -) -> None: - payload = eval_cli.invoke(*args, "--yes", exit_code=2) - assert payload["error"] == "usage" - assert match in payload["message"] - - -def test_gateway_model_alias_selects_agent_and_model(eval_cli: _EvalCli, monkeypatch) -> None: - from hud.utils.gateway import GatewayModelInfo, GatewayProviderInfo - - model = GatewayModelInfo( - id="z-ai/glm-5.2", - model_name="z-ai/glm-5.2", - sdk_agent_type="openai_compatible", - provider=GatewayProviderInfo(name="openai"), - ) - monkeypatch.setattr("hud.utils.gateway.list_gateway_models", lambda *_: [model]) - eval_cli.invoke("tasks.py", "glm-5.2", "--yes") - assert type(eval_cli.agent).__name__ == "OpenAIChatAgent" - assert eval_cli.agent.config.model == "z-ai/glm-5.2" - - -# ─── command: task selection and placement ───────────────────────────── - - -def test_default_runs_only_the_first_task(eval_cli: _EvalCli) -> None: - eval_cli.invoke("tasks.py", "openai", "--yes") - assert eval_cli.taskset is not None - assert [task.args for task in eval_cli.taskset] == [{"n": 0}] - eval_cli.invoke("tasks.py", "openai", "--all", "--yes") - assert eval_cli.taskset is not None and len(eval_cli.taskset) == 2 - eval_cli.invoke("tasks.py", "openai", "--task-ids", "1", "--yes") - assert eval_cli.taskset is not None - assert [task.args for task in eval_cli.taskset] == [{"n": 1}] - - -def test_unknown_task_ids_fail(eval_cli: _EvalCli) -> None: - payload = eval_cli.invoke("tasks.py", "openai", "--task-ids", "nope", "--yes", exit_code=2) - assert "No tasks matching: nope" in payload["message"] - - -def test_group_and_concurrency_reach_the_scheduler(eval_cli: _EvalCli) -> None: - eval_cli.invoke("tasks.py", "openai", "--group", "3", "--max-concurrent", "2", "--yes") - assert eval_cli.kwargs["group"] == 3 - assert eval_cli.kwargs["max_concurrent"] == 2 - - -def test_local_placement_routes_each_row(eval_cli: _EvalCli, tmp_path: Path, monkeypatch) -> None: - docker = MagicMock(name="docker") - subprocesses: dict[object, MagicMock] = {} - - def subprocess_runtime(source: object) -> MagicMock: - return subprocesses.setdefault(source, MagicMock(name=f"subprocess({source})")) - - monkeypatch.setattr(eval_mod, "DockerRuntime", lambda: docker) - monkeypatch.setattr(eval_mod, "SubprocessRuntime", subprocess_runtime) - image = Task(env="image", id="run", runtime_config=RuntimeConfig(image="example:latest")) - data_row = Task(env="demo", id="plain") - (tmp_path / "mixed.py").write_text( - _TASKS_PY + "from hud.eval import RuntimeConfig, Task\n" - "tasks.append(Task(env='image', id='run', " - "runtime_config=RuntimeConfig(image='example:latest')))\n", - encoding="utf-8", - ) - try: - eval_cli.invoke("mixed.py", "openai", "--all", "--yes") - finally: - sys.modules.pop("mixed", None) - placement = eval_cli.kwargs["runtime"] - assert eval_cli.taskset is not None - bound = next(iter(eval_cli.taskset)) - - assert placement(bound) is subprocesses[(tmp_path / "mixed.py").resolve()].return_value - assert placement(image) is docker.return_value - # A row without a bound env is served from the tasks file's directory. - assert placement(data_row) is subprocesses[tmp_path.resolve()].return_value - - -def test_json_rows_are_served_from_their_directory( - eval_cli: _EvalCli, tmp_path: Path, monkeypatch -) -> None: - sources: list[object] = [] - monkeypatch.setattr( - eval_mod, "SubprocessRuntime", lambda source: sources.append(source) or MagicMock() - ) - (tmp_path / "rows.json").write_text( - '[{"env": "demo", "id": "a"}, {"env": "demo", "id": "b"}]', encoding="utf-8" - ) - eval_cli.invoke("rows.json", "openai", "--all", "--yes") - assert eval_cli.taskset is not None and len(eval_cli.taskset) == 2 - assert sources == [tmp_path.resolve()] - - -def test_explicit_placements(eval_cli: _EvalCli) -> None: - eval_cli.invoke("tasks.py", "openai", "--runtime", "hud", "--yes") - assert isinstance(eval_cli.kwargs["runtime"], HUDRuntime) - eval_cli.invoke("tasks.py", "openai", "--remote", "--yes") - assert isinstance(eval_cli.kwargs["runtime"], HostedRuntime) - eval_cli.invoke("tasks.py", "openai", "--runtime", "hosted", "--yes") - assert isinstance(eval_cli.kwargs["runtime"], HostedRuntime) - eval_cli.invoke("tasks.py", "openai", "--runtime", "tcp://127.0.0.1:7000", "--yes") - assert eval_cli.kwargs["runtime"] == Runtime("tcp://127.0.0.1:7000") - for name, cls in ( - ("docker", DockerRuntime), - ("modal", ModalRuntime), - ("daytona", DaytonaRuntime), - ): - eval_cli.invoke("tasks.py", "openai", "--runtime", name, "--yes") - assert isinstance(eval_cli.kwargs["runtime"], cls) - - -def test_unknown_runtime_lists_the_choices(eval_cli: _EvalCli) -> None: - payload = eval_cli.invoke("tasks.py", "openai", "--runtime", "cloud", "--yes", exit_code=2) - assert "'local', 'hud', 'hosted', 'docker', 'modal' or 'daytona'" in payload["message"] - assert "URL scheme should be 'tcp'" not in payload["message"] # 'cloud' is not url-shaped - payload = eval_cli.invoke("tasks.py", "openai", "--runtime", "http://x:1", "--yes", exit_code=2) - assert "URL scheme should be 'tcp'" in payload["message"] - - -def test_python_task_source_loads_on_main_thread(eval_cli: _EvalCli, tmp_path: Path) -> None: - marker = tmp_path / "main-thread.txt" - (tmp_path / "probe.py").write_text( - "import threading\nfrom pathlib import Path\n" - f"Path({str(marker)!r}).write_text(" - "str(threading.current_thread() is threading.main_thread()))\n" - "tasks = []\n", - encoding="utf-8", - ) - try: - payload = eval_cli.invoke("probe.py", "openai", "--yes", exit_code=2) - finally: - sys.modules.pop("probe", None) - assert "No runnable Tasks" in payload["message"] - assert marker.read_text(encoding="utf-8") == "True" - - -# ─── command: agent construction ─────────────────────────────────────── - - -@pytest.mark.parametrize( - "agent_type, key_attr, factory, client_attr", - [ - ("openai", "openai_api_key", "hud.utils.gateway.AsyncOpenAI", "openai_client"), - ("claude", "anthropic_api_key", "anthropic.AsyncAnthropic", "anthropic_client"), - ("gemini", "gemini_api_key", "google.genai.Client", "gemini_client"), - ], -) -@pytest.mark.parametrize( - "force_gateway, provider_key", [(False, "provider-key"), (False, None), (True, "provider-key")] -) -def test_provider_key_wins_unless_gateway_is_forced( - eval_cli: _EvalCli, - monkeypatch, - agent_type, - key_attr, - factory, - client_attr, - force_gateway, - provider_key, -) -> None: - monkeypatch.setattr(settings, key_attr, provider_key) - direct = MagicMock(return_value=object()) - gateway = MagicMock(return_value=object()) - monkeypatch.setattr(factory, direct) - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", gateway) - flags = ["--gateway"] if force_gateway else [] - eval_cli.invoke("tasks.py", agent_type, *flags, "--yes") - if provider_key and not force_gateway: - direct.assert_called_once() - assert direct.call_args.kwargs["api_key"] == provider_key - gateway.assert_not_called() - assert getattr(eval_cli.agent, client_attr) is direct.return_value - else: - gateway.assert_called_once() - direct.assert_not_called() - assert getattr(eval_cli.agent, client_attr) is gateway.return_value - - -def test_hosted_agent_keeps_client_out_of_serialized_config( - eval_cli: _EvalCli, monkeypatch -) -> None: - monkeypatch.setattr(settings, "openai_api_key", "provider-key") - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", MagicMock(return_value=object())) - eval_cli.invoke("tasks.py", "openai", "--remote", "--yes") - assert eval_cli.agent.config.model_client is None - assert "model_client" not in eval_cli.agent.dump() - - -def test_openai_compatible_routes_through_gateway_despite_openai_key( - eval_cli: _EvalCli, monkeypatch -) -> None: - """A third-party chat model is not an OpenAI model: OPENAI_API_KEY must not claim it.""" - monkeypatch.setattr(settings, "openai_api_key", "provider-key") - gateway = MagicMock(return_value=object()) - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", gateway) - eval_cli.invoke("tasks.py", "openai_compatible", "--model", "MiniMax-M3", "--yes") - gateway.assert_called_once_with("openai") - assert eval_cli.agent.oai is gateway.return_value - assert eval_cli.agent.config.base_url is None - - -@pytest.mark.parametrize("force_gateway", [False, True]) -def test_openai_compatible_custom_endpoint_is_used_unless_gateway_is_forced( - eval_cli: _EvalCli, monkeypatch, force_gateway -) -> None: - client = MagicMock(return_value=object()) - gateway = MagicMock(return_value=object()) - monkeypatch.setattr("hud.agents.openai_compatible.agent.AsyncOpenAI", client) - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", gateway) - eval_cli.invoke( - "tasks.py", - "openai_compatible", - "-m", - "custom", - "-c", - "api_key=custom-key", - "-c", - "base_url=https://custom.example", - *(["--gateway"] if force_gateway else []), - "--yes", - ) - if force_gateway: - gateway.assert_called_once_with("openai") - client.assert_not_called() - assert eval_cli.agent.oai is gateway.return_value - else: - client.assert_called_once_with(api_key="custom-key", base_url="https://custom.example") - gateway.assert_not_called() - assert eval_cli.agent.oai is client.return_value - - -def test_bedrock_arn_in_config_selects_bedrock_client(eval_cli: _EvalCli, monkeypatch) -> None: - monkeypatch.setattr(settings, "aws_access_key_id", "AKIATEST") - monkeypatch.setattr(settings, "aws_secret_access_key", "secret") - monkeypatch.setattr(settings, "aws_region", "us-east-1") - bedrock = MagicMock(return_value=object()) - monkeypatch.setattr("anthropic.AsyncAnthropicBedrock", bedrock) - eval_cli.invoke("tasks.py", "claude", "--config", f"checkpoint_name={_BEDROCK_ARN}", "--yes") - assert eval_cli.agent.config.model == _BEDROCK_ARN - assert eval_cli.agent.anthropic_client is bedrock.return_value - - -def test_interactive_picker_offers_current_catalog_models_newest_first( - eval_cli: _EvalCli, monkeypatch -) -> None: - catalog = _catalog( - ("openai_compatible", "MiniMax M3", "MiniMax-M3", "2026-06-20T00:00:00Z"), - ("openai_compatible", "Kimi K2.7", "moonshotai/kimi-k2.7", "2026-09-01T00:00:00Z"), - ("claude", "Claude Opus 5", "claude-opus-5", "2026-07-27T00:00:00Z"), - ) - catalog.append( - GatewayModelInfo( - id="old", - name="Old", - model_name="old", - sdk_agent_type="openai_compatible", - created_at="2026-09-10T00:00:00Z", - deprecated_at="2026-09-11T00:00:00Z", - ) - ) - monkeypatch.setattr("hud.cli.eval.list_gateway_models", lambda: catalog) - offered: list[str] = [] - - def pick(choices: list[Any]) -> Any: - offered.extend(choice["name"] for choice in choices) - return choices[0]["value"] - - monkeypatch.setattr(HUDConsole, "select", _picker("openai_compatible", pick)) - eval_cli.invoke("tasks.py", "--yes") - - assert offered == ["Kimi K2.7 (moonshotai/kimi-k2.7)", "MiniMax M3 (MiniMax-M3)"] - assert eval_cli.agent.config.model == "moonshotai/kimi-k2.7" - assert eval_cli.agent.config.model_name == "Kimi K2.7" - - -def test_interactive_picker_without_a_hud_key_uses_the_agent_default_model( - eval_cli: _EvalCli, monkeypatch -) -> None: - monkeypatch.setattr(settings, "api_key", None) - monkeypatch.setattr(settings, "anthropic_api_key", "provider-key") - monkeypatch.setattr( - "hud.cli.eval.list_gateway_models", lambda: pytest.fail("catalog needs a key") - ) - monkeypatch.setattr( - HUDConsole, "select", _picker("claude", lambda choices: pytest.fail("model prompt")) - ) - eval_cli.invoke("tasks.py", "--yes") - assert isinstance(eval_cli.agent.config, ClaudeConfig) - assert eval_cli.agent.config.model == ClaudeConfig().model - - -# ─── command: source discovery and results ───────────────────────────── - - -def test_missing_source_with_no_tasks_files_is_not_found(eval_cli: _EvalCli, monkeypatch) -> None: - monkeypatch.setattr(HUDConsole, "select", lambda *a, **k: pytest.fail("prompted")) - payload = eval_cli.invoke("--yes", exit_code=1) - assert payload["error"] == "not_found" - - -def test_single_tasks_file_is_picked_without_prompting( - eval_cli: _EvalCli, tmp_path: Path, monkeypatch -) -> None: - (tmp_path / "rows.json").write_text(f"[{_CONTAINER_ROW}]", encoding="utf-8") - (tmp_path / ".hidden.json").write_text("[]", encoding="utf-8") - monkeypatch.setattr( - "hud.cli.eval.list_gateway_models", - lambda: _catalog(("openai", "GPT 5.6", "gpt-5.6", "2026-06-12T00:00:00Z")), - ) - monkeypatch.setattr( - HUDConsole, "select", _picker("openai", lambda choices: choices[0]["value"]) - ) - payload = eval_cli.invoke("--yes") - assert payload["source"] == "rows.json" - assert eval_cli.agent.config.model == "gpt-5.6" - - -def test_several_tasks_files_prompt_for_one( - eval_cli: _EvalCli, tmp_path: Path, monkeypatch -) -> None: - (tmp_path / "rows.json").write_text(f"[{_CONTAINER_ROW}]", encoding="utf-8") - (tmp_path / "more.jsonl").write_text(_CONTAINER_ROW + "\n", encoding="utf-8") - seen: list[str] = [] - - def select(self: HUDConsole, message: str, choices: Any, **_: Any) -> str: - assert message == "Select a tasks file" - seen.extend(choices) - return "more.jsonl" - - monkeypatch.setattr(HUDConsole, "select", select) - # The agent has to come from config: without a source there is no second positional. - (tmp_path / ".hud_eval.toml").write_text('[eval]\nagent = "openai"\n', encoding="utf-8") - payload = eval_cli.invoke("--yes") - assert payload["source"] == "more.jsonl" - assert seen == ["more.jsonl", "rows.json"] - - -def test_result_payload_uses_job_metrics(eval_cli: _EvalCli) -> None: - graded = Run(None, "solve", {}) - graded.grade = Grade(reward=1.0) - graded.slug = "solve" - errored = Run(None, "solve", {}) - errored.grade = Grade(is_error=True) - eval_cli.job.runs.extend([graded, errored]) - - payload = eval_cli.invoke("tasks.py", "openai", "--yes") - - assert payload["job_id"] == "job-1" - assert payload["run_count"] == 2 - assert payload["mean_reward"] == 1.0 - assert payload["error_count"] == 1 - assert [run["slug"] for run in payload["runs"]] == ["solve", None] - assert [run["is_error"] for run in payload["runs"]] == [False, False] - - -@pytest.fixture -def local_eval( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> Iterator[Callable[..., dict[str, Any]]]: - async def answer(self: OpenAIAgent, run: Run) -> None: - run.trace.content = "ok" - - monkeypatch.setattr(OpenAIAgent, "__call__", answer) - monkeypatch.setattr(settings, "openai_api_key", "test-provider-key") - monkeypatch.setenv("HUD_API_KEY", "") - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "false") - monkeypatch.chdir(tmp_path) - original = dict(sys.modules) - - def invoke(source: Path | str, *flags: str) -> dict[str, Any]: - result = CliRunner().invoke( - app, ["eval", str(source), "openai", "--all", "--yes", "--json", *flags] - ) - assert result.exit_code == 0, result.output - payload = json.loads(result.stdout) - assert payload["error_count"] == 0, result.output - assert payload["mean_reward"] == 1.0, result.output - return payload - - yield invoke - for name, module in list(sys.modules.items()): - file = getattr(module, "__file__", None) - if file and Path(file).is_relative_to(tmp_path): - if name in original: - sys.modules[name] = original[name] - else: - sys.modules.pop(name, None) - - -@pytest.mark.parametrize( - "layout", - [ - "single", - "standalone", - "split", - "hooks", - "hooks_source", - "assembled", - "assembled_source", - "json", - "jsonl", - "data_python", - "directory", - "assembled_directory", - "package", - "lazy_package", - ], -) -def test_local_eval_project_layouts( - local_eval: Callable[..., dict[str, Any]], tmp_path: Path, layout: str -) -> None: - project = tmp_path / "project" - project.mkdir() - events = project / "events" - events.mkdir() - (project / "asset.txt").write_text("ok") - package = layout in {"package", "lazy_package"} - prefix = "." if package else "" - if package: - (project / "__init__.py").write_text("") - template = ( - "from pathlib import Path\n" - '@env.template(id="solve")\n' - "async def solve():\n" - ' answer = yield "answer ok"\n' - ' yield 1.0 if answer == Path("asset.txt").read_text() else 0.0\n' - ) - hooks = ( - "import os\nfrom pathlib import Path\n" - "@env.initialize\nasync def start():\n" - ' Path(f"events/{os.getpid()}").write_text("started")\n' - "@env.shutdown\nasync def stop():\n" - ' Path(f"events/{os.getpid()}").write_text("stopped")\n' - ) - if layout == "lazy_package": - (project / "expected.py").write_text( - 'from pathlib import Path\nexpected = Path("asset.txt").read_text()\n' - ) - template = template.replace( - ' answer = yield "answer ok"\n', - ' from .expected import expected\n answer = yield "answer ok"\n', - ).replace('Path("asset.txt").read_text()', "expected") - if layout in { - "hooks", - "hooks_source", - "assembled", - "assembled_source", - "assembled_directory", - "package", - }: - (project / "local_core.py").write_text( - 'from hud import Environment\nenv = Environment("local-test")\n' - ) - (project / "local_templates.py").write_text( - f"from {prefix}local_core import env\n" + template - ) - (project / "local_extra.py").write_text( - f"from {prefix}local_core import env\n" - '@env.template(id="extra")\nasync def extra():\n yield "extra"\n yield 1.0\n' - ) - env_source = ( - f"from {prefix}local_core import env\n" - f"from {prefix}local_templates import solve\n" - + (f"from {prefix}local_extra import extra\n" if not layout.startswith("hooks") else "") - + hooks - ) - else: - env_source = ( - 'from hud import Environment\nenv = Environment("local-test")\n' + template + hooks - ) - if layout in {"single", "standalone", "lazy_package", "hooks_source", "assembled_source"}: - env_source += "tasks = [solve()]\n" - source = project / "env.py" - else: - (project / "tasks.py").write_text( - f"from {prefix}env import env, solve\ntasks = [solve()]\n" - ) - source = project / "tasks.py" - (project / "env.py").write_text(env_source) - if layout == "standalone": - source = project / "standalone.py" - source.write_text(env_source) - (project / "env.py").write_text( - 'from hud import Environment\nenv = Environment("unrelated")\n' - ) - if layout in {"json", "jsonl"}: - source = project / f"tasks.{layout}" - row = {"env": "local-test", "id": "solve"} - source.write_text(json.dumps([row] if layout == "json" else row)) - if layout == "data_python": - source.write_text( - 'from hud.eval import Task\ntasks = [Task(env="local-test", id="solve")]\n' - ) - if layout in {"directory", "assembled_directory"}: - source = project - payload = local_eval(source, "--group", "2", "--max-concurrent", "2") - assert payload["run_count"] == 2 - assert len(list(events.iterdir())) == 2 - assert {event.read_text() for event in events.iterdir()} == {"stopped"} - - -@pytest.mark.parametrize( - "source,same_name", - [ - ("tasks.py", False), - ("tasks.py", True), - (".", False), - (".", True), - ("rows.json", False), - ("rows.json", True), - ("rows.jsonl", False), - ], -) -def test_local_eval_environment_selection( - local_eval: Callable[..., dict[str, Any]], tmp_path: Path, source: str, same_name: bool -) -> None: - project = tmp_path / "project" - project.mkdir() - (project / "bound_env.py").write_text( - 'from hud import Environment\nenv = Environment("bound")\n' - '@env.template(id="solve")\nasync def solve():\n' - ' answer = yield "answer ok"\n yield float(answer == "ok")\n' - ) - name = "bound" if same_name else "unrelated" - (project / "env.py").write_text( - f"from hud import Environment\nenv = Environment({name!r})\n" - '@env.template(id="solve")\nasync def solve():\n' - ' yield "unrelated"\n yield 0.0\n' - ) - (project / "tasks.py").write_text("from bound_env import solve\ntasks = [solve()]\n") - if source in {"rows.json", "rows.jsonl"}: - (project / source).write_text('{"env": "bound", "id": "solve"}\n') - if same_name and source != "tasks.py": - result = CliRunner().invoke( - app, ["eval", str(project / source), "openai", "--all", "--yes", "--json"] - ) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["error_count"] == 1 - assert "multiple Environments" in result.output - else: - assert local_eval(project / source)["run_count"] == 1 - - -@pytest.mark.parametrize("shared", [True, False]) -def test_local_eval_places_nested_verifier( - local_eval: Callable[..., dict[str, Any]], tmp_path: Path, shared: bool -) -> None: - project = tmp_path / "project" - project.mkdir() - (project / "local_core.py").write_text( - 'from hud import Environment\nactor = Environment("actor")\n' - + ("judge = actor\n" if shared else 'judge = Environment("judge")\n') - ) - (project / "local_actor.py").write_text( - "from local_core import actor\n" - '@actor.template(id="solve")\nasync def solve():\n' - ' answer = yield "answer ok"\n yield {"score": 0.0, "answer": answer}\n' - ) - (project / "local_judge.py").write_text( - "from local_core import judge\n" - '@judge.template(id="verify")\nasync def verify():\n' - ' result = yield ""\n yield 1.0 if result["answer"] == "ok" else 0.0\n' - ) - (project / "env.py").write_text( - "from local_core import actor, judge\n" - "from local_actor import solve\nfrom local_judge import verify\n" - ) - (project / "tasks.py").write_text( - "from env import solve, verify\ntasks = [solve()]\ntasks[0].verifier = verify()\n" - ) - assert local_eval(project / "tasks.py")["run_count"] == 1 - - -@pytest.mark.parametrize("container", ["image", "compose"]) -def test_platform_container_rows_can_use_local_placement( - eval_cli: _EvalCli, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, container: str -) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text("services:\n main:\n image: example:latest\n") - config = ( - RuntimeConfig(image="example:latest") - if container == "image" - else RuntimeConfig.model_validate( - {"compose": {"document": str(compose), "root": str(tmp_path)}} - ) - ) - task = Task(env="demo", id="solve", runtime_config=config) - monkeypatch.setattr(Taskset, "from_api", classmethod(lambda cls, name: Taskset(name, [task]))) - docker = MagicMock() - monkeypatch.setattr(eval_mod, "DockerRuntime", lambda: docker) - eval_cli.invoke("Platform Tasks", "openai", "--runtime", "local", "--yes") - assert eval_cli.kwargs["runtime"](task) is docker.return_value - docker.assert_called_once_with(task) - - -@pytest.mark.parametrize("shared,configured", [(True, False), (False, True), (False, False)]) -def test_platform_local_placement_checks_separate_verifier( - eval_cli: _EvalCli, monkeypatch: pytest.MonkeyPatch, shared: bool, configured: bool -) -> None: - config = RuntimeConfig(image="example:latest") - verifier = Task( - env="actor" if shared else "judge", - id="verify", - runtime_config=config if configured else None, - ) - task = Task(env="actor", id="solve", runtime_config=config, verifier=verifier) - monkeypatch.setattr(Taskset, "from_api", classmethod(lambda cls, name: Taskset(name, [task]))) - payload = eval_cli.invoke( - "Platform Tasks", - "openai", - "--runtime", - "local", - "--yes", - exit_code=0 if shared or configured else 2, - ) - if not shared and not configured: - assert "no env source to spawn locally" in payload["message"] - assert eval_cli.taskset is None diff --git a/hud/cli/tests/test_harbor.py b/hud/cli/tests/test_harbor.py deleted file mode 100644 index cf2d3d29b..000000000 --- a/hud/cli/tests/test_harbor.py +++ /dev/null @@ -1,584 +0,0 @@ -"""Tests for ``hud harbor``.""" - -from __future__ import annotations - -import io -import json -import zipfile -from typing import TYPE_CHECKING, Any -from urllib.parse import parse_qs, urlsplit -from uuid import UUID - -import httpx -import pytest -from typer.testing import CliRunner, Result - -from hud.cli import AuthScope, DirectoryState, ExitCode -from hud.cli.__main__ import app -from hud.utils.exceptions import HudRequestError -from hud.utils.platform import PlatformClient - -if TYPE_CHECKING: - from pathlib import Path - -_USER = "11111111-1111-4111-8111-111111111111" -_TEAM = "22222222-2222-4222-8222-222222222222" -_TASKSET = "44444444-4444-4444-8444-444444444444" -_FILE = "55555555-5555-4555-8555-555555555555" -_IMPORT = "66666666-6666-4666-8666-666666666666" -_UPLOAD_URL = "https://storage.test/upload?signature=secret" -_LIVE = {"queued", "starting", "running", "cancelling"} - - -def _report(**changes: Any) -> dict[str, Any]: - return { - "schema_version": "harbor_import_report.v5", - "taskset_id": _TASKSET, - "tasks": [ - { - "name": "hello", - "source": "bundle", - "path": "tasks/hello", - "build_version": 1, - "status": "imported", - }, - ], - "jobs": [ - { - "path": "run-1", - "name": "run-1", - "job_id": "job-hud-1", - "status": "imported", - "trials": [ - {"path": "run-1/hello__a", "status": "imported"}, - { - "path": "run-1/gone__b", - "status": "skipped", - "reason": "it ran a task that is neither in the bundle nor in HUD", - }, - ], - }, - ], - "warnings": [], - "error": None, - **changes, - } - - -class _Platform: - """The platform and storage endpoints a Harbor import touches.""" - - def __init__(self) -> None: - self.tasksets: dict[str, dict[str, Any]] = {} - self.has_imports_api = True - self.offers_harbor_import = True - self.max_input_bytes = 1024**3 - self.import_format = "harbor" - self.import_statuses = ["completed"] - self.import_error: str | None = None - # None: the import published no report. - self.report: Any = _report() - self.storage_status = 200 - self.calls: list[tuple[str, str, Any]] = [] - self.list_queries: list[dict[str, list[str]]] = [] - self.uploaded: bytes | None = None - self.upload_headers: httpx.Headers | None = None - - def body(self, method: str, path: str) -> Any: - [body] = [body for called, at, body in self.calls if (called, at) == (method, path)] - return body - - def paths(self, method: str) -> list[str]: - return [path for called, path, _ in self.calls if called == method] - - def imported(self, status: str) -> dict[str, Any]: - ended = status not in _LIVE - reported = status in {"completed", "stopped"} - return { - "id": _IMPORT, - "format": self.import_format, - "taskset_id": _TASKSET, - "file": {"id": _FILE, "filename": "tb2.zip", "size_bytes": 10}, - "status": status, - "summary": "Imported 1 task(s) and 1 run(s)" if reported else None, - "metrics": {"tasks_imported": 1.0, "trials_imported": 1.0} if reported else None, - "error": self.import_error, - "created_at": "2026-10-07T12:00:00Z", - "completed_at": "2026-10-07T12:01:00Z" if ended else None, - } - - def current_import(self) -> dict[str, Any]: - # Each read moves to the next status; the last one stays. - statuses = self.import_statuses - status = statuses.pop(0) if len(statuses) > 1 else statuses[0] - return self.imported(status) - - def request(self, method: str, url: str, **kwargs: Any) -> Any: - parts = urlsplit(url) - path = parts.path.removeprefix("/v2") - self.calls.append((method, path, kwargs.get("json"))) - if path == "/auth/me": - return {"user_id": _USER, "team_id": _TEAM} - if path.startswith("/imports") and not self.has_imports_api: - raise HudRequestError("Not Found", status_code=404) - if path == "/imports/formats": - harbor = { - "format": "harbor", - "title": "Harbor", - "max_input_bytes": self.max_input_bytes, - } - return {"formats": [harbor] if self.offers_harbor_import else []} - if path.startswith("/tasksets/by-name/"): - name = path.removeprefix("/tasksets/by-name/") - for taskset in self.tasksets.values(): - if taskset["name"] == name: - return {"taskset_id": taskset["id"], "name": name} - raise HudRequestError("Taskset not found", status_code=404) - if method == "GET" and path.startswith("/tasksets/"): - taskset_id = path.removeprefix("/tasksets/") - if taskset_id not in self.tasksets: - raise HudRequestError("Taskset not found", status_code=404) - return self.tasksets[taskset_id] - if method == "POST" and path == "/tasksets": - created = {"id": _TASKSET, "name": kwargs["json"]["name"], "can_edit": True} - self.tasksets[_TASKSET] = created - return created - if method == "POST" and path == "/imports/uploads": - filename = kwargs["json"]["filename"] - return {"file": {"id": _FILE, "filename": filename}, "upload_url": _UPLOAD_URL} - if method == "POST" and path == f"/data/{_FILE}/complete": - return {"id": _FILE, "filename": "tb2.zip", "size_bytes": 10} - if method == "DELETE" and path == f"/data/{_FILE}": - return None - if method == "POST" and path == "/imports": - return self.current_import() - if path == "/imports": - self.list_queries.append(parse_qs(parts.query)) - return {"imports": [self.current_import()], "total": 1} - if path == f"/imports/{_IMPORT}": - return self.current_import() - if path == f"/imports/{_IMPORT}/report": - if self.report is None: - raise HudRequestError("Import report not found", status_code=404) - return self.report - if path == f"/imports/{_IMPORT}/cancel": - return self.imported("cancelling") - if path.startswith("/imports/"): - raise HudRequestError("Import not found", status_code=404) - raise AssertionError((method, url)) - - def storage(self, request: httpx.Request) -> httpx.Response: - assert request.method == "PUT" - self.uploaded = request.read() - self.upload_headers = request.headers - return httpx.Response(self.storage_status) - - -@pytest.fixture -def platform(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> _Platform: - monkeypatch.chdir(tmp_path) - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - monkeypatch.setattr("hud.settings.settings.default_project", None) - monkeypatch.setattr("hud.settings.settings.hud_web_url", "https://hud.test") - monkeypatch.setattr("hud.imports.time.sleep", lambda _seconds: None) - fake = _Platform() - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake.request) - client = httpx.Client - monkeypatch.setattr( - httpx, - "Client", - lambda **kwargs: client(transport=httpx.MockTransport(fake.storage), **kwargs), - ) - return fake - - -@pytest.fixture -def folders(tmp_path: Path) -> list[str]: - task = tmp_path / "tasks" / "hello" - (task / "tests").mkdir(parents=True) - (task / "task.toml").write_text("version = '1.0'\n") - (task / "tests" / "test.sh").write_text("exit 0\n") - job = tmp_path / "run-1" - (job / "hello__a").mkdir(parents=True) - for folder in (job, job / "hello__a"): - (folder / "config.json").write_text("{}") - (folder / "result.json").write_text("{}") - return [str(tmp_path / "tasks"), str(job)] - - -def _invoke(*args: str) -> tuple[Result, dict[str, Any]]: - result = CliRunner().invoke(app, ["harbor", *args, "--json"]) - return result, json.loads(result.stdout) - - -def test_import_creates_a_new_taskset_and_imports_into_it( - platform: _Platform, folders: list[str] -) -> None: - result, payload = _invoke("import", *folders, "--taskset", "TB2 Sample", "--yes") - - assert result.exit_code == 0, result.output - assert platform.body("POST", "/tasksets") == {"name": "tb2-sample", "project_id": None} - assert platform.body("POST", "/imports") == { - "format": "harbor", - "file_id": _FILE, - "taskset_id": _TASKSET, - } - assert platform.paths("POST")[-2:] == [f"/data/{_FILE}/complete", "/imports"] - assert payload["taskset"] == {"id": _TASKSET, "name": "tb2-sample", "created": True} - assert payload["import"]["status"] == "completed" - assert payload["import"]["summary"] == "Imported 1 task(s) and 1 run(s)" - assert payload["report"]["taskset_id"] == _TASKSET - assert [trial["status"] for trial in payload["report"]["jobs"][0]["trials"]] == [ - "imported", - "skipped", - ] - - -def test_import_uploads_each_folder_under_its_name(platform: _Platform, folders: list[str]) -> None: - result, _ = _invoke("import", *folders, "--taskset", "tb2", "--yes") - - assert result.exit_code == 0, result.output - reservation = platform.body("POST", "/imports/uploads") - assert reservation["format"] == "harbor" - assert reservation["filename"] == "tb2.zip" - assert reservation["content_type"] == "application/zip" - assert platform.upload_headers is not None - assert platform.upload_headers["content-length"] == str(reservation["size_bytes"]) - assert platform.uploaded is not None - with zipfile.ZipFile(io.BytesIO(platform.uploaded)) as bundle: - assert sorted(bundle.namelist()) == [ - "run-1/config.json", - "run-1/hello__a/config.json", - "run-1/hello__a/result.json", - "run-1/result.json", - "tasks/hello/task.toml", - "tasks/hello/tests/test.sh", - ] - - -def test_import_defaults_to_the_taskset_named_after_the_first_folder( - platform: _Platform, folders: list[str] -) -> None: - platform.tasksets[_TASKSET] = {"id": _TASKSET, "name": "tasks", "can_edit": True} - - result, payload = _invoke("import", *folders, "--yes") - - assert result.exit_code == 0, result.output - assert "/tasksets" not in platform.paths("POST") - assert payload["taskset"] == {"id": _TASKSET, "name": "tasks", "created": False} - - -def test_rerunning_an_import_finds_the_taskset_its_name_created( - platform: _Platform, folders: list[str] -) -> None: - first, _ = _invoke("import", *folders, "--taskset", "TB2/Sample", "--yes") - second, payload = _invoke("import", *folders, "--taskset", "TB2/Sample", "--yes") - - assert first.exit_code == 0, first.output - assert second.exit_code == 0, second.output - assert platform.paths("POST").count("/tasksets") == 1 - assert platform.paths("GET").count("/tasksets/by-name/tb2sample") == 2 - assert payload["taskset"] == {"id": _TASKSET, "name": "tb2sample", "created": False} - - -def test_import_refuses_an_unknown_taskset_id_instead_of_creating_one( - platform: _Platform, folders: list[str] -) -> None: - result, payload = _invoke("import", *folders, "--taskset", _TASKSET, "--yes") - - assert result.exit_code == ExitCode.FAILURE - assert payload["error"] == "not_found" - assert payload["message"] == f"Taskset {_TASKSET} is inaccessible or deleted." - assert platform.paths("POST") == [] - - -def test_import_link_saves_the_taskset_as_the_directory_default( - platform: _Platform, folders: list[str], tmp_path: Path -) -> None: - result, _ = _invoke("import", *folders, "--taskset", "tb2", "--yes", "--link") - - assert result.exit_code == 0, result.output - state = DirectoryState(AuthScope.resolve(PlatformClient.from_settings()), tmp_path) - assert state.load().taskset_id == UUID(_TASKSET) - - -def test_import_refuses_a_taskset_the_caller_cannot_edit_before_uploading( - platform: _Platform, folders: list[str] -) -> None: - platform.tasksets[_TASKSET] = {"id": _TASKSET, "name": "theirs", "can_edit": False} - - result, payload = _invoke("import", *folders, "--taskset", _TASKSET, "--yes") - - assert result.exit_code == ExitCode.FAILURE - assert payload["error"] == "permission_denied" - assert "/imports/uploads" not in platform.paths("POST") - - -def test_import_refuses_a_bundle_over_the_format_limit( - platform: _Platform, folders: list[str] -) -> None: - platform.max_input_bytes = 10 - - result, payload = _invoke("import", *folders, "--taskset", "tb2", "--yes") - - assert result.exit_code == ExitCode.USAGE - assert payload["message"].endswith("a Harbor import takes at most 10 bytes.") - assert platform.paths("POST") == [] - - -def test_dry_run_plans_without_writing(platform: _Platform, folders: list[str]) -> None: - result, payload = _invoke("import", *folders, "--taskset", "tb2", "--dry-run") - - assert result.exit_code == 0, result.output - assert payload["dry_run"] is True - assert payload["taskset"] == {"id": None, "name": "tb2", "created": True} - assert payload["bundle"]["filename"] == "tb2.zip" - assert payload["bundle"]["files"] == 6 - assert platform.paths("POST") == [] - - -def test_a_failed_upload_deletes_its_reservation_without_leaking_the_signed_url( - platform: _Platform, folders: list[str] -) -> None: - platform.storage_status = 403 - - result, payload = _invoke("import", *folders, "--taskset", "tb2", "--yes") - - assert result.exit_code == ExitCode.FAILURE - assert payload["message"] == "Storage refused the upload: HTTP 403" - assert platform.paths("DELETE") == [f"/data/{_FILE}"] - assert "/imports" not in platform.paths("POST") - assert "signature" not in result.output - - -def test_a_stopped_import_exits_with_failure_and_its_report( - platform: _Platform, folders: list[str] -) -> None: - platform.import_statuses = ["stopped"] - platform.report = _report(error={"step": "add the tasks to the taskset", "message": "boom"}) - - result, payload = _invoke("import", *folders, "--taskset", "tb2", "--yes") - - assert result.exit_code == ExitCode.FAILURE - assert payload["import"]["status"] == "stopped" - assert payload["report"]["error"] == {"step": "add the tasks to the taskset", "message": "boom"} - - -def test_a_failed_import_without_a_report_exits_with_its_error( - platform: _Platform, folders: list[str] -) -> None: - platform.import_statuses = ["failed"] - platform.import_error = "the importer crashed" - - result = CliRunner().invoke(app, ["harbor", "import", *folders, "--taskset", "tb2", "--yes"]) - - assert result.exit_code == ExitCode.FAILURE - assert "The import failed: the importer crashed" in result.output - assert f"/imports/{_IMPORT}/report" not in platform.paths("GET") - - -def test_a_stopped_import_without_a_report_shows_its_summary( - platform: _Platform, folders: list[str] -) -> None: - platform.import_statuses = ["stopped"] - platform.report = None - - result, payload = _invoke("import", *folders, "--taskset", "tb2", "--yes") - - assert result.exit_code == ExitCode.FAILURE - assert payload["report"] is None - assert payload["import"]["summary"] == "Imported 1 task(s) and 1 run(s)" - - -def test_no_wait_returns_the_started_import(platform: _Platform, folders: list[str]) -> None: - platform.import_statuses = ["queued"] - - result, payload = _invoke("import", *folders, "--taskset", "tb2", "--yes", "--no-wait") - - assert result.exit_code == 0, result.output - assert payload["import"]["status"] == "queued" - assert payload["report"] is None - assert f"/imports/{_IMPORT}" not in platform.paths("GET") - - -def test_import_uploads_a_given_zip_as_it_is(platform: _Platform, tmp_path: Path) -> None: - bundle = tmp_path / "prepared.zip" - with zipfile.ZipFile(bundle, "w") as archive: - archive.writestr("hello/task.toml", "version = '1.0'\n") - - result, payload = _invoke("import", str(bundle), "--yes") - - assert result.exit_code == 0, result.output - assert platform.uploaded == bundle.read_bytes() - assert payload["taskset"]["name"] == "prepared" - - -def test_import_packs_a_folder_named_like_a_zip(platform: _Platform, tmp_path: Path) -> None: - folder = tmp_path / "release.zip" - folder.mkdir() - (folder / "task.toml").write_text("version = '1.0'\n") - - result, payload = _invoke("import", str(folder), "--taskset", "tb2", "--yes") - - assert result.exit_code == 0, result.output - assert payload["bundle"]["files"] == 1 - assert platform.uploaded is not None - with zipfile.ZipFile(io.BytesIO(platform.uploaded)) as bundle: - assert bundle.namelist() == ["release.zip/task.toml"] - - -def test_import_refuses_a_zip_mixed_with_folders_before_any_request( - platform: _Platform, folders: list[str], tmp_path: Path -) -> None: - bundle = tmp_path / "prepared.zip" - bundle.write_bytes(b"zip") - - result, payload = _invoke("import", str(bundle), folders[0], "--yes") - - assert result.exit_code == ExitCode.USAGE - assert payload["message"] == "Pass either one .zip bundle or folders, not both or several zips." - assert platform.calls == [] - - -def test_import_fails_where_the_platform_offers_no_harbor_import( - platform: _Platform, folders: list[str] -) -> None: - platform.offers_harbor_import = False - - result, payload = _invoke("import", *folders, "--yes") - - assert result.exit_code == ExitCode.FAILURE - assert payload["error"] == "not_found" - assert payload["message"].startswith("Harbor import is not available on ") - - -def test_import_fails_where_the_platform_has_no_imports( - platform: _Platform, folders: list[str] -) -> None: - platform.has_imports_api = False - - result, payload = _invoke("import", *folders, "--yes") - - assert result.exit_code == ExitCode.FAILURE - assert payload["error"] == "not_found" - assert payload["message"].startswith("Harbor import is not available on ") - - -def test_import_prints_the_report_tables(platform: _Platform, folders: list[str]) -> None: - result = CliRunner().invoke(app, ["harbor", "import", *folders, "--taskset", "tb2", "--yes"]) - - assert result.exit_code == 0, result.output - assert "1 imported, 1 skipped" in result.output - assert "it ran a task that is neither in the bundle nor in HUD" in result.output - assert f"https://hud.test/tasksets/{_TASKSET}/imports" in result.output - - -def test_report_wait_follows_the_import_until_it_ends(platform: _Platform) -> None: - platform.import_statuses = ["queued", "running", "completed"] - - result, payload = _invoke("report", _IMPORT, "--wait") - - assert result.exit_code == 0, result.output - assert payload["import"]["status"] == "completed" - assert payload["report"]["taskset_id"] == _TASKSET - - -def test_report_of_a_running_import_without_wait_has_no_report(platform: _Platform) -> None: - platform.import_statuses = ["running"] - - result, payload = _invoke("report", _IMPORT) - - assert result.exit_code == 0, result.output - assert payload["report"] is None - assert f"/imports/{_IMPORT}/report" not in platform.paths("GET") - - -def test_report_shows_tasks_an_import_updated(platform: _Platform) -> None: - [task] = platform.report["tasks"] - platform.report["tasks"] = [{**task, "build_version": 2, "status": "updated"}] - - result = CliRunner().invoke(app, ["harbor", "report", _IMPORT]) - - assert result.exit_code == 0, result.output - assert "updated" in result.output - - -def test_report_reads_a_v4_report(platform: _Platform) -> None: - platform.report = _report(schema_version="harbor_import_report.v4") - - result, payload = _invoke("report", _IMPORT) - - assert result.exit_code == 0, result.output - assert payload["report"]["schema_version"] == "harbor_import_report.v4" - assert payload["report"]["tasks"][0]["status"] == "imported" - - -def test_report_of_an_unknown_schema_falls_back_to_the_summary(platform: _Platform) -> None: - platform.report = {"schema_version": "harbor_import_report.v9"} - - result, payload = _invoke("report", _IMPORT) - - assert result.exit_code == 0, result.output - assert payload["report"] is None - assert payload["import"]["summary"] == "Imported 1 task(s) and 1 run(s)" - - -def test_report_refuses_an_import_of_another_format(platform: _Platform) -> None: - platform.import_format = "parquet" - - result, payload = _invoke("report", _IMPORT) - - assert result.exit_code == ExitCode.USAGE - assert payload["message"] == f"Import {_IMPORT} is a parquet import, not a Harbor import." - - -def test_report_of_an_unknown_import_is_not_found(platform: _Platform) -> None: - result, payload = _invoke("report", "88888888-8888-4888-8888-888888888888") - - assert result.exit_code == ExitCode.FAILURE - assert payload["error"] == "not_found" - - -def test_cancel_requests_cancellation_of_a_running_import(platform: _Platform) -> None: - platform.import_statuses = ["running"] - - result, payload = _invoke("cancel", _IMPORT, "--yes") - - assert result.exit_code == 0, result.output - assert f"/imports/{_IMPORT}/cancel" in platform.paths("POST") - assert payload["status"] == "cancelling" - - -def test_cancel_leaves_an_ended_import_alone(platform: _Platform) -> None: - result, payload = _invoke("cancel", _IMPORT, "--yes") - - assert result.exit_code == 0, result.output - assert platform.paths("POST") == [] - assert payload["status"] == "completed" - - -def test_imports_filters_by_taskset(platform: _Platform) -> None: - platform.tasksets[_TASKSET] = {"id": _TASKSET, "name": "tb2", "can_edit": True} - - result, payload = _invoke("imports", "--taskset", "tb2") - - assert result.exit_code == 0, result.output - assert [item["id"] for item in payload["imports"]] == [_IMPORT] - assert platform.list_queries == [ - {"limit": ["20"], "format": ["harbor"], "taskset_id": [_TASKSET]} - ] - - -def test_imports_refuses_an_unknown_taskset_id(platform: _Platform) -> None: - result, payload = _invoke("imports", "--taskset", _TASKSET) - - assert result.exit_code == ExitCode.FAILURE - assert payload["error"] == "not_found" - assert platform.list_queries == [] - - -def test_imports_refuses_a_limit_the_platform_does_not_page(platform: _Platform) -> None: - result = CliRunner().invoke(app, ["harbor", "imports", "--limit", "101"]) - - assert result.exit_code == ExitCode.USAGE - assert platform.calls == [] diff --git a/hud/cli/tests/test_init.py b/hud/cli/tests/test_init.py deleted file mode 100644 index 60fa849d4..000000000 --- a/hud/cli/tests/test_init.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Tests for ``hud init``.""" - -from __future__ import annotations - -import io -import json -import tarfile -from typing import TYPE_CHECKING -from unittest.mock import MagicMock - -import httpx -import pytest -from typer.testing import CliRunner - -from hud.cli import CliError -from hud.cli import init as init_module -from hud.cli.__main__ import app -from hud.cli.eval import EvalConfig -from hud.cli.init import init_command -from hud.types import AgentType - -if TYPE_CHECKING: - from pathlib import Path - - -def _sdk_archive(source: str, files: dict[str, bytes]) -> bytes: - payload = io.BytesIO() - with tarfile.open(fileobj=payload, mode="w:gz") as archive: - for name, content in files.items(): - info = tarfile.TarInfo(f"hud-python-release/environments/{source}/{name}") - info.size = len(content) - archive.addfile(info, io.BytesIO(content)) - return payload.getvalue() - - -@pytest.fixture -def installed_sdk(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Make ``hud init`` behave as an installed release: no ``environments/`` beside the package.""" - monkeypatch.setattr(init_module, "__file__", str(tmp_path / "site" / "hud" / "cli" / "init.py")) - monkeypatch.setattr(init_module, "__version__", "1.2.3") - - -def _init( - tmp_path: Path, - name: str | None, - preset: str | None, - *, - force: bool = False, - dry_run: bool = False, -) -> dict[str, object]: - """Call the command as the CLI would, with every option bound.""" - return init_command( - name=name, directory=str(tmp_path), force=force, dry_run=dry_run, preset=preset - ) - - -def _release(payload: bytes) -> MagicMock: - return MagicMock( - return_value=httpx.Response( - 200, content=payload, request=httpx.Request("GET", "https://example.test") - ) - ) - - -# ─── choosing the example ─────────────────────────────────────────────── - - -def test_name_alone_uses_the_coding_example(tmp_path: Path) -> None: - _init(tmp_path, "my-cool-env", None) - - target = tmp_path / "my-cool-env" - assert (target / "README.md").exists() - assert 'Environment(name="my-cool-env")' in (target / "env.py").read_text() - - -def test_example_without_name_uses_its_id_as_directory(tmp_path: Path) -> None: - _init(tmp_path, None, "coding") - assert (tmp_path / "coding" / "env.py").exists() - - -def test_name_overrides_the_example_directory_and_env_name(tmp_path: Path) -> None: - _init(tmp_path, "custom", "cua") - - assert 'Environment(name="custom")' in (tmp_path / "custom" / "env.py").read_text() - assert not (tmp_path / "cua").exists() - - -def test_blank_materializes_a_runnable_example(tmp_path: Path) -> None: - _init(tmp_path, "berry", "blank") - - target = tmp_path / "berry" - assert {path.name for path in target.iterdir()} == { - "README.md", - "pyproject.toml", - "env.py", - "tasks.py", - "Dockerfile.hud", - ".dockerignore", - ".hud_eval.toml", - } - assert 'Environment(name="berry")' in (target / "env.py").read_text() - assert "package = false" in (target / "pyproject.toml").read_text() - assert 'CMD ["uv", "run", "hud", "serve"' in (target / "Dockerfile.hud").read_text() - assert ".venv" in (target / ".dockerignore").read_text() - # The template is all comments: a fresh project evaluates with built-in defaults. - assert EvalConfig.load(target / ".hud_eval.toml") == EvalConfig( - agent_config={agent.value: {} for agent in AgentType} - ) - - -def test_env_name_is_normalized(tmp_path: Path) -> None: - _init(tmp_path, "My Cool_Env", "blank") - assert 'Environment(name="my-cool-env")' in (tmp_path / "My Cool_Env" / "env.py").read_text() - - -def test_without_name_or_example_errors_when_noninteractive(tmp_path: Path) -> None: - with pytest.raises(CliError, match="Nothing to create"): - _init(tmp_path, None, None) - - -def test_unknown_example_is_a_usage_error(tmp_path: Path) -> None: - with pytest.raises(CliError, match="Unknown example environment 'does-not-exist'"): - _init(tmp_path, None, "does-not-exist") - - -def test_dry_run_never_prompts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(init_module.sys.stdin, "isatty", lambda: True) - monkeypatch.setattr(init_module.sys.stdout, "isatty", lambda: True) - plan = _init(tmp_path, "thing", None, dry_run=True) - assert plan == { - "dry_run": True, - "action": "init", - "path": str(tmp_path / "thing"), - "preset": "coding", - } - assert not (tmp_path / "thing").exists() - - -# ─── destination safety ───────────────────────────────────────────────── - - -def test_refuses_to_clobber_nonempty_directory(tmp_path: Path) -> None: - target = tmp_path / "taken" - target.mkdir() - (target / "precious.txt").write_text("data") - - with pytest.raises(CliError, match="not empty"): - _init(tmp_path, "taken", "blank") - assert (target / "precious.txt").read_text() == "data" - - -def test_force_overwrites_existing_files(tmp_path: Path) -> None: - target = tmp_path / "env" - target.mkdir() - (target / "env.py").write_text("old") - - _init(tmp_path, "env", "blank", force=True) - assert "Environment" in (target / "env.py").read_text() - - -def test_refuses_to_copy_over_a_symlinked_file(tmp_path: Path) -> None: - target = tmp_path / "project" - target.mkdir() - outside = tmp_path / "outside.py" - outside.write_text("original") - (target / "env.py").symlink_to(outside) - - with pytest.raises(CliError, match="symlinks"): - _init(tmp_path, "project", "blank", force=True) - assert outside.read_text() == "original" - - -def test_refuses_a_symlinked_destination(tmp_path: Path) -> None: - target = tmp_path / "project" - target.symlink_to(tmp_path / "outside", target_is_directory=True) - - with pytest.raises(CliError, match="symlink"): - _init(tmp_path, "project", "blank") - - -def test_failure_is_a_json_error_and_removes_the_partial_directory( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - def fail(source: Path, target: Path, **_: object) -> None: - target.mkdir() - (target / "partial").touch() - raise OSError("copy failed") - - monkeypatch.setattr(init_module.shutil, "copytree", fail) - result = CliRunner().invoke(app, ["init", "example", "--dir", str(tmp_path), "--json"]) - assert result.exit_code != 0 - assert "copy failed" in json.loads(result.stdout)["message"] - assert not (tmp_path / "example").exists() - - -# ─── installed SDK: the example comes from the release archive ───────── - - -def test_local_copy_skips_caches(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - repository = tmp_path / "repository" - source = repository / "environments" / "coding" - source.mkdir(parents=True) - (source / "env.py").write_text('env = Environment(name="coding")') - (source / ".venv").mkdir() - (source / ".venv" / "ignored").write_text("ignored") - (source / "__pycache__").mkdir() - (source / "__pycache__" / "ignored.pyc").write_bytes(b"ignored") - monkeypatch.setattr(init_module, "__file__", str(repository / "hud" / "cli" / "init.py")) - - _init(tmp_path, "coding", "coding") - - target = tmp_path / "coding" - assert (target / "env.py").read_text() == 'env = Environment(name="coding")' - assert not (target / ".venv").exists() - assert not (target / "__pycache__").exists() - - -def test_release_archive_is_extracted_for_the_installed_version( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, installed_sdk: None -) -> None: - get = _release( - _sdk_archive( - "coding", - {"README.md": b"# Coding", "scripts/run.sh": b"#!/bin/sh\n", "env.py": b"x"}, - ) - ) - monkeypatch.setattr(init_module.httpx, "get", get) - - _init(tmp_path, "coding", "coding") - - assert get.call_args.args[0].endswith("/tar.gz/refs/tags/v1.2.3") - target = tmp_path / "coding" - assert (target / "README.md").read_text() == "# Coding" - assert (target / "scripts" / "run.sh").read_text() == "#!/bin/sh\n" - - -def test_release_archive_paths_cannot_escape_the_target( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, installed_sdk: None -) -> None: - monkeypatch.setattr( - init_module.httpx, "get", _release(_sdk_archive("coding", {"../../escape": b"unsafe"})) - ) - - with pytest.raises(CliError, match="unsafe path"): - _init(tmp_path, "project", "coding") - assert not (tmp_path / "escape").exists() - assert not (tmp_path / "project").exists() - - -def test_development_version_needs_a_checkout( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, installed_sdk: None -) -> None: - monkeypatch.setattr(init_module, "__version__", "1.2.3.dev0") - with pytest.raises(CliError, match="development version"): - _init(tmp_path, "project", "coding") diff --git a/hud/cli/tests/test_jobs.py b/hud/cli/tests/test_jobs.py deleted file mode 100644 index 03b0cd7ce..000000000 --- a/hud/cli/tests/test_jobs.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any - -import pytest -from typer.testing import CliRunner - -from hud.cli import jobs -from hud.cli.__main__ import app -from hud.settings import settings -from hud.utils.platform import PlatformClient - - -class _Client: - def __init__(self) -> None: - self.calls: list[tuple[str, dict[str, Any] | None]] = [] - - def get(self, path: str, *, params: dict[str, Any] | None = None) -> dict[str, Any]: - self.calls.append((path, params)) - return {"items": []} - - -def test_job_detail_accepts_compact_id_and_prints_canonical_link( - monkeypatch: pytest.MonkeyPatch, -) -> None: - compact_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - canonical_id = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" - client = _Client() - monkeypatch.setattr(settings, "api_key", "test-key") - monkeypatch.setattr(settings, "hud_web_url", "https://hud.test") - monkeypatch.setattr(PlatformClient, "from_settings", classmethod(lambda cls: client)) - - result = CliRunner().invoke(jobs.jobs_app, ["get", compact_id]) - - assert result.exit_code == 0 - assert client.calls == [(f"/jobs/{canonical_id}/traces", {"limit": 20})] - assert f"https://hud.test/jobs/{canonical_id}" in result.stdout - - -@pytest.mark.parametrize("verb", [[], ["get"]]) -@pytest.mark.parametrize( - ("before", "after"), - [ - ([], ["--json", "--limit", "7", "--quiet"]), - (["--json", "--limit", "7", "--quiet"], []), - (["--json", "--limit", "5", "--quiet"], ["--limit", "7"]), - ], -) -def test_job_detail_shorthand_preserves_options(monkeypatch, verb, before, after): - job_id = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" - client = _Client() - monkeypatch.setattr(PlatformClient, "from_settings", classmethod(lambda cls: client)) - result = CliRunner().invoke(app, ["jobs", *verb, *before, job_id, *after]) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout) == [] - assert client.calls == [(f"/jobs/{job_id}/traces", {"limit": 7})] - - -def test_bare_jobs_still_lists(monkeypatch): - client = _Client() - monkeypatch.setattr(PlatformClient, "from_settings", classmethod(lambda cls: client)) - result = CliRunner().invoke(app, ["jobs", "--json", "-n", "7"]) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout) == [] - assert client.calls == [("/jobs", {"limit": 7})] - - -@pytest.mark.parametrize("command", ["jobs", "trace"]) -def test_unknown_resource_verb_is_a_usage_error(command): - result = CliRunner().invoke(app, [command, "unknown", "--json"]) - assert result.exit_code == 2 - assert json.loads(result.stdout)["error"] == "usage" diff --git a/hud/cli/tests/test_main.py b/hud/cli/tests/test_main.py deleted file mode 100644 index d2ae70cb8..000000000 --- a/hud/cli/tests/test_main.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Tests for hud.cli.__main__.""" - -from __future__ import annotations - -import os -import subprocess -import sys - - -class TestMainModule: - def test_main_module_imports_correctly(self) -> None: - import hud.cli.__main__ - - assert hasattr(hud.cli.__main__, "main") - - def test_main_module_executes(self) -> None: - env = {**os.environ, "HUD_SKIP_VERSION_CHECK": "1"} - result = subprocess.run( - [sys.executable, "-m", "hud.cli", "--version"], - capture_output=True, - text=True, - timeout=30, - env=env, - ) - assert result.returncode == 0 - assert "version" in result.stdout.lower() or "hud" in result.stdout.lower() diff --git a/hud/cli/tests/test_project.py b/hud/cli/tests/test_project.py deleted file mode 100644 index 3abe3d3c9..000000000 --- a/hud/cli/tests/test_project.py +++ /dev/null @@ -1,284 +0,0 @@ -"""Project lookup and placement precedence for CLI create-and-link flows.""" - -from __future__ import annotations - -import json -from typing import TYPE_CHECKING, Any -from urllib.parse import parse_qs, urlsplit -from uuid import UUID - -import pytest -from typer.testing import CliRunner - -from hud.cli import AuthScope, CliError, DirectoryLink, DirectoryState -from hud.cli.__main__ import app -from hud.cli.project import Placement, Project -from hud.utils.exceptions import HudRequestError -from hud.utils.platform import PlatformClient - -if TYPE_CHECKING: - from pathlib import Path - -_DEFAULT_ID = "11111111-1111-4111-8111-111111111111" -_BROWSER_ID = "22222222-2222-4222-8222-222222222222" -_READONLY_ID = "33333333-3333-4333-8333-333333333333" - - -def _record( - project_id: str, name: str, *, is_default: bool = False, create: bool = True -) -> dict[str, Any]: - return { - "id": project_id, - "name": name, - "is_default": is_default, - "capabilities": {"view": True, "create": create, "manage": False}, - } - - -@pytest.fixture -def calls() -> list[str]: - """URLs the fake platform transport was asked for.""" - return [] - - -@pytest.fixture -def platform(monkeypatch: pytest.MonkeyPatch, calls: list[str]) -> PlatformClient: - """A client whose ``GET /projects`` returns a fixed three-project team.""" - - def fake_request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - calls.append(url) - records = [ - _record(_DEFAULT_ID, "default", is_default=True), - _record(_BROWSER_ID, "browser-evals"), - _record(_READONLY_ID, "locked-down", create=False), - ] - project_id = url.rsplit("/", 1)[-1] - if project_id in {_DEFAULT_ID, _BROWSER_ID, _READONLY_ID}: - return next(record for record in records if record["id"] == project_id) - search = (kwargs.get("params") or {}).get("search") - if search: - records = [record for record in records if search in record["name"]] - return {"items": records, "total": len(records), "limit": 50, "offset": 0} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - return PlatformClient("https://api.example", "key") - - -def _no_global_default(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("hud.settings.settings.default_project", None) - - -def test_list_reads_paginated_items(platform: PlatformClient, monkeypatch) -> None: - monkeypatch.setattr(PlatformClient, "from_settings", classmethod(lambda cls: platform)) - result = CliRunner().invoke(app, ["project", "list", "--json"]) - assert result.exit_code == 0, result.output - assert [project["name"] for project in json.loads(result.stdout)] == [ - "default", - "browser-evals", - "locked-down", - ] - - -def test_resolve_matches_a_normalized_name(platform: PlatformClient) -> None: - assert Project.resolve(platform, "browser-evals").id == _BROWSER_ID - assert Project.resolve(platform, "Browser Evals").id == _BROWSER_ID - - -def test_resolve_rejects_partial_and_unknown_names(platform: PlatformClient) -> None: - with pytest.raises(CliError, match="No Project named 'browser'") as error: - Project.resolve(platform, "browser") - assert error.value.error == "not_found" - with pytest.raises(CliError, match="No Project named 'nope'"): - Project.resolve(platform, "nope") - - -def test_resolve_matches_an_id(platform: PlatformClient) -> None: - assert Project.resolve(platform, _BROWSER_ID).name == "browser-evals" - assert Project.resolve(platform, _BROWSER_ID.upper()).name == "browser-evals" - - -def test_flag_outranks_directory_config( - platform: PlatformClient, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr("hud.settings.settings.default_project", "locked-down") - source = DirectoryLink(project_id=UUID(_DEFAULT_ID)) - - placement = Placement.resolve(platform, source, flag=_BROWSER_ID) - - assert placement.project is not None - assert placement.project.id == _BROWSER_ID - assert placement.source == "--project" - - -def test_directory_config_applies_without_a_flag( - platform: PlatformClient, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Placement is a property of the environment, not of who deploys it.""" - monkeypatch.setattr("hud.settings.settings.default_project", "default") - source = DirectoryLink(project_id=UUID(_BROWSER_ID)) - - placement = Placement.resolve(platform, source, flag=None) - - assert placement.project is not None - assert placement.project.id == _BROWSER_ID - assert placement.source == ".hud/config.json" - - -def test_global_default_applies_to_an_unpinned_directory( - platform: PlatformClient, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr("hud.settings.settings.default_project", _BROWSER_ID) - - placement = Placement.resolve(platform, DirectoryLink(), flag=None) - - assert placement.project is not None - assert placement.project.id == _BROWSER_ID - assert placement.source == "HUD_DEFAULT_PROJECT" - - -def test_unconfigured_placement_sends_no_project_and_makes_no_call( - platform: PlatformClient, - calls: list[str], - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The zero-config path stays free: no project on the wire, no lookup.""" - _no_global_default(monkeypatch) - placement = Placement.resolve(platform, DirectoryLink(), flag=None) - - assert placement.project_id is None - assert placement.source == "team default" - assert placement.label == "team default Project" - assert calls == [] - - -def test_placement_resolves_a_project_the_caller_cannot_create_in( - platform: PlatformClient, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - _no_global_default(monkeypatch) - source = DirectoryLink() - placement = Placement.resolve(platform, source, flag=_READONLY_ID) - assert placement.project is not None - assert placement.project.id == _READONLY_ID - - with pytest.raises(CliError, match="permission") as info: - placement.require_writable() - assert info.value.error == "permission_denied" - - -def test_from_record_defaults_capabilities_to_read_only() -> None: - """A response without capabilities is not assumed writable.""" - assert Project.from_record({"id": "x", "name": "y"}).can_create is False - - -_COMMAND_PROJECT_ID = "22222222-2222-4222-8222-222222222222" -_COMMAND_SCOPE = AuthScope( - origin="https://api.example", - user_id="11111111-1111-4111-8111-111111111111", - team_id=_COMMAND_PROJECT_ID, -) -_COMMAND_RECORD = { - "id": _COMMAND_PROJECT_ID, - "name": "browser-evals", - "capabilities": {"create": True}, -} - - -class TestProjectCommand: - @pytest.fixture(autouse=True) - def platform(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - monkeypatch.setattr("hud.settings.settings.hud_api_url", _COMMAND_SCOPE.origin) - monkeypatch.setattr("hud.settings.settings.default_project", None) - - def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - if url.endswith("/auth/me"): - return _COMMAND_SCOPE.model_dump(mode="json") - if url.endswith(f"/projects/{_COMMAND_PROJECT_ID}") or method == "POST": - return _COMMAND_RECORD - raise AssertionError((method, url)) - - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) - - @pytest.mark.parametrize("override", [False, True]) - def test_use_honors_directory_options(self, tmp_path: Path, override: bool) -> None: - group = tmp_path / "group" - target = tmp_path / "override" if override else group - args = ["project", "-C", str(group), "use", _COMMAND_PROJECT_ID, "--json"] - if override: - args += ["-C", str(target)] - result = CliRunner().invoke(app, args) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["id"] == _COMMAND_PROJECT_ID - assert DirectoryState(_COMMAND_SCOPE, target).load().project_id == UUID(_COMMAND_PROJECT_ID) - assert (target / ".hud" / "config.json").exists() - if override: - assert DirectoryState(_COMMAND_SCOPE, group).load().project_id is None - - def test_create_links_group_directory(self, tmp_path: Path) -> None: - result = CliRunner().invoke( - app, ["project", "-C", str(tmp_path), "create", "browser-evals", "--json"] - ) - assert result.exit_code == 0, result.output - assert DirectoryState(_COMMAND_SCOPE, tmp_path).load().project_id == UUID( - _COMMAND_PROJECT_ID - ) - - def test_use_dry_run_does_not_persist(self, tmp_path: Path) -> None: - legacy = tmp_path / "env" / ".hud" / "deploy.json" - legacy.parent.mkdir(parents=True) - legacy.write_text('{"projectId":"old"}') - result = CliRunner().invoke( - app, - [ - "project", - "use", - _COMMAND_PROJECT_ID, - "-C", - str(legacy.parent.parent), - "--dry-run", - "--json", - ], - ) - assert result.exit_code == 0, result.output - assert not (legacy.parent / "config.json").exists() - assert legacy.read_text() == '{"projectId":"old"}' - - def test_create_permission_error_is_one_json_document( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - def denied(*args: Any, **kwargs: Any) -> None: - raise HudRequestError("Projects are not enabled", status_code=403) - - monkeypatch.setattr("hud.utils.platform.make_request_sync", denied) - result = CliRunner().invoke( - app, ["project", "create", "browser-evals", "--no-use", "--json"] - ) - assert result.exit_code == 1 - assert json.loads(result.stdout)["error"] == "permission_denied" - - def test_list_reads_every_page(self, monkeypatch: pytest.MonkeyPatch) -> None: - offsets: list[int] = [] - - def page(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - offset = int(parse_qs(urlsplit(url).query)["offset"][0]) - offsets.append(offset) - records = [ - {**_COMMAND_RECORD, "id": str(UUID(int=i + 1))} - for i in range(offset, min(offset + 50, 51)) - ] - return {"items": records, "total": 51} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", page) - result = CliRunner().invoke(app, ["project", "list", "--json"]) - assert result.exit_code == 0, result.output - assert len(json.loads(result.stdout)) == 51 - assert offsets == [0, 50] diff --git a/hud/cli/tests/test_qa.py b/hud/cli/tests/test_qa.py deleted file mode 100644 index 456b248c9..000000000 --- a/hud/cli/tests/test_qa.py +++ /dev/null @@ -1,586 +0,0 @@ -"""CLI behavior for HUD's QA checks on evaluation traces and tasks.""" - -from __future__ import annotations - -import json -from typing import Any -from unittest.mock import MagicMock, call, patch - -import pytest -from typer.testing import CliRunner - -from hud.cli.__main__ import app -from hud.cli.qa import presentation_for_result - -runner = CliRunner() - -_TRACE_ID = "00000000-0000-4000-a000-000000000002" -_RESULT_ID = "00000000-0000-4000-a000-000000000003" -_ANALYSIS_TRACE_ID = "00000000-0000-4000-a000-000000000004" -_TASK_ID = "00000000-0000-4000-a000-000000000005" -_TASK_VERSION_ID = "00000000-0000-4000-a000-000000000006" -_CHECK = "failure_analysis" -_OTHER_CHECK = "reward_hacking" -_TASK_CHECK = "task_reward_exploit" - - -def _check( - key: str = _CHECK, title: str = "Failure Analysis", level: str = "trace" -) -> dict[str, object]: - return { - "key": key, - "subject_type": level, - "title": title, - "question": "Why did the agent fail?", - "description": "Attributes each failure to the agent, the evaluation, or the platform.", - } - - -_CATALOG = [ - _check(), - _check(_OTHER_CHECK, "Reward Hacking"), - _check(_TASK_CHECK, "Task Reward Exploit", "task"), -] - - -def _platform(*polls: list[dict[str, Any]]) -> MagicMock: - """A platform serving the check catalog and one results reply per poll (the last repeats).""" - platform = MagicMock() - replies = list(polls) - - def get(path: str, params: dict[str, Any] | None = None) -> dict[str, Any]: - if path == "/qa/checks": - return {"checks": _CATALOG} - return {"results": replies.pop(0) if len(replies) > 1 else replies[0]} - - platform.get.side_effect = get - return platform - - -def _row( - status: str = "completed", - *, - check_key: str = _CHECK, - verdict: str | None = None, - result: dict[str, object] | None = None, - legacy_result: dict[str, object] | None = None, - **fields: object, -) -> dict[str, Any]: - return { - "id": _RESULT_ID, - "check_key": check_key, - "subject_type": "trace", - "subject_trace_id": _TRACE_ID, - "source": "analysis", - "status": status, - "verdict": verdict, - "result": result, - "legacy_result": legacy_result, - "note": None, - "error": None, - "created_at": "2026-10-02T00:00:00Z", - "completed_at": None, - "run_id": None, - **fields, - } - - -def _result(verdict: str = "passed", *, check_key: str = _CHECK) -> dict[str, Any]: - return _row( - check_key=check_key, - verdict=verdict, - result={ - "schema_version": "qa_agent_result.v1", - "verdict": verdict, - "summary": "Looks good." if verdict == "passed" else "A gap was found.", - "findings": [], - "metadata": {}, - }, - ) - - -def _task_result(verdict: str = "passed") -> dict[str, Any]: - return { - **_result(verdict, check_key=_TASK_CHECK), - "subject_type": "task", - "subject_trace_id": None, - "subject_task_id": _TASK_ID, - "subject_task_version_id": _TASK_VERSION_ID, - } - - -def _invoke(platform: MagicMock, args: list[str]): - with patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform): - return runner.invoke(app, args, env={"TERM": "xterm", "COLUMNS": "200"}) - - -def _table_row(output: str, *cells: str) -> str: - """The table line holding every given cell.""" - return next(line for line in output.splitlines() if all(cell in line for cell in cells)) - - -@pytest.mark.parametrize("args", [["qa"], ["qa", "list"]]) -def test_qa_lists_checks_as_a_table(args: list[str]) -> None: - platform = MagicMock() - platform.get.return_value = {"checks": [_check()]} - - result = _invoke(platform, args) - - assert result.exit_code == 0 - assert "QA Checks" in result.output - assert _table_row(result.output, _CHECK, "trace", "Failure Analysis", "Why did the agent fail?") - platform.get.assert_called_once_with("/qa/checks") - - -def test_qa_list_quiet_prints_keys() -> None: - platform = MagicMock() - platform.get.return_value = {"checks": [_check(), _check(_OTHER_CHECK, "Reward Hacking")]} - - result = _invoke(platform, ["qa", "list", "--quiet"]) - - assert result.exit_code == 0 - assert result.output.split() == [_CHECK, _OTHER_CHECK] - - -@pytest.mark.parametrize( - ("level", "keys"), [("task", [_TASK_CHECK]), ("TRACE", [_CHECK, _OTHER_CHECK])] -) -def test_qa_list_filters_by_level(level: str, keys: list[str]) -> None: - result = _invoke(_platform(), ["qa", "list", "--level", level, "--quiet"]) - - assert result.exit_code == 0 - assert result.output.split() == keys - - -@pytest.mark.parametrize("args", [[_CHECK], [" , ", _TRACE_ID]]) -def test_qa_run_requires_checks_and_traces(args: list[str]) -> None: - platform = MagicMock() - - result = _invoke(platform, ["qa", "run", *args]) - - assert result.exit_code == 2 - platform.post.assert_not_called() - - -def test_qa_run_rejects_non_uuid_traces() -> None: - platform = MagicMock() - - result = _invoke(platform, ["qa", "run", _CHECK, "not-a-trace"]) - - assert result.exit_code == 2 - platform.post.assert_not_called() - - -def test_qa_run_no_wait_posts_comma_separated_checks_as_one_run() -> None: - platform = _platform() - platform.post.return_value = { - "results": [_row("queued"), _row("queued", check_key=_OTHER_CHECK)] - } - - result = _invoke( - platform, - [ - "qa", - "run", - f"{_CHECK}, {_OTHER_CHECK},{_CHECK}", - _TRACE_ID.upper(), - "--no-wait", - "--json", - ], - ) - - assert result.exit_code == 0 - assert [row["status"] for row in json.loads(result.output)] == ["queued", "queued"] - platform.post.assert_called_once_with( - "/qa/runs", - json={"check_keys": [_CHECK, _OTHER_CHECK], "trace_ids": [_TRACE_ID], "overwrite": False}, - ) - assert platform.get.call_args_list == [call("/qa/checks")] - - -def test_qa_run_task_checks_take_task_ids_and_wait_on_tasks() -> None: - platform = _platform([_task_result("failed")]) - platform.post.return_value = {"results": [{**_task_result(), "status": "queued"}]} - - with patch("hud.cli.qa.time.sleep"): - result = _invoke(platform, ["qa", "run", _TASK_CHECK, _TASK_ID]) - - assert result.exit_code == 1 - platform.post.assert_called_once_with( - "/qa/runs", - json={"check_keys": [_TASK_CHECK], "task_ids": [_TASK_ID], "overwrite": False}, - ) - platform.get.assert_called_with("/qa/results", params={"task_ids": [_TASK_ID]}) - assert "Task " in result.output - assert _table_row(result.output, _TASK_ID, _TASK_CHECK, "failed") - assert "hud qa results --level task " in result.output - - -@pytest.mark.parametrize( - ("checks", "message"), - [ - ("missing_check", "Unknown QA check: missing_check"), - (f"{_CHECK},{_TASK_CHECK}", "Run trace-level and task-level checks separately."), - ], -) -def test_qa_run_rejects_unknown_or_mixed_level_checks(checks: str, message: str) -> None: - platform = _platform() - - result = _invoke(platform, ["qa", "run", checks, _TRACE_ID]) - - assert result.exit_code == 2 - assert message in result.output - platform.post.assert_not_called() - - -def test_qa_run_dry_run_reports_the_level_without_launching() -> None: - platform = _platform() - - result = _invoke(platform, ["qa", "run", _TASK_CHECK, _TASK_ID, "--dry-run", "--json"]) - - assert result.exit_code == 0 - planned = json.loads(result.stdout) - assert planned["level"] == "task" - assert planned["task_ids"] == [_TASK_ID] - platform.post.assert_not_called() - - -def test_qa_run_forwards_overwrite() -> None: - platform = _platform() - platform.post.return_value = {"results": [_result()]} - - result = _invoke(platform, ["qa", "run", _CHECK, _TRACE_ID, "--overwrite"]) - - assert result.exit_code == 0 - assert platform.post.call_args.kwargs["json"]["overwrite"] is True - - -@pytest.mark.parametrize(("verdict", "exit_code"), [("failed", 1), ("passed", 0)]) -def test_qa_run_waits_and_scores(verdict: str, exit_code: int) -> None: - platform = _platform([_row("running")], [_result(verdict)]) - platform.post.return_value = {"results": [_row("queued")]} - - with patch("hud.cli.qa.time.sleep"): - result = _invoke(platform, ["qa", "run", _CHECK, _TRACE_ID]) - - assert result.exit_code == exit_code - assert _table_row(result.output, _TRACE_ID, _CHECK, verdict) - platform.get.assert_called_with("/qa/results", params={"trace_ids": [_TRACE_ID]}) - - -def test_qa_run_does_not_poll_settled_results() -> None: - platform = _platform() - platform.post.return_value = {"results": [_result("passed")]} - - result = _invoke(platform, ["qa", "run", _CHECK, _TRACE_ID]) - - assert result.exit_code == 0 - assert platform.get.call_args_list == [call("/qa/checks")] - - -def test_qa_run_table_shows_the_summary_headline_as_markdown() -> None: - row = _result("failed") - row["result"] = { - **row["result"], - "summary": "**Verdict:** The grader missed it.\n\n- The answer was right.", - } - platform = _platform() - platform.post.return_value = {"results": [row]} - - result = _invoke(platform, ["qa", "run", _CHECK, _TRACE_ID]) - - assert result.exit_code == 1 - assert _table_row(result.output, _TRACE_ID, _CHECK, "failed", "Verdict: The grader missed it.") - assert "**" not in result.output - assert "The answer was right." not in result.output - - -def test_qa_run_wait_ignores_unrequested_checks() -> None: - platform = _platform([_result("failed", check_key=_OTHER_CHECK), _result("passed")]) - platform.post.return_value = {"results": [_row("queued")]} - - with patch("hud.cli.qa.time.sleep"): - result = _invoke(platform, ["qa", "run", _CHECK, _TRACE_ID]) - - assert result.exit_code == 0 - assert _OTHER_CHECK not in result.output - - -def test_qa_run_errored_check_fails() -> None: - platform = _platform() - platform.post.return_value = { - "results": [_row("error", error="The QA check produced an invalid result.")] - } - - result = _invoke(platform, ["qa", "run", _CHECK, _TRACE_ID]) - - assert result.exit_code == 1 - assert "The QA check produced an invalid result." in result.output - - -def test_qa_run_times_out() -> None: - platform = _platform([_row("running")]) - platform.post.return_value = {"results": [_row("queued")]} - clock = [0.0] - - def sleep(seconds: float) -> None: - clock[0] += seconds - - with ( - patch("hud.cli.qa.time.sleep", side_effect=sleep), - patch("hud.cli.qa.time.monotonic", side_effect=lambda: clock[0]), - ): - result = _invoke(platform, ["qa", "run", _CHECK, _TRACE_ID, "--timeout", "1"]) - - assert result.exit_code != 0 - assert "Timed out" in result.output - - -def test_qa_results_queries_traces() -> None: - platform = MagicMock() - platform.get.return_value = {"results": [_result()]} - - result = _invoke(platform, ["qa", "results", _TRACE_ID, "--json"]) - - assert result.exit_code == 0 - assert json.loads(result.output)[0]["verdict"] == "passed" - platform.get.assert_called_once_with("/qa/results", params={"trace_ids": [_TRACE_ID]}) - - -def test_qa_results_reads_task_history_without_a_trace_link() -> None: - platform = MagicMock() - platform.get.return_value = { - "results": [{**_task_result("failed"), "analysis_trace_id": _ANALYSIS_TRACE_ID}] - } - - result = _invoke(platform, ["qa", "results", "--level", "task", _TASK_ID, "--history"]) - - assert result.exit_code == 0 - platform.get.assert_called_once_with( - "/qa/results", params={"task_ids": [_TASK_ID], "include_history": True} - ) - assert _TASK_ID in result.output - assert f"Task version: {_TASK_VERSION_ID}" in result.output - assert "Verdict: failed" in result.output - assert "View:" not in result.output - assert f"Analysis: https://hud.ai/trace/{_ANALYSIS_TRACE_ID}" in result.output - - -def test_qa_results_tui_renders_findings_and_trace_link() -> None: - platform = MagicMock() - platform.get.return_value = { - "results": [ - _row( - verdict="failed", - result={ - "schema_version": "qa_agent_result.v1", - "verdict": "failed", - "summary": "**Verdict:** The agent never wrote /app/[regex].txt.", - "findings": [ - { - "finding_type": "missing_output", - "severity": "error", - "summary": "Required [/output] file was never created", - "recommended_action": "Write the regex to /app/regex.txt.", - "evidence_refs": ["trajectory.json"], - } - ], - "metadata": {"confidence": 0.9}, - }, - ) - ] - } - - result = _invoke(platform, ["qa", "results", _TRACE_ID]) - - assert result.exit_code == 0 - assert _CHECK in result.output - assert "Verdict: failed" in result.output - assert "Confidence: 90%" in result.output - assert "Verdict: The agent never wrote /app/[regex].txt." in result.output - assert "**" not in result.output - assert "1. Required [/output] file was never created" in result.output - assert "Action: Write the regex to /app/regex.txt." in result.output - assert "severity: error" in result.output - assert "evidence: trajectory.json" in result.output - assert f"View: https://hud.ai/trace/{_TRACE_ID}" in result.output - assert "Analysis:" not in result.output - - -def test_qa_results_links_the_analysis_trace_when_returned() -> None: - platform = MagicMock() - platform.get.return_value = { - "results": [{**_result(), "analysis_trace_id": _ANALYSIS_TRACE_ID}] - } - - result = _invoke(platform, ["qa", "results", _TRACE_ID]) - - assert result.exit_code == 0 - assert f"https://hud.ai/trace/{_ANALYSIS_TRACE_ID}" in result.output - - -def test_qa_results_skipped_check_shows_source_and_note() -> None: - platform = MagicMock() - platform.get.return_value = { - "results": [ - _row( - verdict="passed", - source="skipped", - note="A full reward passes this check without analysis.", - ) - ] - } - - result = _invoke(platform, ["qa", "results", _TRACE_ID]) - - assert result.exit_code == 0 - assert "Verdict: passed" in result.output - assert "Source: skipped" in result.output - assert "A full reward passes this check without analysis." in result.output - - -def test_qa_results_legacy_failure_analysis() -> None: - platform = MagicMock() - platform.get.return_value = { - "results": [ - _row( - legacy_result={ - "content": json.dumps( - { - "summary": "The agent never wrote /app/regex.txt.", - "confidence": "high", - "problems": [ - { - "problem": "Required output file was never created", - "description": "The agent did not save any regex.", - "fault": "agent", - } - ], - } - ), - "reward": 0.0, - } - ) - ] - } - - result = _invoke(platform, ["qa", "results", _TRACE_ID]) - - assert result.exit_code == 0 - assert "Verdict: failed" in result.output - assert "Cause: Agent failure" in result.output - assert "Required output file was never created" in result.output - assert "The agent did not save any regex." in result.output - - -def test_qa_results_legacy_boolean_omits_findings() -> None: - platform = MagicMock() - platform.get.return_value = { - "results": [ - _row( - check_key="false_negative", - legacy_result={ - "content": json.dumps( - { - "is_false_negative": False, - "reasoning": "The zero reward matches the missing file.", - "confidence": "high", - } - ), - }, - ) - ] - } - - result = _invoke(platform, ["qa", "results", _TRACE_ID]) - - assert result.exit_code == 0 - assert "Verdict: passed" in result.output - assert "False Negative: no" in result.output - assert "1. " not in result.output - assert "The zero reward matches the missing file." in result.output - - -def test_row_verdict_overrides_the_output_verdict() -> None: - view = presentation_for_result( - { - **_result("failed"), - "source": "manual", - "verdict": "passed", - "note": "Reviewed.", - } - ) - - assert view.tag == "passed" - assert view.summary == "A gap was found." - - -def test_legacy_failure_analysis_problems_are_a_failed_agent_finding() -> None: - view = presentation_for_result( - _row( - legacy_result={ - "content": ( - '{"summary": "Missing file.", "problems": [' - '{"problem": "No regex", "fault": "agent", "description": "Never wrote it."}' - '], "confidence": "high"}' - ) - } - ) - ) - - assert view.kind == "problems" - assert view.tag == "failed" - assert view.answer == "Agent failure" - assert view.findings[0].title == "No regex" - assert view.findings[0].fault == "agent" - - -def test_legacy_failure_analysis_empty_problems_is_passed() -> None: - view = presentation_for_result( - _row(legacy_result={"summary": "Clean.", "problems": [], "confidence": "high"}) - ) - - assert view.tag == "passed" - assert view.answer == "No failure" - assert view.findings == () - - -def test_legacy_mixed_faults_are_labeled_mixed_failure() -> None: - view = presentation_for_result( - _row( - legacy_result={ - "problems": [ - {"problem": "Bad regex", "fault": "agent"}, - {"problem": "Cut off", "fault": "unclear"}, - ] - } - ) - ) - - assert view.tag == "failed" - assert view.answer == "Mixed failure" - - -def test_legacy_false_negative_yes_is_failed_without_findings() -> None: - view = presentation_for_result( - _row( - legacy_result={ - "content": '{"is_false_negative": true, "reasoning": "Grader missed it."}' - } - ) - ) - - assert view.kind == "boolean" - assert view.tag == "failed" - assert view.label == "False Negative" - assert view.answer == "yes" - assert view.findings == () - assert view.summary == "Grader missed it." - - -def test_queued_results_are_pending_not_passed() -> None: - view = presentation_for_result(_row("queued")) - - assert view.kind == "pending" - assert view.tag == "unknown" - assert view.label == "queued" diff --git a/hud/cli/tests/test_sync.py b/hud/cli/tests/test_sync.py deleted file mode 100644 index fa752b4b0..000000000 --- a/hud/cli/tests/test_sync.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Tests for ``hud.cli.sync``.""" - -from __future__ import annotations - -import json -from typing import TYPE_CHECKING, Any -from urllib.parse import parse_qs, urlsplit - -import pytest -from typer.testing import CliRunner - -from hud.cli import AuthScope, CliError, DirectoryState -from hud.cli.__main__ import app -from hud.cli.sync import RegistryEnvironment -from hud.eval import Task, Taskset -from hud.utils.exceptions import HudRequestError -from hud.utils.platform import PlatformClient - -if TYPE_CHECKING: - from pathlib import Path - - -_TASKSET_ID = "44444444-4444-4444-8444-444444444444" -_READONLY_PROJECT = "33333333-3333-4333-8333-333333333333" -_WRITABLE_PROJECT = "22222222-2222-4222-8222-222222222222" -_ROW = {"env": "example", "id": "solve", "slug": "one"} - - -def _stub_platform( - monkeypatch: pytest.MonkeyPatch, *, remote_rows: list[dict[str, Any]], uploads: list[Any] -) -> None: - """A platform holding taskset ``demo`` with ``remote_rows`` and two projects.""" - - def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - if url.endswith("/auth/me"): - return { - "user_id": "11111111-1111-4111-8111-111111111111", - "team_id": _WRITABLE_PROJECT, - } - if url.endswith(f"/projects/{_READONLY_PROJECT}"): - return {"id": _READONLY_PROJECT, "name": "locked-down", "capabilities": {}} - if url.endswith(f"/projects/{_WRITABLE_PROJECT}"): - return { - "id": _WRITABLE_PROJECT, - "name": "browser-evals", - "capabilities": {"create": True}, - } - if url.endswith("/tasksets/by-name/demo"): - return {"taskset_id": _TASKSET_ID, "name": "demo"} - if url.endswith(f"/tasksets/{_TASKSET_ID}"): - return {"id": _TASKSET_ID, "name": "demo"} - if url.endswith(f"/tasksets/{_TASKSET_ID}/export"): - return { - "name": "demo", - "tasks": [ - {"env": row["env"], "scenario": row["id"], "name": row["slug"]} - for row in remote_rows - ], - } - if url.endswith("/tasks/upload"): - uploads.append(kwargs["json"]) - return {"taskset_id": _TASKSET_ID, "tasks_created": 1} - raise AssertionError((method, url)) - - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - monkeypatch.setattr("hud.settings.settings.default_project", None) - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) - - -@pytest.mark.parametrize("dry_run", [False, True]) -def test_read_only_project_allows_no_op_and_preview( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, dry_run: bool -) -> None: - """A read-only project is only refused when something would actually upload.""" - monkeypatch.chdir(tmp_path) - (tmp_path / "tasks.json").write_text(json.dumps([_ROW])) - uploads: list[Any] = [] - # Up to date: identical remote row; dry run: empty remote, so a create is planned. - _stub_platform(monkeypatch, remote_rows=[] if dry_run else [_ROW], uploads=uploads) - - args = [ - "sync", - "tasks", - "demo", - "tasks.json", - "--project", - _READONLY_PROJECT, - "--yes", - "--json", - ] - result = CliRunner().invoke(app, [*args, "--dry-run"] if dry_run else args) - - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["dry_run"] is dry_run - assert uploads == [] - - if dry_run: - refused = CliRunner().invoke(app, args) - assert refused.exit_code == 1, refused.output - assert json.loads(refused.stdout)["error"] == "permission_denied" - assert uploads == [] - - -def test_project_override_does_not_pin_directory( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.chdir(tmp_path) - (tmp_path / "tasks.json").write_text(json.dumps([_ROW])) - uploads: list[Any] = [] - _stub_platform(monkeypatch, remote_rows=[], uploads=uploads) - - result = CliRunner().invoke( - app, - ["sync", "tasks", "demo", "tasks.json", "--project", _WRITABLE_PROJECT, "--yes", "--json"], - ) - - assert result.exit_code == 0, result.output - assert uploads[0]["project_id"] == _WRITABLE_PROJECT - assert not (tmp_path / ".hud" / "config.json").exists() - - -@pytest.mark.parametrize("stale", [False, True]) -def test_sync_uses_stored_id_after_rename_and_never_recreates_stale_link( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, stale: bool -) -> None: - source = tmp_path / "tasks.py" - source.write_text( - "from hud.eval import Task\ntasks = [Task(env='example', id='solve', slug='one')]\n" - ) - monkeypatch.chdir(tmp_path) - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - monkeypatch.setattr("hud.settings.settings.default_project", None) - taskset_id = "55555555-5555-4555-8555-555555555555" - uploads: list[dict[str, Any]] = [] - - def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - if url.endswith("/auth/me"): - return { - "user_id": "11111111-1111-4111-8111-111111111111", - "team_id": "22222222-2222-4222-8222-222222222222", - } - if "/by-name/" in url: - raise HudRequestError("missing", status_code=404) - if url.endswith("/tasks/upload"): - uploads.append(kwargs["json"]) - return {"taskset_id": kwargs["json"].get("taskset_id", taskset_id), "tasks_created": 1} - if "/tasksets/" in url: - if stale: - raise HudRequestError("deleted", status_code=404) - return {"id": taskset_id, "name": "renamed", "tasks": []} - raise AssertionError(url) - - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) - first = CliRunner().invoke(app, ["sync", "tasks", "demo", str(source), "--yes", "--json"]) - assert first.exit_code == 0, first.output - state = DirectoryState(AuthScope.resolve(PlatformClient.from_settings()), tmp_path) - before = state.load() - second = CliRunner().invoke(app, ["sync", "tasks", "--yes", "--json", "--force"]) - assert state.load() == before - if stale: - assert second.exit_code == 1, second.output - assert json.loads(second.stdout)["error"] == "not_found" - assert len(uploads) == 1 - else: - assert second.exit_code == 0, second.output - assert uploads[-1]["taskset_id"] == taskset_id - assert uploads[-1]["taskset_name"] == "renamed" - - other = "66666666-6666-4666-8666-666666666666" - args = ["sync", "tasks", other, str(source), "--force", "--yes", "--json"] - override = CliRunner().invoke(app, args) - assert override.exit_code == 0, override.output - assert uploads[-1]["taskset_id"] == other - assert state.load() == before - planned_link = CliRunner().invoke(app, [*args, "--link", "--dry-run"]) - assert planned_link.exit_code == 0, planned_link.output - assert state.load() == before - relinked = CliRunner().invoke(app, [*args, "--link"]) - assert relinked.exit_code == 0, relinked.output - assert str(state.load().taskset_id) == other - - alias = CliRunner().invoke(app, ["sync", "tasks", "--yes", "--json", "--force"]) - assert alias.exit_code == 0, alias.output - assert json.loads(alias.stdout)["taskset_id"] == other - - -@pytest.mark.parametrize( - ("status_code", "exit_code", "error"), - [ - (400, 1, "failure"), - (403, 1, "permission_denied"), - (500, 1, "server_error"), - ], -) -def test_rejected_upload_exits_with_failure( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - status_code: int, - exit_code: int, - error: str, -) -> None: - project_id = "22222222-2222-4222-8222-222222222222" - detail = "Taskset belongs to another Project" if status_code == 400 else "Upload rejected" - source = tmp_path / "tasks.json" - source.write_text(json.dumps([{"env": "example", "id": "solve", "slug": "one"}])) - monkeypatch.chdir(tmp_path) - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - monkeypatch.setattr("hud.settings.settings.default_project", None) - - def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - if url.endswith("/auth/me"): - return { - "user_id": "11111111-1111-4111-8111-111111111111", - "team_id": "22222222-2222-4222-8222-222222222222", - } - if url.endswith(f"/projects/{project_id}"): - return {"id": project_id, "name": "browser-evals", "capabilities": {"create": True}} - if "/by-name/" in url: - raise HudRequestError("missing", status_code=404) - if url.endswith("/tasks/upload"): - raise HudRequestError(detail, status_code=status_code, response_json={"detail": detail}) - raise AssertionError(url) - - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) - result = CliRunner().invoke( - app, - [ - "sync", - "tasks", - "demo", - str(source), - "--project", - project_id, - "--force", - "--yes", - "--json", - ], - ) - - assert result.exit_code == exit_code, result.output - payload = json.loads(result.stdout) - assert payload["error"] == error - assert detail in payload["message"] - assert "Sync complete" not in result.output - assert not (tmp_path / ".hud" / "config.json").exists() - - -def test_export_csv_flattens_args(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.chdir(tmp_path) - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - remote = Taskset( - "demo", - [ - Task(env="e", id="solve", args={"n": 1}, slug="one"), - Task(env="e", id="solve", args={"n": {"x": 2}}, slug="two"), - ], - ) - monkeypatch.setattr(Taskset, "from_api", classmethod(lambda cls, name: remote)) - monkeypatch.setattr( - "hud.utils.platform.make_request_sync", - lambda method, url, **kw: { - "user_id": "11111111-1111-4111-8111-111111111111", - "team_id": _WRITABLE_PROJECT, - }, - ) - - result = CliRunner().invoke(app, ["sync", "tasks", "demo", "--export", "tasks.csv", "--json"]) - - assert result.exit_code == 0, result.output - assert json.loads(result.stdout) == { - "action": "export", - "taskset": "demo", - "path": "tasks.csv", - "task_count": 2, - } - csv_text = (tmp_path / "tasks.csv").read_text() - assert "slug,id,env,arg:n" in csv_text - assert "one,solve,e,1" in csv_text - assert 'two,solve,e,"{""x"": 2}"' in csv_text - - -def test_sync_env_noninteractive_requires_name(tmp_path: Path) -> None: - result = CliRunner().invoke(app, ["sync", "env", "--json"]) - assert result.exit_code == 2, result.output - assert json.loads(result.stdout)["error"] == "usage" - - -@pytest.mark.parametrize("failure", ["source", "export", "upload", "registry"]) -def test_sync_failures_render_one_structured_error( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str -) -> None: - monkeypatch.chdir(tmp_path) - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - monkeypatch.setattr("hud.settings.settings.default_project", None) - source = tmp_path / "tasks.py" - source.write_text("from hud.eval import Task\ntasks = [Task(env='e', id='solve')]\n") - registry_id = "33333333-3333-4333-8333-333333333333" - - def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - if url.endswith("/auth/me"): - return { - "user_id": "11111111-1111-4111-8111-111111111111", - "team_id": "22222222-2222-4222-8222-222222222222", - } - if failure == "upload" and "/by-name/" in url: - raise HudRequestError("missing", status_code=404) - raise HudRequestError("Access denied", status_code=403) - - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) - args = { - "source": ["tasks", "demo", str(tmp_path / "missing.json")], - "export": ["tasks", "demo", "--export", str(tmp_path / "out.json")], - "upload": ["tasks", "demo", str(source), "--yes"], - "registry": ["env", registry_id], - }[failure] - result = CliRunner().invoke(app, ["sync", *args, "--json"]) - assert result.exit_code == 1, result.output - payload = json.loads(result.stdout) - assert payload["error"] == ("not_found" if failure == "source" else "permission_denied") - assert payload["message"] not in (result.stderr or "") - - -def test_relinking_same_environment_reports_unchanged( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - registry_id = "33333333-3333-4333-8333-333333333333" - - def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - if url.endswith("/auth/me"): - return { - "user_id": "11111111-1111-4111-8111-111111111111", - "team_id": "22222222-2222-4222-8222-222222222222", - } - assert url.endswith(f"/registry/{registry_id}") - return {"id": registry_id, "name": "example"} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) - args = ["sync", "env", registry_id, str(tmp_path), "--json"] - for changed in (True, False): - result = CliRunner().invoke(app, args) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["changed"] is changed - - -def test_resolve_verifies_uuid(monkeypatch: pytest.MonkeyPatch) -> None: - def request(method: str, url: str, **kwargs: object) -> dict[str, str]: - assert url.endswith("/registry/12345678-1234-5678-1234-567812345678") - return {"id": "12345678-1234-5678-1234-567812345678", "name": "verified"} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) - env = RegistryEnvironment.resolve( - PlatformClient("https://api.example", "key"), - "12345678-1234-5678-1234-567812345678", - ) - - assert env == RegistryEnvironment( - id="12345678-1234-5678-1234-567812345678", - name="verified", - ) - - -def test_get_registry_environment_treats_404_as_missing(monkeypatch: pytest.MonkeyPatch) -> None: - def fake_request(method: str, url: str, **kwargs: object) -> dict[str, Any]: - raise HudRequestError("not found", status_code=404) - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - - with pytest.raises(CliError, match="inaccessible or deleted") as error: - RegistryEnvironment.resolve( - PlatformClient("https://api.example", "key"), "12345678-1234-5678-1234-567812345678" - ) - assert error.value.exit_code == 1 - - -def test_resolve_matches_a_name_exactly(monkeypatch: pytest.MonkeyPatch) -> None: - records = [ - {"id": "12345678-1234-5678-1234-567812345678", "name": "browser"}, - {"id": "87654321-4321-8765-4321-876543218765", "name": "browser-anchor"}, - ] - - def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - parts = urlsplit(url) - assert parts.path.endswith("/registry") - search = parse_qs(parts.query)["search"][0] - items = [record for record in records if search in record["name"]] - return {"items": items, "total": len(items)} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", request) - platform = PlatformClient("https://api.example", "key") - - assert RegistryEnvironment.resolve(platform, "Browser").id == records[0]["id"] - assert RegistryEnvironment.resolve(platform, "browser_anchor").id == records[1]["id"] - with pytest.raises(CliError, match="No environment named 'anchor'") as error: - RegistryEnvironment.resolve(platform, "anchor") - assert error.value.error == "not_found" diff --git a/hud/cli/tests/test_task.py b/hud/cli/tests/test_task.py deleted file mode 100644 index 0b16dfb80..000000000 --- a/hud/cli/tests/test_task.py +++ /dev/null @@ -1,149 +0,0 @@ -import asyncio -import json - -import pytest -from typer.testing import CliRunner - -from hud.cli import task as task_module -from hud.cli.__main__ import app -from hud.eval import Task, Taskset - - -@pytest.mark.parametrize("override", [None, {}]) -async def test_source_resolves_authored_task_for_existing_runtime(tmp_path, override): - authored = Task( - env="coding", - id="coding-task", - slug="flask-4992", - args={"description": "Fix Flask", "test_script": "pytest"}, - ) - source = Taskset("authored", [authored]).to_file(tmp_path / "tasks.json") - - task_id, args, placement = task_module._resolve( - "flask-4992", - str(source), - "tcp://127.0.0.1:9000", - override, - ) - - assert task_id == "coding-task" - assert args == (authored.args if override is None else override) - async with placement as runtime: - assert runtime.url == "tcp://127.0.0.1:9000" - - -async def test_url_without_source_uses_raw_task_and_args(monkeypatch): - def fail(cls, source): - raise AssertionError(f"unexpected task source: {source}") - - monkeypatch.setattr(Taskset, "from_file", classmethod(fail)) - - task_id, args, placement = task_module._resolve( - "coding-task", - None, - "tcp://127.0.0.1:9000", - {"description": "Fix Flask"}, - ) - - assert task_id == "coding-task" - assert args == {"description": "Fix Flask"} - async with placement as runtime: - assert runtime.url == "tcp://127.0.0.1:9000" - - -async def test_task_source_uses_sibling_environment_for_start_and_grade(tmp_path, monkeypatch): - import sys - - from hud.clients import connect - - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "false") - monkeypatch.delitem(sys.modules, "env", raising=False) - (tmp_path / "env.py").write_text( - 'from hud import Environment\nenv = Environment("example")\n' - '@env.template(id="solve")\nasync def solve():\n' - ' answer = yield "question"\n yield 1.0 if answer == "answer" else 0.0\n' - ) - source = tmp_path / "tasks.py" - source.write_text("from env import solve\n\ntasks = [solve()]\n") - try: - task_id, args, placement = task_module._resolve("solve", str(source), None, {}) - async with placement as runtime, connect(runtime) as client: - await client.start_task(task_id, args) - result = await client.grade({"answer": "answer"}) - finally: - sys.modules.pop("env", None) - assert result["score"] == 1.0 - - -@pytest.mark.parametrize("mode", ["empty", "parked", "failed_grade", "ambiguous"]) -async def test_grade_only_starts_when_no_task_is_in_progress(mode): - from hud.clients import connect - from hud.environment import Environment - from hud.eval import LocalRuntime - - env = Environment("grading") - starts = 0 - - @env.template() - async def solve(): - nonlocal starts - starts += 1 - yield "question" - if mode == "failed_grade": - raise ValueError("grader failed") - yield 1.0 - - async with LocalRuntime(env)(Task(env="grading", id="solve")) as runtime: - for _ in range(2 if mode == "ambiguous" else int(mode != "empty")): - async with connect(runtime) as client: - await client.start_task("solve", {}) - result = await asyncio.to_thread( - CliRunner().invoke, - app, - ["task", "grade", "solve", "--url", runtime.url, "--json"], - ) - assert starts == (2 if mode == "ambiguous" else 1) - if mode in {"empty", "parked"}: - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["score"] == 1.0 - else: - assert result.exit_code != 0 - assert ( - "grader failed" in result.output - if mode == "failed_grade" - else "2 parked sessions" in result.output - ) - - -async def test_json_source_spawns_the_env_beside_it(tmp_path, monkeypatch): - from hud.clients import connect - - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "false") - (tmp_path / "env.py").write_text( - 'from hud import Environment\nenv = Environment("example")\n' - '@env.template(id="solve")\nasync def solve():\n' - ' answer = yield "question"\n yield 1.0 if answer == "answer" else 0.0\n' - ) - source = Taskset( - "authored", - [Task(env="example", id="solve", slug="solve")], - ).to_file(tmp_path / "tasks.json") - - task_id, args, placement = task_module._resolve("solve", str(source), None, None) - async with placement as runtime, connect(runtime) as client: - await client.start_task(task_id, args) - result = await client.grade({"answer": "answer"}) - assert result["score"] == 1.0 - - -def test_task_id_matching_multiple_rows_requires_unique_slug(tmp_path): - source = Taskset( - "authored", - [ - Task(env="example", id="solve", slug="first"), - Task(env="example", id="solve", slug="second"), - ], - ).to_file(tmp_path / "tasks.json") - result = CliRunner().invoke(app, ["task", "start", "solve", "--source", str(source), "--json"]) - assert result.exit_code == 2 - assert "Ambiguous task" in json.loads(result.stdout)["message"] diff --git a/hud/cli/tests/test_trace.py b/hud/cli/tests/test_trace.py deleted file mode 100644 index 0214deb7b..000000000 --- a/hud/cli/tests/test_trace.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import json -from typing import TYPE_CHECKING - -import pytest -from typer.testing import CliRunner - -from hud.cli.__main__ import app -from hud.settings import settings -from hud.utils.platform import PlatformClient - -if TYPE_CHECKING: - from pathlib import Path - - -def test_trace_link_uses_web_uuid(monkeypatch: pytest.MonkeyPatch) -> None: - trace_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - monkeypatch.setattr(settings, "api_key", "test-key") - monkeypatch.setattr(settings, "telemetry_local_dir", None) - monkeypatch.setattr( - PlatformClient, - "get", - lambda self, path: {"events": [{"kind": "agent_message", "text": "done"}]}, - ) - - result = CliRunner().invoke(app, ["trace", "get", trace_id]) - - assert result.exit_code == 0 - assert "https://hud.ai/trace/03dd2a73-d3df-4d10-a54a-e3d87c2d530d" in result.stdout - - -@pytest.mark.parametrize("options_first", [False, True]) -@pytest.mark.parametrize("verb", [[], ["get"]]) -def test_trace_json_accepts_options_on_either_side_of_id( - monkeypatch: pytest.MonkeyPatch, options_first: bool, verb: list[str] -) -> None: - trace_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - events = [{"kind": "agent_message", "text": "done"}] - monkeypatch.setattr(settings, "api_key", "test-key") - monkeypatch.setattr(settings, "telemetry_local_dir", None) - - def get_events(self: PlatformClient, path: str) -> dict[str, object]: - assert path == f"/trace/{trace_id}/events" - return {"events": events} - - monkeypatch.setattr(PlatformClient, "get", get_events) - - args = ["--json", trace_id] if options_first else [trace_id, "--json"] - result = CliRunner().invoke(app, ["trace", *verb, *args]) - - assert result.exit_code == 0, result.output - assert json.loads(result.stdout) == events - - -def test_local_trace_skips_a_record_cut_short_by_an_interrupted_writer( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - trace_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - monkeypatch.setattr(settings, "telemetry_local_dir", str(tmp_path)) - step = { - "start_time": "2026-01-01T00:00:00Z", - "attributes": { - "hud.schema": "hud.step.v1", - "hud.payload": {"source": "agent", "content": "done"}, - }, - } - (tmp_path / f"{trace_id}.jsonl").write_text( - json.dumps(step) + "\n" + json.dumps(step)[:40], encoding="utf-8" - ) - - result = CliRunner().invoke(app, ["trace", "get", trace_id, "--json"]) - - assert result.exit_code == 0, result.output - assert [event["kind"] for event in json.loads(result.stdout)] == ["agent_message"] - assert "Skipped 1 incomplete span record" in result.output diff --git a/hud/cli/trace.py b/hud/cli/trace.py index bc2f0b86b..032b326e8 100644 --- a/hud/cli/trace.py +++ b/hud/cli/trace.py @@ -92,23 +92,23 @@ def get_command( } ) elif payload.get("source") == "tool": - for msg in payload.get("messages", []): - if msg.get("role") != "tool": - continue - content = msg.get("content", "") - if isinstance(content, list): - content = "\n".join( - part.get("text", "") - for part in content - if isinstance(part, dict) and part.get("type") == "text" - ) - events.append( - { - "kind": "tool_call", - "tool_name": msg.get("name") or msg.get("tool_call_id"), - "result_text": str(content), - } - ) + # A tool step pairs the originating call with its MCP result. + call = payload.get("call") or {} + result = payload.get("result") or {} + text = "\n".join( + block.get("text", "") + for block in result.get("content") or [] + if block.get("type") == "text" + ) + events.append( + { + "kind": "tool_call", + "tool_name": call.get("name"), + "arguments": call.get("arguments") or {}, + "result_text": text, + "error": payload.get("error") or (text if result.get("isError") else None), + } + ) else: source = "platform" try: diff --git a/hud/clients/tests/__init__.py b/hud/clients/tests/__init__.py deleted file mode 100644 index b885a6036..000000000 --- a/hud/clients/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for the HUD wire-protocol client.""" diff --git a/hud/clients/tests/test_connect.py b/hud/clients/tests/test_connect.py deleted file mode 100644 index 7aa42455c..000000000 --- a/hud/clients/tests/test_connect.py +++ /dev/null @@ -1,427 +0,0 @@ -"""``connect()`` readiness: the handshake retries until the env actually serves. - -A provisioned substrate can sit behind a proxied port (``docker -p``, a -port-forward) that *accepts* TCP before the env behind it is up — those -connections die with EOF at the handshake. Readiness is therefore -protocol-level: ``connect`` keeps retrying through both refused connects and -handshake EOFs until ``hello`` answers or the deadline passes. -""" - -from __future__ import annotations - -import asyncio -import logging -from typing import ClassVar -from urllib.parse import urlsplit - -import pytest - -import hud.clients.client as client_module -from hud.capabilities import Capability, CapabilityClient, Connection -from hud.clients import HudProtocolError, connect -from hud.environment import WorkspaceRoute -from hud.environment.utils import read_frame, send_frame -from hud.eval.runtime import Runtime - -HELLO_RESULT = {"session_id": "s-1", "env": {"name": "stub", "version": "1.0"}, "bindings": []} - - -def test_workspace_route_from_url_extracts_transport_address() -> None: - assert WorkspaceRoute.from_url("ssh", "https://inference.hud.so/v1") == WorkspaceRoute( - "ssh", - "inference.hud.so", - 443, - ) - assert WorkspaceRoute.from_url("shell", "http://gateway.test:8080") == WorkspaceRoute( - "shell", - "gateway.test", - 8080, - ) - - -async def test_connect_sends_workspace_routes_in_hello() -> None: - requests: list[dict[str, object]] = [] - - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - requests.append(hello) - await send_frame(writer, {"jsonrpc": "2.0", "id": hello["id"], "result": HELLO_RESULT}) - await read_frame(reader) - finally: - writer.close() - - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - runtime = Runtime(f"tcp://127.0.0.1:{port}") - route = WorkspaceRoute("ssh", "inference.hud.so", 443) - try: - async with connect(runtime, workspace_routes=(route,)): - pass - finally: - server.close() - await server.wait_closed() - - assert [request["method"] for request in requests] == ["hello"] - params = requests[0]["params"] - assert isinstance(params, dict) - assert params == {"workspace_routes": [route.to_wire()]} - - -async def test_connect_sends_controller_connections_in_hello() -> None: - requests: list[dict[str, object]] = [] - - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - requests.append(hello) - await send_frame(writer, {"jsonrpc": "2.0", "id": hello["id"], "result": HELLO_RESULT}) - await read_frame(reader) - finally: - writer.close() - - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer secret"}, - ) - try: - async with connect( - Runtime(f"tcp://127.0.0.1:{port}"), - connections=(connection,), - ): - pass - finally: - server.close() - await server.wait_closed() - - params = requests[0]["params"] - assert isinstance(params, dict) - assert params == {"connections": [connection.to_wire()]} - - -async def test_open_retries_transient_capability_connection_failures( - monkeypatch: pytest.MonkeyPatch, -) -> None: - class RetryingClient(CapabilityClient): - protocol: ClassVar[str] = "test/1" - attempts = 0 - - @classmethod - async def connect(cls, cap: Capability) -> RetryingClient: - del cap - cls.attempts += 1 - if cls.attempts < 3: - raise ConnectionError("connection reset") - return cls() - - async def close(self) -> None: - pass - - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - await send_frame( - writer, - { - "jsonrpc": "2.0", - "id": hello["id"], - "result": { - **HELLO_RESULT, - "bindings": [ - { - "name": "test", - "protocol": RetryingClient.protocol, - "url": "tcp://environment:1234", - } - ], - }, - }, - ) - await read_frame(reader) - finally: - writer.close() - - monkeypatch.setitem(client_module._CLIENT_REGISTRY, RetryingClient.protocol, RetryingClient) - monkeypatch.setattr(client_module, "_CAPABILITY_CONNECT_BASE_DELAY_SECONDS", 0) - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - async with connect(Runtime(f"tcp://127.0.0.1:{port}")) as client: - opened = await client.open("test") - assert await client.open("test") is opened - finally: - server.close() - await server.wait_closed() - - assert RetryingClient.attempts == 3 - - -async def test_connect_retries_through_accept_then_eof_until_the_env_serves() -> None: - attempts = 0 - - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - nonlocal attempts - attempts += 1 - if attempts <= 2: - # The docker-proxy shape: accept, then hang up without serving. - writer.close() - return - try: - msg = await read_frame(reader) - assert msg is not None - await send_frame(writer, {"jsonrpc": "2.0", "id": msg["id"], "result": HELLO_RESULT}) - await read_frame(reader) # hold the connection until the client closes - finally: - # 3.12's Server.wait_closed() waits on every connection; a handler - # that returns without closing its writer deadlocks teardown. - writer.close() - - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - async with connect(Runtime(f"tcp://127.0.0.1:{port}"), ready_timeout=10) as client: - assert client.manifest is not None - assert client.manifest.server_info.name == "stub" - finally: - server.close() - await server.wait_closed() - - assert attempts == 3 - - -async def test_tunnel_connection_failure_warns_with_peer( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - await send_frame( - writer, - { - "jsonrpc": "2.0", - "id": hello["id"], - "result": { - **HELLO_RESULT, - "bindings": [ - { - "name": "shell", - "protocol": "ssh/2", - "url": "ssh://agent@environment:22", - } - ], - }, - }, - ) - await read_frame(reader) - finally: - writer.close() - - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - open_connection = asyncio.open_connection - - async def fail_tunnel_connection(host: str, peer_port: int): - assert (host, peer_port) == ("127.0.0.1", port) - raise OSError("peer unavailable") - - try: - async with connect(Runtime(f"tcp://127.0.0.1:{port}")) as client: - route = urlsplit(client.binding("shell").url) - monkeypatch.setattr(client_module.asyncio, "open_connection", fail_tunnel_connection) - caplog.set_level(logging.WARNING, logger="hud.clients") - - reader, writer = await open_connection(route.hostname, route.port) - try: - assert await reader.read() == b"" - finally: - writer.close() - await writer.wait_closed() - finally: - server.close() - await server.wait_closed() - - assert f"tunnel peer 127.0.0.1:{port} connection failed: peer unavailable" in caplog.messages - - -async def test_heartbeat_serializes_calls_without_aborting_on_protocol_errors( - monkeypatch: pytest.MonkeyPatch, -) -> None: - requests: list[dict[str, object]] = [] - heartbeat_received = asyncio.Event() - release_heartbeat = asyncio.Event() - - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - requests.append(hello) - await send_frame( - writer, - {"jsonrpc": "2.0", "id": hello["id"], "result": HELLO_RESULT}, - ) - - heartbeat = await read_frame(reader) - assert heartbeat is not None - requests.append(heartbeat) - heartbeat_received.set() - await release_heartbeat.wait() - await send_frame( - writer, - { - "jsonrpc": "2.0", - "id": heartbeat["id"], - "error": {"code": -32601, "message": "heartbeat refused"}, - }, - ) - - grade = await read_frame(reader) - assert grade is not None - requests.append(grade) - await send_frame( - writer, - {"jsonrpc": "2.0", "id": grade["id"], "result": {"score": 1.0}}, - ) - await read_frame(reader) - finally: - writer.close() - - monkeypatch.setattr(client_module, "_CONTROL_HEARTBEAT_INTERVAL_SECONDS", 0.01) - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - async with connect(Runtime(f"tcp://127.0.0.1:{port}")) as client: - await asyncio.wait_for(heartbeat_received.wait(), 1.0) - grade = asyncio.create_task(client.grade({"answer": "done"})) - await asyncio.sleep(0) - assert [request["method"] for request in requests] == ["hello", "hello"] - release_heartbeat.set() - assert await asyncio.wait_for(grade, 1.0) == {"score": 1.0} - finally: - server.close() - await server.wait_closed() - - assert [request["method"] for request in requests] == ["hello", "hello", "tasks.grade"] - assert requests[1]["params"] == {"session_id": "s-1"} - - -async def test_heartbeat_transport_failure_interrupts_the_connection_body( - monkeypatch: pytest.MonkeyPatch, -) -> None: - heartbeat_received = asyncio.Event() - body_interrupted = asyncio.Event() - - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - await send_frame( - writer, - {"jsonrpc": "2.0", "id": hello["id"], "result": HELLO_RESULT}, - ) - heartbeat = await read_frame(reader) - assert heartbeat is not None - heartbeat_received.set() - await reader.read() - finally: - writer.close() - - monkeypatch.setattr(client_module, "_CONTROL_HEARTBEAT_INTERVAL_SECONDS", 0.01) - monkeypatch.setattr(client_module, "_CONTROL_HEARTBEAT_TIMEOUT_SECONDS", 0.01) - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - with pytest.raises(TimeoutError): - async with connect(Runtime(f"tcp://127.0.0.1:{port}")): - await asyncio.wait_for(heartbeat_received.wait(), 1.0) - try: - await asyncio.Event().wait() - except asyncio.CancelledError: - body_interrupted.set() - raise - finally: - server.close() - await server.wait_closed() - - assert body_interrupted.is_set() - - -async def test_connect_uses_runtime_ready_timeout_param( - monkeypatch: pytest.MonkeyPatch, -) -> None: - seen: dict[str, float | int | str] = {} - - class _FakeClient: - async def close(self) -> None: - pass - - async def fake_connect_ready( - host: str, - port: int, - *, - ready_timeout: float, - workspace_routes: tuple[WorkspaceRoute, ...], - connections: tuple[Connection, ...], - interval: float = 0.5, - ) -> _FakeClient: - seen["host"] = host - seen["port"] = port - seen["ready_timeout"] = ready_timeout - assert workspace_routes == () - assert connections == () - seen["interval"] = interval - return _FakeClient() - - monkeypatch.setattr(client_module, "_connect_ready", fake_connect_ready) - - async with client_module.connect( - Runtime("tcp://127.0.0.1:1234", params={"ready_timeout": 300.0}) - ): - pass - - assert seen == { - "host": "127.0.0.1", - "port": 1234, - "ready_timeout": 300.0, - "interval": 0.5, - } - - -@pytest.mark.parametrize("protocol_error", [False, True]) -async def test_connect_reports_handshake_failure(protocol_error: bool) -> None: - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - await read_frame(reader) - if protocol_error: - await send_frame( - writer, - { - "jsonrpc": "2.0", - "id": None, - "error": {"code": -32000, "message": "frame exceeds size limit"}, - }, - ) - finally: - writer.close() - - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - expected = HudProtocolError if protocol_error else EOFError - message = ( - "frame exceeds size limit" if protocol_error else "closed connection during 'hello'" - ) - with pytest.raises(expected, match=message): - async with connect(Runtime(f"tcp://127.0.0.1:{port}"), ready_timeout=1.2): - pass - finally: - server.close() - await server.wait_closed() diff --git a/hud/conftest.py b/hud/conftest.py deleted file mode 100644 index ab9d4b705..000000000 --- a/hud/conftest.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Root test fixtures: isolate unit tests from developer settings. - -Without this, any test that exercises ``Taskset.run`` (or other platform -reporting paths) makes real HTTP calls to the platform whenever the developer -has ``HUD_API_KEY`` configured — spamming the platform with fake jobs/traces -and stalling the suite on network retries. CI never catches it because CI has -no API key. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - - -@pytest.fixture(autouse=True) -def _isolate_hud_settings(request: pytest.FixtureRequest) -> None: - """Disable telemetry and the API key for unit tests. - - Tests marked ``integration`` keep the real settings (they require - ``HUD_API_KEY`` and network access by contract). - """ - if request.node.get_closest_marker("integration") is not None: - return - - from hud.settings import settings - - mp = pytest.MonkeyPatch() - request.addfinalizer(mp.undo) - mp.setattr(settings, "telemetry_enabled", False) - mp.setenv("HUD_CLI_ANALYTICS_ENABLED", "0") - mp.setattr(settings, "api_key", None) - mp.setattr(settings, "telemetry_local_dir", None) - home = request.getfixturevalue("tmp_path") - mp.setattr(Path, "home", lambda: home) diff --git a/hud/environment/__init__.py b/hud/environment/__init__.py index baa2f36e7..100832ade 100644 --- a/hud/environment/__init__.py +++ b/hud/environment/__init__.py @@ -91,7 +91,7 @@ def load_environment( raise ValueError(f"multiple Environments in {path}; select one by name") return next(iter(matched.values())) - if path.is_file() or "/" in str(target): + if path.suffix == ".py" or "/" in str(target): raise FileNotFoundError(f"no environment source at {target}") obj = getattr(importlib.import_module(str(target)), name or "env") env = obj if isinstance(obj, Environment) or not callable(obj) else obj(**args or {}) diff --git a/hud/environment/robot/bridge.py b/hud/environment/robot/bridge.py index ae47087e9..0844bfbb3 100644 --- a/hud/environment/robot/bridge.py +++ b/hud/environment/robot/bridge.py @@ -360,8 +360,11 @@ async def _handle_client(self, ws: Any) -> None: async def _tick_loop(self) -> None: """Barrier: wait for live claimed actions, step once, fan out.""" while True: + # asyncio.timeout, not wait_for: on 3.11 wait_for drops a cancel that lands as + # the event is set, and stop() would wait on this loop forever. with contextlib.suppress(TimeoutError): - await asyncio.wait_for(self._action_event.wait(), timeout=self.step_timeout) + async with asyncio.timeout(self.step_timeout): + await self._action_event.wait() self._action_event.clear() claimed = self._registry.claimed() @@ -376,7 +379,8 @@ async def _tick_loop(self) -> None: while any(s.action is None for s in live): self._action_event.clear() try: - await asyncio.wait_for(self._action_event.wait(), timeout=self.step_timeout) + async with asyncio.timeout(self.step_timeout): + await self._action_event.wait() except TimeoutError: # Connected but silent too long — fail it (its agent would # otherwise hang on an observation) and drop out so peers proceed. diff --git a/hud/environment/robot/control.py b/hud/environment/robot/control.py index 7f0cd3dd8..8205be55f 100644 --- a/hud/environment/robot/control.py +++ b/hud/environment/robot/control.py @@ -908,7 +908,8 @@ def _off(self, proprio: NDArray[np.float64], goal: NDArray[np.float64]) -> str: axis = self._axes[name] if axis.kind == "scalar": if self._reads_grip or self._quantity[name] != "grip": - parts.append(f"{name}={float(proprio[axis.index] - goal[axis.index]):.4f}") + offset = float(proprio[axis.index] - goal[axis.index]) + parts.append(f"{name}={_decimal(offset)}") continue assert axis.block is not None key = id(axis.block) @@ -916,7 +917,7 @@ def _off(self, proprio: NDArray[np.float64], goal: NDArray[np.float64]) -> str: continue reported.add(key) angle = angular_distance(axis.block.read(proprio), axis.block.read(goal)) - parts.append(f"orientation={angle:.4f} rad") + parts.append(f"orientation={_decimal(angle)} rad") return ", ".join(parts) @@ -967,7 +968,12 @@ def _offset(gap: _Gap) -> str: def _labeled(pairs: Iterable[tuple[str, Any]]) -> str: - return ", ".join(f"{name}={value:.4f}" for name, value in pairs) + return ", ".join(f"{name}={_decimal(float(value))}" for name, value in pairs) + + +def _decimal(value: float) -> str: + """Four decimals, unsigned when the value rounds to zero, so ``-0.0000`` never shows.""" + return f"{round(value, 4) + 0.0:.4f}" def _png(frame: NDArray[Any]) -> str: diff --git a/hud/environment/server.py b/hud/environment/server.py index a7c1bb6c2..a39c521c0 100644 --- a/hud/environment/server.py +++ b/hud/environment/server.py @@ -409,15 +409,16 @@ async def error_to(msg_id: int | str | None, code: int, message: str) -> None: if not isinstance(task_id, str): await error_to(msg_id, -32602, "tasks.start: 'id' must be a string") continue - args = params.get("args") or {} + args = params.get("args") + if args is None: + args = {} if not isinstance(args, dict): await error_to(msg_id, -32602, "tasks.start: 'args' must be an object") continue - try: - prompt = await self.start(session_id, task_id, args) - except KeyError: + if task_id not in env.tasks: await error_to(msg_id, -32602, f"unknown task: {task_id!r}") continue + prompt = await self.start(session_id, task_id, args) try: await reply_to(msg_id, prompt) except FrameTooLargeError: @@ -580,23 +581,45 @@ async def serve(env: Environment, host: str = "127.0.0.1", port: int = 0) -> Non server = await bind(env, host, port) port_line = f"{PORT_ANNOUNCEMENT}{server.sockets[0].getsockname()[1]}" print(port_line, flush=True) # noqa: T201 - the spawn provider reads this from stdout - async with server: - await server.serve_forever() + # The bound server is already accepting. Not serve_forever(): cancelling it + # waits for open connections to close, and only _shutdown() closes them. + await asyncio.Event().wait() finally: if server is not None: await _shutdown(server) await env.stop() -async def _serve_until_terminated(env: Environment, host: str, port: int) -> None: +async def serve_until_terminated(env: Environment, host: str = "127.0.0.1", port: int = 0) -> None: + """``serve`` as a process's main task: SIGTERM stops it cleanly and it returns. + + SIGTERM (a container stop, the spawn provider's teardown) ends serving the + way cancellation does, so shutdown hooks run and backing daemons don't + orphan. The handler is process-wide, so only entry points that own the + process call this. + """ main_task = asyncio.current_task() assert main_task is not None - # SIGTERM (the spawn provider's teardown) cancels serving so env.stop() - # runs and backing daemons don't orphan. Not available on Windows loops. + loop = asyncio.get_running_loop() + terminated = False + + def terminate() -> None: + nonlocal terminated + terminated = True + main_task.cancel() + + # Signal handlers are not available on Windows loops. with contextlib.suppress(NotImplementedError): - asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, main_task.cancel) - with contextlib.suppress(asyncio.CancelledError): + loop.add_signal_handler(signal.SIGTERM, terminate) + try: await serve(env, host, port) + except asyncio.CancelledError: + if not terminated: + raise + main_task.uncancel() + finally: + with contextlib.suppress(NotImplementedError): + loop.remove_signal_handler(signal.SIGTERM) def main() -> None: @@ -611,7 +634,7 @@ def main() -> None: parser.add_argument("--port", type=int, default=0, help="Port to bind (0 = ephemeral).") args = parser.parse_args() asyncio.run( - _serve_until_terminated(load_environment(args.path, name=args.env), args.host, args.port) + serve_until_terminated(load_environment(args.path, name=args.env), args.host, args.port) ) diff --git a/hud/environment/tests/conftest.py b/hud/environment/tests/conftest.py deleted file mode 100644 index bb52f5b22..000000000 --- a/hud/environment/tests/conftest.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Harnesses for protocol-level environment tests. - -Inline-defined envs have no source file to ``spawn``, so :func:`served` drives -the connect path against a loopback substrate served by this process. This is -a test harness, not an engine placement. -""" - -from __future__ import annotations - -from contextlib import asynccontextmanager -from typing import TYPE_CHECKING - -from hud.clients import connect -from hud.eval import LocalRuntime, Task - -if TYPE_CHECKING: - from collections.abc import AsyncIterator - - from hud.clients import HudClient - from hud.environment import Environment - - -@asynccontextmanager -async def served(env: Environment) -> AsyncIterator[HudClient]: - """Serve *env* on a loopback substrate and yield a connected client.""" - task = Task(env=env.name, id="test") - async with LocalRuntime(env)(task) as runtime, connect(runtime) as client: - yield client diff --git a/hud/environment/tests/test_arguments.py b/hud/environment/tests/test_arguments.py deleted file mode 100644 index c4424ae11..000000000 --- a/hud/environment/tests/test_arguments.py +++ /dev/null @@ -1,85 +0,0 @@ -from __future__ import annotations - -from typing import Literal - -from pydantic import BaseModel - -from hud.environment import ( - DataFileArg, - DataFileRef, - DataFilesArg, - Environment, - GradingArg, - PromptArg, -) -from hud.eval import Run - -from .conftest import served - - -class _Attachment(DataFileRef): - expand: bool = False - - -class _Fixture(DataFileRef): - channel: Literal["mail", "calendar"] - - -class _Criterion(BaseModel): - requirement: str - weight: float = 1.0 - guidance: str | None = None - - -def test_argument_types_publish_editor_hints_and_model_schemas() -> None: - env = Environment("typed-args") - - @env.template() - async def task( - prompt: PromptArg, - attachment: DataFileArg[_Attachment], - fixtures: DataFilesArg[_Fixture], - criteria: GradingArg[_Criterion], - ): - yield prompt - yield 1.0 - - schema = task.manifest_entry()["args"] - properties = schema["properties"] - - assert properties["prompt"]["x-hud-hint"] == "prompt" - assert properties["attachment"]["x-hud-hint"] == "data-file" - assert properties["fixtures"]["x-hud-hint"] == "data-files" - assert properties["criteria"]["x-hud-hint"] == "grading" - assert properties["fixtures"]["items"]["$ref"] == "#/$defs/_Fixture" - assert properties["criteria"]["items"]["$ref"] == "#/$defs/_Criterion" - assert "channel" in schema["$defs"]["_Fixture"]["properties"] - assert "guidance" in schema["$defs"]["_Criterion"]["properties"] - - -async def test_typed_argument_values_reach_task_as_models() -> None: - env = Environment("typed-args") - - @env.template() - async def task( - attachments: DataFilesArg[_Attachment], - criteria: GradingArg[_Criterion], - ): - assert isinstance(attachments[0], _Attachment) - assert isinstance(criteria[0], _Criterion) - yield f"{attachments[0].path}:{criteria[0].requirement}" - yield 1.0 - - async with ( - served(env) as client, - Run( - client, - "task", - { - "attachments": [{"file_id": "file-1", "path": "brief.pdf"}], - "criteria": [{"requirement": "Answer the question", "weight": 2.0}], - }, - ) as run, - ): - assert run.prompt_text == "brief.pdf:Answer the question" - run.trace.content = "done" diff --git a/hud/environment/tests/test_capability_backing.py b/hud/environment/tests/test_capability_backing.py deleted file mode 100644 index 94621a61c..000000000 --- a/hud/environment/tests/test_capability_backing.py +++ /dev/null @@ -1,431 +0,0 @@ -"""Env-run daemons publish capabilities at serve time, never at declaration. - -``env.workspace(root)`` (and, generally, ``env.add_capability(...)`` from an -``@env.initialize`` hook) defers everything — keys, sockets, the directory — -until the env actually serves. The manifest carries the published address, -and ``env.stop()`` runs the matching shutdown hooks. -""" - -from __future__ import annotations - -import asyncio -import contextlib -import os -import shlex -import shutil -import signal -import subprocess -import sys -from typing import TYPE_CHECKING, cast -from urllib.parse import urlsplit - -import pytest - -import hud.environment.workspace as workspace_module -from hud.capabilities import Capability -from hud.environment import Environment - -from .conftest import served - -if TYPE_CHECKING: - from pathlib import Path - - from hud.capabilities.ssh import SSHClient - - -def test_attaching_a_workspace_writes_nothing(tmp_path: Path) -> None: - env = Environment("pure") - env.workspace(tmp_path / "root") - - assert env.capabilities == [] # published at serve time, not declaration - assert not (tmp_path / "root").exists() - - -async def test_serving_publishes_the_workspace_capability( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from hud.settings import settings - - monkeypatch.setattr(settings, "file_tracking_enabled", True) - monkeypatch.setattr(workspace_module, "usable_bwrap", lambda: None) - env = Environment("ws-env") - env.workspace(tmp_path / "root") - - async with served(env) as client: - assert client.manifest is not None - shell = next(cap for cap in client.manifest.bindings if cap.name == "shell") - assert shell.params["isolation"] == "none" - cap = client.binding("shell") - assert cap.protocol == "ssh/2" - assert cap.url.startswith("ssh://") - assert cap.params["host_pubkey"].startswith("ssh-ed25519") - filetracking = client.binding("filetracking") - assert filetracking.protocol == "filetracking/1" - assert filetracking.params == { - "root": (tmp_path / "root").resolve().as_posix(), - "setup_diff": True, - } - # Key material lives outside the served root (the agent's surface). - assert not (tmp_path / "root" / ".hud").exists() - - -async def test_serving_reports_granted_bwrap_isolation( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr( - workspace_module, - "usable_bwrap", - lambda: workspace_module.Bubblewrap("/usr/bin/bwrap"), - ) - env = Environment("ws-env") - env.workspace(tmp_path / "root") - - async with served(env) as client: - assert client.manifest is not None - shell = next(cap for cap in client.manifest.bindings if cap.name == "shell") - assert shell.params["isolation"] == "bwrap" - - -async def test_workspace_file_tracking_can_be_opted_out( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from hud.settings import settings - - monkeypatch.setattr(settings, "file_tracking_enabled", True) - env = Environment("ws-env") - env.workspace(tmp_path / "root", track_files=False) - - async with served(env) as client: - assert client.binding("shell").protocol == "ssh/2" - with pytest.raises(KeyError): - client.binding("filetracking") - - -async def test_reconnecting_reuses_the_same_workspace(tmp_path: Path) -> None: - from hud.clients import connect - from hud.eval import LocalRuntime, Task - - env = Environment("ws-env") - env.workspace(tmp_path / "root") - - # Client-side urls are per-connection (forwarded); the daemon's identity - # is its host key, which only stays stable if the workspace is reused. - async with LocalRuntime(env)(Task(env=env.name, id="test")) as runtime: - async with connect(runtime) as client: - first = client.binding("shell").params["host_pubkey"] - async with connect(runtime) as client: - assert client.binding("shell").params["host_pubkey"] == first - - -def _pid_status(pid: int) -> str | None: - result = subprocess.run( - ["ps", "-o", "stat=", "-p", str(pid)], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ) - return result.stdout.strip() or None - - -def _pid_is_running(pid: int) -> bool: - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - status = _pid_status(pid) - if status is None: - return False - return not status.startswith("Z") - - -async def _wait_for_pid_inactive(pid: int, max_wait: float = 2.0) -> bool: - loop = asyncio.get_running_loop() - deadline = loop.time() + max_wait - while loop.time() < deadline: - if not _pid_is_running(pid): - return True - await asyncio.sleep(0.05) - return not _pid_is_running(pid) - - -@pytest.mark.skipif(sys.platform == "win32", reason="POSIX process-group regression") -async def test_workspace_command_teardown_kills_background_children(tmp_path: Path) -> None: - env = Environment("ws-env") - env.workspace(tmp_path / "root", track_files=False) - pid_file = tmp_path / "root" / "child.pid" - pid: int | None = None - - try: - async with served(env) as client: - ssh = cast("SSHClient", await client.open("shell")) - await ssh.conn.run( - "sleep 120 >/dev/null 2>&1 < /dev/null & echo $! > child.pid", - check=True, - ) - pid = int(pid_file.read_text()) - assert await _wait_for_pid_inactive(pid) - finally: - if pid is not None and _pid_is_running(pid): - with contextlib.suppress(ProcessLookupError): - os.kill(pid, signal.SIGKILL) - - -@pytest.mark.skipif(sys.platform == "win32", reason="POSIX process-group regression") -async def test_workspace_session_close_kills_running_children(tmp_path: Path) -> None: - env = Environment("ws-env") - env.workspace(tmp_path / "root", track_files=False) - pid: int | None = None - - try: - async with served(env) as client: - ssh = cast("SSHClient", await client.open("shell")) - remote = await ssh.conn.create_process( - "trap '' TERM; sleep 120 & echo $! > child.pid; wait" - ) - try: - result = await asyncio.wait_for( - ssh.conn.run( - "while [ ! -s child.pid ]; do sleep 0.01; done; cat child.pid", - check=True, - ), - 5.0, - ) - assert isinstance(result.stdout, str) - pid = int(result.stdout) - assert _pid_is_running(pid) - finally: - remote.close() - await asyncio.wait_for(remote.wait_closed(), 5.0) - - assert await _wait_for_pid_inactive(pid, max_wait=5.0) - finally: - if pid is not None and _pid_is_running(pid): - with contextlib.suppress(ProcessLookupError): - os.kill(pid, signal.SIGKILL) - - -@pytest.mark.skipif(sys.platform == "win32", reason="POSIX process-group regression") -async def test_workspace_channel_close_during_teardown_keeps_connection_usable( - tmp_path: Path, -) -> None: - env = Environment("ws-env") - env.workspace(tmp_path / "root", track_files=False) - pid: int | None = None - - try: - async with served(env) as client: - ssh = cast("SSHClient", await client.open("shell")) - # The pid file doubles as the child's readiness signal: it is - # written after the TERM trap is installed, and teardown's TERM - # only fires once the foreground command exits. Exiting before - # the child is ready would kill it while still untrapped. - remote = await ssh.conn.create_process( - "sh -c 'trap \"echo started > teardown-started; " - 'while :; do sleep 1; done" TERM; ' - "echo $$ > close-race-child.pid; while :; do sleep 1; done' & " - "while [ ! -s close-race-child.pid ]; do sleep 0.01; done; " - "printf output" - ) - try: - result = await asyncio.wait_for( - ssh.conn.run( - "while [ ! -s teardown-started ]; do sleep 0.01; done; " - "cat close-race-child.pid", - check=True, - ), - 5.0, - ) - assert isinstance(result.stdout, str) - pid = int(result.stdout) - assert _pid_is_running(pid) - - remote.close() - await asyncio.wait_for(remote.wait_closed(), 5.0) - assert await _wait_for_pid_inactive(pid, max_wait=5.0) - await ssh.conn.run("true", check=True) - finally: - remote.close() - await asyncio.wait_for(remote.wait_closed(), 5.0) - finally: - if pid is not None and _pid_is_running(pid): - with contextlib.suppress(ProcessLookupError): - os.kill(pid, signal.SIGKILL) - - -@pytest.mark.skipif(sys.platform == "win32", reason="POSIX process-group regression") -async def test_workspace_client_timeout_keeps_connection_usable(tmp_path: Path) -> None: - env = Environment("ws-env") - env.workspace(tmp_path / "root", track_files=False) - - async with served(env) as client: - ssh = cast("SSHClient", await client.open("shell")) - started = asyncio.get_running_loop().time() - - with pytest.raises(TimeoutError): - await ssh.run("sleep 120", timeout=0.1) - - assert asyncio.get_running_loop().time() - started < 2.0 - completed = await ssh.run("printf healthy", check=True, timeout=5.0) - assert completed.stdout == "healthy" - - -@pytest.mark.skipif(sys.platform == "win32", reason="POSIX process-group regression") -async def test_workspace_shell_replaces_invalid_utf8_without_losing_transport( - tmp_path: Path, -) -> None: - env = Environment("ws-env") - env.workspace(tmp_path / "root", track_files=False) - - async with served(env) as client: - ssh = cast("SSHClient", await client.open("shell")) - connection = ssh.conn - - completed = await ssh.run("printf '\\226\\214 split-utf8'", check=True) - - assert completed.stdout == "�� split-utf8" - assert completed.returncode == 0 - assert ssh.conn is connection - assert not connection.is_closed() - - -@pytest.mark.skipif(sys.platform == "win32", reason="requires POSIX setsid") -async def test_workspace_command_bounds_detached_output_drain(tmp_path: Path) -> None: - env = Environment("ws-env") - env.workspace(tmp_path / "root", track_files=False) - pid: int | None = None - setsid = shutil.which("setsid") - if setsid is not None: - detached_command = ( - f"{shlex.quote(setsid)} sh -c " - "'echo $$ > detached.pid; printf ready; kill -KILL $PPID; sleep 120'" - ) - else: - code = ( - "import os,signal,time;" - "from pathlib import Path;" - "parent=os.getppid();" - "os.setsid();" - "Path('detached.pid').write_text(str(os.getpid()));" - "os.write(1,b'ready');" - "os.kill(parent,signal.SIGKILL);" - "time.sleep(120)" - ) - detached_command = f"{shlex.quote(sys.executable)} -c {shlex.quote(code)}" - - try: - async with served(env) as client: - ssh = cast("SSHClient", await client.open("shell")) - remote = await ssh.conn.create_process(f"{detached_command} & wait") - try: - result = await asyncio.wait_for( - ssh.conn.run( - "while [ ! -s detached.pid ]; do sleep 0.01; done; cat detached.pid", - check=True, - ), - 5.0, - ) - assert isinstance(result.stdout, str) - pid = int(result.stdout) - assert _pid_is_running(pid) - - completed = await remote.wait(timeout=5.0) - assert completed.stdout == "ready" - assert _pid_is_running(pid) - finally: - remote.close() - await asyncio.wait_for(remote.wait_closed(), 5.0) - finally: - if pid is not None and _pid_is_running(pid): - with contextlib.suppress(ProcessLookupError): - os.killpg(pid, signal.SIGKILL) - - -async def test_stop_tears_down_the_workspace(tmp_path: Path) -> None: - import asyncio - from urllib.parse import urlsplit - - env = Environment("ws-env") - env.workspace(tmp_path / "root") - - async with served(env): - # The substrate-local address (the manifest carries a forwarded one). - backing_port = urlsplit(env.capability("shell").url).port - assert backing_port is not None - - with pytest.raises(OSError): - _, writer = await asyncio.open_connection("127.0.0.1", backing_port) - writer.close() - - -async def test_restarting_replaces_the_published_address_without_duplicates( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from hud.settings import settings - - monkeypatch.setattr(settings, "file_tracking_enabled", True) - env = Environment("ws-env") - env.workspace(tmp_path / "root") - - async with served(env): - pass - async with served(env): - assert [c.name for c in env.capabilities] == ["shell", "filetracking"] - - -async def test_any_initialize_hook_can_publish_a_capability() -> None: - """Publication is protocol-agnostic: no SDK type per daemon kind.""" - import asyncio - - server = await asyncio.start_server(lambda r, w: w.close(), "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - torn_down = False - - env = Environment("browser-env") - - @env.initialize - async def _up() -> None: - env.add_capability(Capability.cdp(name="browser", url=f"ws://127.0.0.1:{port}")) - - @env.shutdown - async def _down() -> None: - nonlocal torn_down - torn_down = True - server.close() - - assert env.capabilities == [] - async with served(env) as client: - cap = client.binding("browser") - assert cap.protocol == "cdp/1.3" - assert cap.url.startswith("ws://127.0.0.1:") - assert torn_down # env.stop() ran the shutdown hook - - -async def test_manifest_keeps_environment_addresses_and_client_bindings_are_forwarded() -> None: - local = Capability.cdp(name="browser", url="ws://127.0.0.1:9222") - remote = Capability.ssh(name="box", url="ssh://box.example.com:22", host_pubkey="ssh-ed25519 x") - env = Environment("mixed-env", capabilities=[local, remote]) - - async with served(env) as client: - assert client.manifest is not None - assert client.manifest.bindings == [local, remote] - for declared in (local, remote): - forwarded = client.binding(declared.name) - assert forwarded.url != declared.url - assert urlsplit(forwarded.url).hostname == "127.0.0.1" - assert (forwarded.name, forwarded.protocol) == ( - declared.name, - declared.protocol, - ) - - -def test_a_capability_without_a_url_is_rejected() -> None: - with pytest.raises(ValueError, match="initialize hook"): - Environment("bad", capabilities=[Capability(name="b", protocol="cdp/1.3", url="")]) - - -def test_non_capability_entries_are_rejected() -> None: - with pytest.raises(TypeError, match="expected Capability"): - Environment("bad", capabilities=cast("list[Capability]", [object()])) diff --git a/hud/environment/tests/test_file_tracker.py b/hud/environment/tests/test_file_tracker.py deleted file mode 100644 index 6da1c48bf..000000000 --- a/hud/environment/tests/test_file_tracker.py +++ /dev/null @@ -1,343 +0,0 @@ -"""FileTracker: snapshot diffing, excludes, gitignore, and the secrets deny-list.""" - -from __future__ import annotations - -import base64 -from typing import TYPE_CHECKING - -from hud.environment.file_tracker import FileTracker - -if TYPE_CHECKING: - from pathlib import Path - - import pytest - - -def test_modified_file_produces_a_unified_diff(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("line1\nline2\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "a.txt").write_text("line1\nCHANGED\n") - diff = tracker.take_snapshot() - - assert diff.files_changed == 1 - patch = diff.patches[0] - assert patch.rel_path == "a.txt" - assert patch.status == "modified" - assert "CHANGED" in patch.patch - - -def test_added_and_deleted_files(tmp_path: Path) -> None: - (tmp_path / "keep.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "new.txt").write_text("hello\n") - (tmp_path / "keep.txt").unlink() - diff = tracker.take_snapshot() - - by_path = {p.rel_path: p for p in diff.patches} - assert by_path["new.txt"].status == "added" - assert by_path["new.txt"].size_before == 0 - assert by_path["keep.txt"].status == "deleted" - - -def test_no_changes_yields_empty_diff(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - diff = tracker.take_snapshot() - - assert diff.files_changed == 0 - assert diff.patches == [] - - -def test_exclude_patterns_skip_build_output(tmp_path: Path) -> None: - (tmp_path / "node_modules").mkdir() - (tmp_path / "node_modules" / "dep.js").write_text("module.exports = 1;\n") - go_cache = tmp_path / ".tmp" / "go-build123" - go_cache.mkdir(parents=True) - (go_cache / "_pkg_.a").write_text("compiled\n") - (tmp_path / "go-build.go").write_text("package main\n") - (tmp_path / "src.py").write_text("x = 1\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest_paths = {entry["path"] for entry in tracker.current_manifest()} - assert "go-build.go" in manifest_paths - assert "src.py" in manifest_paths - assert not any(p.startswith("node_modules/") for p in manifest_paths) - assert not any(p.startswith(".tmp/") for p in manifest_paths) - - -def test_gitignore_is_honored(tmp_path: Path) -> None: - (tmp_path / ".gitignore").write_text("ignored.txt\n") - (tmp_path / "ignored.txt").write_text("secret-ish\n") - (tmp_path / "tracked.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest_paths = {entry["path"] for entry in tracker.current_manifest()} - assert "tracked.txt" in manifest_paths - assert "ignored.txt" not in manifest_paths - - -def test_vcs_metadata_dirs_are_excluded(tmp_path: Path) -> None: - git_dir = tmp_path / ".git" - git_dir.mkdir() - (git_dir / "config").write_text('[remote "origin"]\nurl = https://token@example.com/repo.git\n') - (tmp_path / "tracked.txt").write_text("x\n") - - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest_paths = {entry["path"] for entry in tracker.current_manifest()} - assert "tracked.txt" in manifest_paths - assert not any(path == ".git" or path.startswith(".git/") for path in manifest_paths) - - (git_dir / "config").write_text( - '[remote "origin"]\nurl = https://new-token@example.com/repo.git\n' - ) - diff = tracker.take_snapshot() - - assert diff.files_changed == 0 - assert diff.patches == [] - - -def test_secret_files_are_tracked_but_content_is_never_emitted(tmp_path: Path) -> None: - (tmp_path / ".env").write_text("API_KEY=supersecretvalue\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - assert tracker.current_manifest() == [] - - (tmp_path / ".env").write_text("API_KEY=supersecretvalue\nDB_PASSWORD=hunter2\n") - diff = tracker.take_snapshot() - - assert diff.files_changed == 1 - patch = diff.patches[0] - assert patch.rel_path == ".env" - assert patch.status == "modified" - # The change is detected, but the content is redacted — never in the patch. - assert "redacted" in patch.patch.lower() - assert "supersecretvalue" not in patch.patch - assert "hunter2" not in patch.patch - - -def test_vcs_config_files_are_redacted(tmp_path: Path) -> None: - (tmp_path / ".gitmodules").write_text( - '[submodule "private"]\nurl = https://token@example.com/private.git\n' - ) - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / ".gitmodules").write_text( - '[submodule "private"]\nurl = https://new-token@example.com/private.git\n' - ) - diff = tracker.take_snapshot() - - assert diff.files_changed == 1 - patch = diff.patches[0] - assert patch.rel_path == ".gitmodules" - assert "redacted" in patch.patch.lower() - assert "token@example.com" not in patch.patch - assert "new-token@example.com" not in patch.patch - - -def test_capture_changed_deliverables_since_setup_snapshot(tmp_path: Path) -> None: - (tmp_path / "setup.xlsx").write_bytes(b"setup workbook") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - tracker.take_setup_snapshot() - - (tmp_path / "analysis.py").write_text("print('tracked by diff')\n") - (tmp_path / "notes.txt").write_text("tracked by diff\n") - (tmp_path / "deliverable.xlsx").write_bytes(b"workbook bytes") - (tmp_path / "report.html").write_text("report\n") - - payload = tracker.flush_changes()["capture"] - - assert payload["files_changed"] == 4 - assert payload["files_eligible"] == 2 - assert payload["files_captured"] == 2 - assert payload["files_skipped"] == 0 - files = {file["path"]: file for file in payload["files"]} - - xlsx = files["deliverable.xlsx"] - assert xlsx["status"] == "added" - assert ( - xlsx["content_type"] == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ) - assert len(xlsx["content_hash"]) == 64 - assert base64.b64decode(xlsx["file"]["data"]) == b"workbook bytes" - assert xlsx["file"]["type"] == "file" - assert xlsx["file"]["media_type"] == xlsx["content_type"] - - html = files["report.html"] - assert html["status"] == "added" - assert html["content_type"] == "text/html" - assert base64.b64decode(html["file"]["data"]) == b"report\n" - assert html["file"]["media_type"] == "text/html" - - -def test_capture_skips_secret_shaped_files(tmp_path: Path) -> None: - (tmp_path / ".env.xlsx").write_bytes(b"API_KEY=before") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / ".env.xlsx").write_bytes(b"API_KEY=after") - payload = tracker.flush_changes()["capture"] - - assert payload["files_changed"] == 1 - assert payload["files_eligible"] == 0 - assert payload["files_captured"] == 0 - assert payload["files"] == [] - - -def test_capture_marks_cap_skips_as_truncated( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(FileTracker, "_MAX_CAPTURE_FILE_BYTES", 4) - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "large.pdf").write_bytes(b"12345") - payload = tracker.flush_changes()["capture"] - - assert payload["files_eligible"] == 1 - assert payload["files_captured"] == 0 - assert payload["files_skipped"] == 1 - assert payload["truncated"] is True - - -def test_per_file_diff_cap_emits_a_placeholder( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(FileTracker, "_MAX_DIFF_FILE_BYTES", 4) - (tmp_path / "big.txt").write_text("aaaaaaaa\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "big.txt").write_text("bbbbbbbb\n") - diff = tracker.take_snapshot() - - assert diff.files_changed == 1 - assert "too large to diff" in diff.patches[0].patch - - -def test_budget_skipped_files_stay_pending_until_emitted( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - (tmp_path / "a.txt").write_text("a1\n") - (tmp_path / "b.txt").write_text("b1\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "a.txt").write_text("a2\n") - (tmp_path / "b.txt").write_text("b2\n") - - # Cap so small no patch fits: both changes are skipped this poll. - monkeypatch.setattr(FileTracker, "_MAX_DIFF_BYTES", 1) - first = tracker.take_snapshot() - assert first.truncated - assert first.patches == [] - assert set(first.skipped_paths) == {"a.txt", "b.txt"} - - # Next poll with headroom: the skipped changes must not be lost — the - # baseline kept them pending, so they re-diff now. - monkeypatch.setattr(FileTracker, "_MAX_DIFF_BYTES", 50 * 1024 * 1024) - second = tracker.take_snapshot() - by_path = {p.rel_path: p for p in second.patches} - assert set(by_path) == {"a.txt", "b.txt"} - assert "a2" in by_path["a.txt"].patch - - -def test_nested_gitignore_is_not_honored_or_leaked(tmp_path: Path) -> None: - # Root has no .gitignore; a nested package does. With root-only honoring the - # nested rule must neither take effect locally nor leak tree-wide (the old - # basename match would have excluded "data.txt" everywhere). - pkg = tmp_path / "pkg" - pkg.mkdir() - (pkg / ".gitignore").write_text("data.txt\n") - (pkg / "data.txt").write_text("local\n") - (tmp_path / "data.txt").write_text("root\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest_paths = {entry["path"] for entry in tracker.current_manifest()} - assert "data.txt" in manifest_paths - assert "pkg/data.txt" in manifest_paths - - -def test_manifest_carries_paths_and_hashes(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest = tracker.current_manifest() - - assert len(manifest) == 1 - entry = manifest[0] - assert entry["path"] == "a.txt" - assert entry["size"] == (tmp_path / "a.txt").stat().st_size - assert len(entry["content_hash"]) == 64 # sha256 hex - - -def test_setup_snapshot_returns_changes_and_resets_agent_baseline(tmp_path: Path) -> None: - source = tmp_path / "source.py" - source.write_text("before\n") - tracker = FileTracker(tmp_path) - initial = tracker.take_baseline() - - source.write_text("after\n") - (tmp_path / "generated.txt").write_text("setup\n") - setup = tracker.take_setup_snapshot() - - patches = {patch.rel_path: patch for patch in setup.patches} - assert set(patches) == {"generated.txt", "source.py"} - assert patches["source.py"].content_hash_before == initial.files["source.py"].content_hash - assert patches["source.py"].content_hash_after == tracker.current_manifest()[1]["content_hash"] - assert tracker.take_snapshot().files_changed == 0 - - -def test_setup_snapshot_keeps_complete_agent_baseline_when_diff_is_capped( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - tracker = FileTracker(tmp_path) - tracker.take_baseline() - (tmp_path / "generated.txt").write_text("setup\n") - monkeypatch.setattr(FileTracker, "_MAX_DIFF_BYTES", 0) - - setup = tracker.take_setup_snapshot() - - assert setup.truncated is True - assert setup.patches == [] - assert tracker.take_snapshot().files_changed == 0 - - -def test_to_dict_shape_matches_wire_contract(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("1\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - (tmp_path / "a.txt").write_text("2\n") - - payload = tracker.take_snapshot().to_dict() - - assert set(payload) >= { - "snapshot_timestamp", - "scan_duration_ms", - "files_scanned", - "files_changed", - "patches", - } - assert set(payload["patches"][0]) == { - "path", - "status", - "patch", - "size_before", - "size_after", - "content_hash_before", - "content_hash_after", - } diff --git a/hud/environment/tests/test_file_tracking.py b/hud/environment/tests/test_file_tracking.py deleted file mode 100644 index 181d20011..000000000 --- a/hud/environment/tests/test_file_tracking.py +++ /dev/null @@ -1,82 +0,0 @@ -"""filetracking/1 wire roundtrip: serve a FileTracker, drive the wire directly.""" - -from __future__ import annotations - -import asyncio -import base64 -from typing import TYPE_CHECKING, Any - -from hud.environment.file_tracker import FileTracker, serve_file_tracking -from hud.environment.utils import read_frame, send_frame - -if TYPE_CHECKING: - from pathlib import Path - - -async def _call( - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, - method: str, -) -> dict[str, Any]: - await send_frame( - writer, - {"jsonrpc": "2.0", "id": 1, "method": method, "params": {}}, - ) - reply = await read_frame(reader) - assert reply is not None - assert "error" not in reply - result = reply.get("result") - assert isinstance(result, dict) - return result - - -async def test_diff_snapshot_setup_roundtrip(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - server = await serve_file_tracking(tracker) - host, port = server.sockets[0].getsockname()[:2] - reader, writer = await asyncio.open_connection(host, port) - try: - # Nothing changed yet. - assert (await _call(reader, writer, "diff"))["files_changed"] == 0 - - # An edit shows up as a diff on the next pull. - (tmp_path / "a.txt").write_text("x\ny\n") - diff = await _call(reader, writer, "diff") - assert diff["files_changed"] == 1 - assert diff["patches"][0]["path"] == "a.txt" - - # diff() advanced the baseline, so a re-pull is empty. - assert (await _call(reader, writer, "diff"))["files_changed"] == 0 - - # snapshot() returns the full manifest. - snapshot = await _call(reader, writer, "snapshot") - assert any(entry["path"] == "a.txt" for entry in snapshot["files"]) - - # advance() remains available to clients that predate setup diffs. - (tmp_path / "legacy.txt").write_text("legacy\n") - assert await _call(reader, writer, "advance") == {"advanced": True} - assert (await _call(reader, writer, "diff"))["files_changed"] == 0 - - # setup() emits scenario changes and starts a clean agent-edit layer. - (tmp_path / "b.txt").write_text("z\n") - setup = await _call(reader, writer, "setup") - assert setup["files_changed"] == 1 - assert setup["patches"][0]["path"] == "b.txt" - assert (await _call(reader, writer, "diff"))["files_changed"] == 0 - - # flush() returns the trailing diff plus bounded changed deliverables. - (tmp_path / "report.xlsx").write_bytes(b"spreadsheet") - flush = await _call(reader, writer, "flush") - assert flush["diff"]["files_changed"] == 1 - capture = flush["capture"] - assert capture["files_captured"] == 1 - assert capture["files"][0]["path"] == "report.xlsx" - assert base64.b64decode(capture["files"][0]["file"]["data"]) == b"spreadsheet" - finally: - writer.close() - await writer.wait_closed() - server.close() - await server.wait_closed() diff --git a/hud/environment/tests/test_loader.py b/hud/environment/tests/test_loader.py deleted file mode 100644 index f3ec4b22b..000000000 --- a/hud/environment/tests/test_loader.py +++ /dev/null @@ -1,309 +0,0 @@ -"""``load_environment``: resolve env references — source paths, modules, factories.""" - -from __future__ import annotations - -import importlib -import json -import sys -from typing import TYPE_CHECKING - -import pytest - -from hud.environment import load_environment - -if TYPE_CHECKING: - from collections.abc import Iterator - from pathlib import Path - - -def test_load_environment_selects_by_attr_or_env_name(tmp_path) -> None: - module = tmp_path / "envs.py" - module.write_text( - """ -from hud import Environment - -first = Environment("env-one") -second = Environment("env-two") -""".strip(), - encoding="utf-8", - ) - - assert load_environment(module, name="first").name == "env-one" - assert load_environment(module, name="env-two").name == "env-two" - with pytest.raises(ValueError, match="multiple Environments"): - load_environment(module) - with pytest.raises(ValueError, match="no Environment named 'missing'"): - load_environment(module, name="missing") - - single = tmp_path / "single.py" - single.write_text("from hud import Environment\nenv = Environment('only')\n", encoding="utf-8") - assert load_environment(single).name == "only" - - -@pytest.fixture -def factory_module(request): - """An importable module exposing an env and a factory, cleaned up after.""" - import sys - from types import ModuleType - - from hud.environment import Environment - - name = f"_loader_target_{request.node.name}" - mod = ModuleType(name) - setattr(mod, "env", Environment("declared")) - setattr(mod, "make_env", lambda name="built": Environment(name)) - sys.modules[name] = mod - yield name - del sys.modules[name] - - -def test_module_factory_is_called_with_args(factory_module) -> None: - env = load_environment(factory_module, name="make_env", args={"name": "from-factory"}) - - assert env.name == "from-factory" - - -def test_module_env_attribute_is_returned_not_called(factory_module) -> None: - import sys - - from hud.environment import Environment - - class CallableEnvironment(Environment): - def __call__(self) -> None: - raise AssertionError("Environment instance was called as a factory") - - setattr(sys.modules[factory_module], "env", CallableEnvironment("callable")) - - assert load_environment(factory_module).name == "callable" - - -def test_module_factory_returning_non_environment_raises(factory_module) -> None: - import sys - - setattr(sys.modules[factory_module], "make_env", lambda: object()) - - with pytest.raises(ValueError, match="not an Environment"): - load_environment(factory_module, name="make_env") - - -def test_unresolvable_references_raise() -> None: - with pytest.raises(ModuleNotFoundError): - load_environment("no.such.module") - with pytest.raises(FileNotFoundError, match="no environment source"): - load_environment("missing/env.py") - with pytest.raises(ValueError, match="args= applies to factory targets"): - load_environment(__file__, args={"a": "b"}) - - -def test_package_dir_does_not_shadow_factory_target(tmp_path, monkeypatch) -> None: - # `mypkg:make_env` with a plain mypkg/ package in cwd is a module - # reference; source scanning is only for env-declaring source trees. - pkg = tmp_path / "shadowpkg" - pkg.mkdir() - (pkg / "__init__.py").write_text( - "from hud.environment import Environment\n\n" - "def make_env(name='shadowed'):\n return Environment(name)\n", - encoding="utf-8", - ) - monkeypatch.chdir(tmp_path) - monkeypatch.syspath_prepend(str(tmp_path)) - - env = load_environment("shadowpkg", name="make_env") - - assert env.name == "shadowed" - - -def test_named_attribute_resolves_a_package_that_also_has_env_py(tmp_path, monkeypatch) -> None: - # `pkg:make_env` addresses an attribute, so an env.py sitting inside the - # package must not capture it into a source scan. - pkg = tmp_path / "bothpkg" - pkg.mkdir() - (pkg / "__init__.py").write_text( - "from hud.environment import Environment\n\n" - "def make_env(name='from-factory'):\n return Environment(name)\n", - encoding="utf-8", - ) - (pkg / "env.py").write_text( - "from hud.environment import Environment\n\nenv = Environment('from-source')\n", - encoding="utf-8", - ) - monkeypatch.chdir(tmp_path) - monkeypatch.syspath_prepend(str(tmp_path)) - - assert load_environment("bothpkg", name="make_env").name == "from-factory" - # ...while a bare reference still scans the source tree. - assert load_environment("bothpkg").name == "from-source" - - -def test_every_reference_form_resolves(tmp_path, monkeypatch) -> None: - """The full matrix, in one place: the shapes that competed for the same - spelling are what made this resolution subtle.""" - (tmp_path / "env.py").write_text( - "from hud.environment import Environment\n\nenv = Environment('from-env-py')\n", - encoding="utf-8", - ) - envs = tmp_path / "envs" # a plain directory: an importable namespace package - envs.mkdir() - (envs / "one.py").write_text( - "from hud.environment import Environment\n\nfoo = Environment('from-tree')\n", - encoding="utf-8", - ) - pkg = tmp_path / "pkg" # a real package exposing a factory - pkg.mkdir() - (pkg / "__init__.py").write_text( - "from hud.environment import Environment\n\n" - "def make_env(name='from-factory'):\n return Environment(name)\n", - encoding="utf-8", - ) - (pkg / "env.py").write_text( - "from hud.environment import Environment\n\nenv = Environment('from-pkg-source')\n", - encoding="utf-8", - ) - monkeypatch.chdir(tmp_path) - monkeypatch.syspath_prepend(str(tmp_path)) - - # a source file, however it is spelled — including the common serve form - assert load_environment("env").name == "from-env-py" - assert load_environment("env", name="env").name == "from-env-py" - assert load_environment("env.py").name == "from-env-py" - assert load_environment(tmp_path / "env.py").name == "from-env-py" - - # a source tree, with a name selecting inside it - assert load_environment("envs", name="foo").name == "from-tree" - - # a package attribute: the factory wins over the env.py beside it - assert load_environment("pkg", name="make_env").name == "from-factory" - assert load_environment("pkg", name="make_env", args={"name": "x"}).name == "x" - # ...while a bare reference to the same package still scans its source - assert load_environment("pkg").name == "from-pkg-source" - - -def test_environment_reexports_are_not_ambiguous(tmp_path) -> None: - source = tmp_path / "env.py" - source.write_text('from hud import Environment\nenv = Environment("shared")\nalias = env\n') - assert load_environment(source, name="shared").name == "shared" - assert load_environment(source).name == "shared" - - -def test_distinct_environments_with_the_same_name_are_ambiguous(tmp_path) -> None: - source = tmp_path / "env.py" - source.write_text( - 'from hud import Environment\none = Environment("shared")\ntwo = Environment("shared")\n' - ) - with pytest.raises(ValueError, match="multiple Environments"): - load_environment(source, name="shared") - - -def test_source_supports_package_relative_imports(tmp_path) -> None: - package = tmp_path / "relative_env_package" - package.mkdir() - (package / "__init__.py").write_text("") - (package / "world.py").write_text( - 'from hud import Environment\nenv = Environment("relative")\n' - ) - (package / "env.py").write_text("from .world import env\n") - assert load_environment(package / "env.py").name == "relative" - - -def test_directory_imports_each_module_once(tmp_path) -> None: - (tmp_path / "scan_core.py").write_text( - 'from hud import Environment\nenv = Environment("scanned")\n' - ) - (tmp_path / "scan_templates.py").write_text( - "from scan_core import env\n" - '@env.template(id="solve")\nasync def solve():\n yield "ok"\n yield 1.0\n' - ) - (tmp_path / "assembly.py").write_text( - "from scan_core import env\nfrom scan_templates import solve\n" - ) - env = load_environment(tmp_path, name="scanned") - assert set(env.tasks) == {"solve"} - assert load_environment(tmp_path, name="scanned") is env - - -@pytest.mark.parametrize("export", ["task", "list", "tuple", "taskset"]) -def test_source_resolves_environments_from_exported_tasks(tmp_path, request, export) -> None: - name = f"bound_source_{export}" - (tmp_path / f"{name}.py").write_text( - 'from hud import Environment\nenv = Environment("bound")\n' - '@env.template(id="solve")\nasync def solve():\n yield "ok"\n yield 1.0\n' - "def make_task():\n return solve()\n" - ) - request.addfinalizer(lambda: sys.modules.pop(name, None)) - expression = { - "task": "make_task()", - "list": "[make_task()]", - "tuple": "(make_task(),)", - "taskset": 'Taskset("rows", [make_task()])', - }[export] - source = tmp_path / "tasks.py" - source.write_text( - f"from {name} import make_task\nfrom hud.eval import Taskset\nrows = {expression}\n" - ) - assert set(load_environment(source, name="bound").tasks) == {"solve"} - - -@pytest.mark.parametrize("source", ["json.py", "."]) -def test_source_does_not_replace_an_imported_module(tmp_path, source) -> None: - (tmp_path / "json.py").write_text( - "import json\nfrom hud import Environment\nenv = Environment(json.loads('\"local\"'))\n" - ) - assert load_environment(tmp_path / source).name == "local" - assert importlib.import_module("json") is json - - -def test_package_init_can_reexport_its_environment(tmp_path) -> None: - package = tmp_path / "reexported_env_package" - package.mkdir() - (package / "__init__.py").write_text("from .env import env\n") - (package / "core.py").write_text( - 'from hud import Environment\nenv = Environment("reexported")\n' - ) - (package / "env.py").write_text( - 'from .core import env\n@env.template(id="solve")\n' - 'async def solve():\n yield "ok"\n yield 1.0\n' - ) - env = load_environment(package / "env.py") - assert env.name == "reexported" - assert load_environment(package, name="reexported") is env - - -@pytest.fixture -def package_sources(tmp_path: Path) -> Iterator[tuple[Path, Path]]: - packages = tuple(tmp_path / label / "source_root_package" for label in ("first", "second")) - for package in packages: - package.mkdir(parents=True) - (package / "__init__.py").write_text("from .env import env\n") - (package / "core.py").write_text( - f"from hud import Environment\nenv = Environment({package.parent.name!r})\n" - ) - (package / "env.py").write_text("from .core import env\n") - try: - yield packages[0], packages[1] - finally: - for name in list(sys.modules): - if name == "source_root_package" or name.startswith("source_root_package."): - del sys.modules[name] - - -@pytest.mark.parametrize("source", ["env.py", "__init__.py", "."]) -def test_package_sources_reject_a_cached_namespace_from_another_root( - package_sources: tuple[Path, Path], source: str -) -> None: - first, second = (package / source for package in package_sources) - assert load_environment(first).name == "first" - - with pytest.raises(ValueError, match="already imported from a different source root"): - load_environment(second) - - assert load_environment(first).name == "first" - - -def test_package_source_takes_precedence_over_other_import_roots( - package_sources: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch -) -> None: - first, second = package_sources - monkeypatch.syspath_prepend(str(second.parent)) - monkeypatch.syspath_prepend(str(first.parent)) - - assert load_environment(second / "env.py").name == "second" diff --git a/hud/environment/tests/test_manifest.py b/hud/environment/tests/test_manifest.py deleted file mode 100644 index 8b7c9e2ba..000000000 --- a/hud/environment/tests/test_manifest.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Manifest entries: the task contract published over ``tasks.list``. - -``args`` describes the task function's parameters (what stored task args -must satisfy — validated platform-side at sync time); ``input``/``returns`` -are the agent's declared I/O types. -""" - -from __future__ import annotations - -from typing import Literal - -from pydantic import BaseModel - -from hud.environment import Environment - - -class _Point(BaseModel): - x: int - y: int - - -class _Payload(BaseModel): - text: str - - -_Mode = Literal["easy", "hard"] -_Retries = int | None - - -def test_args_schema_captures_params_defaults_and_required() -> None: - env = Environment("manifests") - - @env.template() - async def fix_bug(difficulty: int, suite: str = "coding"): - yield "go" - yield 1.0 - - entry = env.tasks["fix_bug"].manifest_entry() - - schema = entry["args"] - assert set(schema["properties"]) == {"difficulty", "suite"} - assert schema["properties"]["difficulty"]["type"] == "integer" - assert schema["properties"]["suite"]["default"] == "coding" - assert schema["required"] == ["difficulty"] - assert schema["additionalProperties"] is False - - -def test_args_schema_for_no_param_task_rejects_args() -> None: - env = Environment("manifests") - - @env.template() - async def bare(): - yield "go" - yield 1.0 - - schema = env.tasks["bare"].manifest_entry()["args"] - assert schema["properties"] == {} - assert schema["additionalProperties"] is False - - -def test_args_schema_var_keyword_allows_additional() -> None: - env = Environment("manifests") - - @env.template() - async def flexible(n: int, **rest: str): - yield "go" - yield 1.0 - - schema = env.tasks["flexible"].manifest_entry()["args"] - assert set(schema["properties"]) == {"n"} - assert schema["additionalProperties"] is True - - -def test_args_schema_unannotated_param_accepts_anything() -> None: - env = Environment("manifests") - - @env.template() - async def loose(anything): - yield "go" - yield 1.0 - - schema = env.tasks["loose"].manifest_entry()["args"] - assert schema["required"] == ["anything"] - assert "type" not in schema["properties"]["anything"] - - -def test_input_and_returns_schemas_still_published() -> None: - env = Environment("manifests") - - @env.template(input=_Point, returns=_Point) - async def typed(): - yield "go" - yield 1.0 - - entry = env.tasks["typed"].manifest_entry() - assert entry["input"]["properties"]["x"]["type"] == "integer" - assert entry["returns"]["properties"]["y"]["type"] == "integer" - assert entry["args"]["properties"] == {} - - -def test_args_schema_resolves_postponed_rich_annotations() -> None: - env = Environment("manifests") - - @env.template() - async def rich(mode: _Mode, payload: _Payload, retries: _Retries = None): - yield "go" - yield 1.0 - - assert callable(rich) - schema = env.tasks["rich"].manifest_entry()["args"] - assert schema["properties"]["mode"]["enum"] == ["easy", "hard"] - assert schema["properties"]["retries"]["default"] is None - assert "$ref" in schema["properties"]["payload"] - assert set(schema["required"]) == {"mode", "payload"} diff --git a/hud/environment/tests/test_platform.py b/hud/environment/tests/test_platform.py deleted file mode 100644 index 86a22def8..000000000 --- a/hud/environment/tests/test_platform.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Regression test for importing the package where unix sockets do not exist. - -Both routes out of a workspace are served on unix sockets and a session runs on a pty, -and each reached for its platform at module scope: a -``socketserver.ThreadingUnixStreamServer`` subclass in :mod:`hud.environment.egress`, -and a top-level ``import pty`` in :mod:`hud.environment.namespace`. Neither exists on -Windows, so ``import hud`` raised there before any of it was used, taking down the -commands that never go near a workspace with it. Both now sit behind the platform -guard the rest of the package already applies. - -The breakage is at import time, so the check runs in a fresh interpreter. That -interpreter imports the package once as the platform it really is, because the -standard library and the dependencies settle their own platform at their first import -and will not be told otherwise afterwards, and then imports it again with only the -package's own modules dropped, the platform reported as Windows, and the unix pieces -taken away. -""" - -from __future__ import annotations - -import subprocess -import sys - -_IMPORT_AS_WINDOWS = """ -import socketserver -import sys - -import hud - -for name in [n for n in sys.modules if n == "hud" or n.startswith("hud.")]: - del sys.modules[name] - -sys.platform = "win32" -sys.modules["pty"] = None # a Windows interpreter has no pty module to import -socketserver.__dict__.pop("ThreadingUnixStreamServer", None) # nor this server - -import hud - -print("IMPORTED") -""" - - -def test_the_package_imports_where_unix_sockets_do_not_exist() -> None: - result = subprocess.run( - [sys.executable, "-c", _IMPORT_AS_WINDOWS], - capture_output=True, - text=True, - timeout=120, - ) - - assert result.returncode == 0, f"stdout={result.stdout!r}\nstderr={result.stderr!r}" - assert "IMPORTED" in result.stdout diff --git a/hud/environment/tests/test_process_connection.py b/hud/environment/tests/test_process_connection.py deleted file mode 100644 index d98304272..000000000 --- a/hud/environment/tests/test_process_connection.py +++ /dev/null @@ -1,368 +0,0 @@ -"""Process-bound controller connection integration.""" - -from __future__ import annotations - -import asyncio -import shlex -import subprocess -import sys -import threading -from http.server import BaseHTTPRequestHandler, HTTPServer -from pathlib import Path -from typing import Any, cast - -import pytest - -from hud.capabilities import Connection, SSHClient -from hud.clients import connect -from hud.environment import Environment, Peer -from hud.environment.egress import ANY_HOST, BRIDGE_PORT -from hud.environment.process_guard import ProcessConnectionGuard, process_connections_supported -from hud.environment.workspace import usable_bwrap -from hud.eval import LocalRuntime, Task - -pytestmark = pytest.mark.skipif( - not process_connections_supported(), - reason="process connection guards are unavailable", -) - -_GUARD_PATH = Path(__file__).parents[1] / "process_guard.py" -_PTRACE_SUPPORTED = ( - sys.platform == "linux" - and subprocess.run( - [sys.executable, str(_GUARD_PATH), "--probe", "ptrace"], - check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ).returncode - == 0 -) - - -class _ProtectedUpstream(BaseHTTPRequestHandler): - authorization: str | None = None - - def do_GET(self) -> None: - type(self).authorization = self.headers.get("Authorization") - body = b"ok" - self.send_response(200) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: Any) -> None: - pass - - -class _OrdinaryUpstream(_ProtectedUpstream): - authorization: str | None = None - - -def _server(handler: type[BaseHTTPRequestHandler]) -> tuple[HTTPServer, threading.Thread]: - server = HTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - return server, thread - - -def _fetch_source(url: str) -> str: - return ( - "import urllib.request;" - f"request=urllib.request.Request({url!r},headers={{'Authorization':'Bearer visible'}});" - "print(urllib.request.urlopen(request,timeout=5).read().decode())" - ) - - -def _fetch_script(url: str) -> str: - return f"exec {shlex.quote(sys.executable)} -c {shlex.quote(_fetch_source(url))}" - - -def _threaded_fetch_source(url: str) -> str: - return ( - "import threading,urllib.request;" - "result=[];" - f"request=urllib.request.Request({url!r});" - "thread=threading.Thread(target=lambda:result.append(" - "urllib.request.urlopen(request,timeout=5).read().decode()));" - "thread.start();thread.join();print(result[0])" - ) - - -def _proxy_fetch_source(url: str) -> str: - return ( - "import http.client,sys;" - f"connection=http.client.HTTPConnection('127.0.0.1',{BRIDGE_PORT},timeout=5);" - f"connection.request('GET',{url!r});" - "response=connection.getresponse();body=response.read();" - "sys.exit(0 if response.status==200 and body==b'ok' else 9)" - ) - - -def _direct_fetch_source(host: str, port: int) -> str: - return ( - "import http.client;" - f"connection=http.client.HTTPConnection({host!r},{port},timeout=5);" - "connection.request('GET','/');" - "print(connection.getresponse().read().decode())" - ) - - -def _proxy_environment_source() -> str: - return ( - "import os;" - "print(os.environ.get('http_proxy',''));" - "print(os.environ.get('https_proxy',''));" - "print(os.environ.get('no_proxy',''))" - ) - - -def test_guard_projects_a_standalone_helper(tmp_path: Path) -> None: - guard = ProcessConnectionGuard(tmp_path / "guard", set(), set(), backend="notify") - try: - guard.start() - assert guard.helper_path.read_bytes() == _GUARD_PATH.read_bytes() - assert guard.helper_path.stat().st_mode & 0o777 == 0o500 - finally: - guard.close() - - -@pytest.mark.skipif(not _PTRACE_SUPPORTED, reason="ptrace guard backend is unavailable") -@pytest.mark.asyncio -async def test_ptrace_backend_emulates_connects_and_blocks_descendants(tmp_path: Path) -> None: - protected, protected_thread = _server(_ProtectedUpstream) - ordinary, ordinary_thread = _server(_OrdinaryUpstream) - protected_url = f"http://127.0.0.1:{protected.server_address[1]}" - ordinary_url = f"http://127.0.0.1:{ordinary.server_address[1]}" - child = ( - "import subprocess,sys,threading,urllib.request;" - f"protected={_fetch_source(protected_url)!r};" - f"ordinary={_fetch_source(ordinary_url)!r};" - f"print(urllib.request.urlopen({protected_url!r},timeout=5).read().decode());" - "threaded=[];" - f"thread=threading.Thread(target=lambda:threaded.append(urllib.request.urlopen({protected_url!r},timeout=5).read().decode()));" - "thread.start();thread.join();print(threaded[0]);" - "blocked=subprocess.run([sys.executable,'-c',protected]);" - "print(f'blocked={blocked.returncode}',flush=True);" - "permitted=subprocess.run([sys.executable,'-c',ordinary]);" - "print(f'ordinary={permitted.returncode}')" - ) - target = ("127.0.0.1", protected.server_address[1]) - guard = ProcessConnectionGuard(tmp_path / "guard", {target}, {target}, backend="ptrace") - process = None - try: - guard.start() - process = await asyncio.create_subprocess_exec( - sys.executable, - str(_GUARD_PATH), - "--backend", - "ptrace", - str(guard.socket_path), - "--", - sys.executable, - "-c", - child, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - await guard.wait_ready() - stdout, stderr = await asyncio.wait_for(process.communicate(), 20) - assert process.returncode == 0, stderr.decode(errors="replace") - lines = stdout.decode().splitlines() - assert lines[:2] == ["ok", "ok"] - assert lines[2] != "blocked=0" - assert lines[3:] == ["ok", "ordinary=0"] - finally: - if process is not None and process.returncode is None: - process.kill() - await process.wait() - guard.close() - for server, thread in ( - (protected, protected_thread), - (ordinary, ordinary_thread), - ): - server.shutdown() - server.server_close() - thread.join() - - -@pytest.mark.skipif(usable_bwrap() is None, reason="bubblewrap isolation is unavailable") -async def test_only_bound_process_reaches_controller_connection(tmp_path: Path) -> None: - protected, protected_thread = _server(_ProtectedUpstream) - ordinary, ordinary_thread = _server(_OrdinaryUpstream) - connection = Connection( - name="inference", - capability="ssh", - url=f"http://127.0.0.1:{protected.server_address[1]}", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - env = Environment("process-connection") - env.workspace( - tmp_path / "root", - peers=( - Peer( - "ordinary.hud.invalid", - 80, - target=("127.0.0.1", ordinary.server_address[1]), - ), - ), - allowed_hosts={ANY_HOST}, - require_isolation=True, - track_files=False, - ) - task = Task(env=env.name, id="test") - try: - async with ( - LocalRuntime(env)(task) as runtime, - connect(runtime, connections=(connection,)) as client, - ): - ssh = cast("SSHClient", await client.open("ssh")) - assert ssh.capability.params["process_connections"] is True - - unbound = await ssh.run(_fetch_script(connection.client_url), check=False) - assert unbound.returncode != 0 - - bound = await ssh.create_process( - _fetch_script(connection.client_url), - connections=(connection,), - ) - completed = await bound.wait() - assert completed.returncode == 0 - assert completed.stdout == b"ok\n" - - proxy_environment = await ssh.create_process( - f"exec {shlex.quote(sys.executable)} -c {shlex.quote(_proxy_environment_source())}", - connections=(connection,), - ) - proxy_environment_result = await proxy_environment.wait() - assert proxy_environment_result.returncode == 0 - assert isinstance(proxy_environment_result.stdout, bytes) - proxy_values = proxy_environment_result.stdout.decode().splitlines() - assert proxy_values[:2] == [f"http://127.0.0.1:{BRIDGE_PORT}"] * 2 - assert "ordinary.hud.invalid" in proxy_values[2].split(",") - - threaded = await ssh.create_process( - f"exec {shlex.quote(sys.executable)} -c " - f"{shlex.quote(_threaded_fetch_source(connection.client_url))}", - connections=(connection,), - ) - threaded_result = await threaded.wait() - assert threaded_result.returncode == 0 - assert threaded_result.stdout == b"ok\n" - - child_source = ( - "import subprocess,sys;" - f"result=subprocess.run([sys.executable,'-c',{_fetch_source(connection.client_url)!r}]);" - "print(result.returncode)" - ) - child = await ssh.create_process( - f"exec {shlex.quote(sys.executable)} -c {shlex.quote(child_source)}", - connections=(connection,), - ) - child_result = await child.wait() - assert child_result.returncode == 0 - assert isinstance(child_result.stdout, bytes) - assert child_result.stdout.strip() != b"0" - - mapped_fetch = _direct_fetch_source("::ffff:127.0.0.1", connection.port) - mapped_child_source = ( - "import subprocess,sys;" - f"result=subprocess.run([sys.executable,'-c',{mapped_fetch!r}]);" - "print(result.returncode)" - ) - mapped_child = await ssh.create_process( - f"exec {shlex.quote(sys.executable)} -c {shlex.quote(mapped_child_source)}", - connections=(connection,), - ) - mapped_child_result = await mapped_child.wait() - assert mapped_child_result.returncode == 0 - assert isinstance(mapped_child_result.stdout, bytes) - assert mapped_child_result.stdout.strip() != b"0" - - proxy_child_source = ( - "import subprocess,sys;" - f"result=subprocess.run([sys.executable,'-c',{_proxy_fetch_source(connection.client_url)!r}]);" - "print(result.returncode)" - ) - proxy_child = await ssh.create_process( - f"exec {shlex.quote(sys.executable)} -c {shlex.quote(proxy_child_source)}", - connections=(connection,), - ) - proxy_child_result = await proxy_child.wait() - assert proxy_child_result.returncode == 0 - assert isinstance(proxy_child_result.stdout, bytes) - assert proxy_child_result.stdout.strip() != b"0" - - io_uring_source = ( - "import ctypes,errno,sys;" - "libc=ctypes.CDLL(None,use_errno=True);" - "result=libc.syscall(425,0,None);" - "sys.exit(0 if result == -1 and ctypes.get_errno() == errno.EPERM else 1)" - ) - io_uring = await ssh.create_process( - f"exec {shlex.quote(sys.executable)} -c {shlex.quote(io_uring_source)}", - connections=(connection,), - ) - io_uring_result = await io_uring.wait() - assert io_uring_result.returncode == 0 - - ordinary_result = await ssh.run( - _fetch_script("http://ordinary.hud.invalid"), - check=False, - ) - assert ordinary_result.returncode == 0 - assert ordinary_result.stdout == "ok\n" - finally: - for server, thread in ( - (protected, protected_thread), - (ordinary, ordinary_thread), - ): - server.shutdown() - server.server_close() - thread.join() - - assert _ProtectedUpstream.authorization == "Bearer scoped-runtime-token" - assert _OrdinaryUpstream.authorization == "Bearer visible" - - -@pytest.mark.skipif(usable_bwrap() is None, reason="bubblewrap isolation is unavailable") -async def test_a_later_session_cannot_launch_against_a_released_connection(tmp_path: Path) -> None: - protected, protected_thread = _server(_ProtectedUpstream) - connection = Connection( - name="inference", - capability="ssh", - url=f"http://127.0.0.1:{protected.server_address[1]}", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - env = Environment("process-connection") - env.workspace( - tmp_path / "root", - allowed_hosts={ANY_HOST}, - require_isolation=True, - track_files=False, - ) - task = Task(env=env.name, id="test") - try: - async with LocalRuntime(env)(task) as runtime: - async with connect(runtime, connections=(connection,)) as first: - ssh = cast("SSHClient", await first.open("ssh")) - bound = await ssh.create_process( - _fetch_script(connection.client_url), - connections=(connection,), - ) - assert (await bound.wait()).returncode == 0 - - async with connect(runtime) as later: - ssh = cast("SSHClient", await later.open("ssh")) - stale = await ssh.create_process( - _fetch_script(connection.client_url), - connections=(connection,), - ) - result = await stale.wait() - finally: - protected.shutdown() - protected.server_close() - protected_thread.join() - - assert result.returncode == 1 - assert isinstance(result.stderr, bytes) - assert b"process connection is not bound: inference" in result.stderr diff --git a/hud/environment/tests/test_robot_control.py b/hud/environment/tests/test_robot_control.py deleted file mode 100644 index 59942b6b1..000000000 --- a/hud/environment/tests/test_robot_control.py +++ /dev/null @@ -1,926 +0,0 @@ -"""Direct control gates: an LLM-style agent drives a robot sim through MCP motion tools. - -Each test is a whole rollout: an in-process bridge serving the ``openpi/0`` wire, -an env publishing it plus :class:`DirectControl`'s ``mcp`` capability, and a -scripted agent calling the tools through the tunneled manifest binding, graded -by the sim. -""" - -from __future__ import annotations - -import asyncio -import base64 -import copy -import io -import itertools -import math -import time -from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime -from contextlib import asynccontextmanager -from datetime import datetime -from typing import TYPE_CHECKING, Any, cast - -import av -import numpy as np -import pytest - -from hud.agents.base import Agent -from hud.agents.openai.tools.strict_schema import ensure_strict_json_schema -from hud.environment import Environment -from hud.environment.robot import DirectControl, RobotBridge, RobotEndpoint -from hud.eval import LocalRuntime, Task, rollout -from hud.telemetry.robot import TraceRecorder -from hud.telemetry.span import PAYLOAD_ATTRIBUTE, TASK_RUN_ID_ATTRIBUTE - -if TYPE_CHECKING: - from collections.abc import AsyncIterator - - from numpy.typing import NDArray - - from hud.capabilities import MCPClient - from hud.eval.run import Run - from hud.types import MCPToolResult - - -def _contract( - action_type: str, - low: list[float], - high: list[float], - names: list[str] | None = None, -) -> dict[str, Any]: - """One camera, a state, and an action of *action_type*. Default action is ``[x, grip]``.""" - labels = names or ["x", "grip"] - return { - "control_rate": 10, - "features": { - "observation/image": {"role": "observation", "type": "rgb"}, - "observation/state": {"role": "observation", "names": list(labels)}, - "action": { - "role": "action", - "type": action_type, - "names": list(labels), - "stats": {"min": low, "max": high}, - }, - }, - } - - -class _Arm(RobotBridge): - """A 1-D arm plus gripper: succeeds (and terminates) holding ``x >= 0.5`` closed.""" - - def __init__(self, contract: dict[str, Any]) -> None: - super().__init__() - self.contract = contract - self.delta = contract["features"]["action"]["type"] == "ee_del" - self.state = np.zeros(2) - self.actions: list[NDArray[Any]] = [] - - def reset(self, **task_args: Any) -> str: - self.state = np.zeros(2) - self.actions.clear() - return "move x to 0.5, then close the gripper" - - def step(self, action: NDArray[Any]) -> None: - self.actions.append(np.array(action[0])) - self.state = self.state + action[0] if self.delta else np.array(action[0]) - self.success = bool(self.state[0] >= 0.5 and self.state[1] >= 0.9) - - def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: - data = { - "observation/image": np.zeros((1, 8, 8, 3), dtype=np.uint8), - "observation/state": self.state[None].astype(np.float32), - } - return data, np.array([self.success]) - - -class _Sparse(dict[str, Any]): - """The dimensions a scripted call sets, by dotted name; the model nulls the rest.""" - - -def _nulled(schema: dict[str, Any], values: _Sparse) -> dict[str, Any]: - """``values`` laid out as the schema's ``targets``: every dimension listed, unset ones null.""" - targets: dict[str, Any] = {} - for key, prop in schema["properties"].items(): - if "anyOf" in prop: - leaves = {leaf: values.get(f"{key}.{leaf}") for leaf in prop["anyOf"][0]["properties"]} - targets[key] = leaves if any(v is not None for v in leaves.values()) else None - else: - targets[key] = values.get(key) - return targets - - -class _ScriptedLLM(Agent): - """Stands in for a tool-calling LLM: plays fixed MCP calls, keeps their results.""" - - def __init__(self, *calls: tuple[str, dict[str, Any]]) -> None: - super().__init__() - self.calls = calls - self.tools: set[str] = set() - self.schemas: dict[str, dict[str, Any]] = {} - self.descriptions: dict[str, str] = {} - self.results: list[MCPToolResult] = [] - - async def __call__(self, run: Run) -> None: - client = cast("MCPClient", await run.client.open("control")) - listed = await client.list_tools() - self.tools = {tool.name for tool in listed} - self.schemas = {tool.name: tool.inputSchema for tool in listed} - self.descriptions = {tool.name: tool.description or "" for tool in listed} - for name, arguments in self.calls: - properties = self.schemas[name]["properties"] - if isinstance(arguments.get("targets"), _Sparse): - arguments = { - **arguments, - "targets": _nulled(properties["targets"], arguments["targets"]), - } - if "note" not in properties: - arguments = {k: v for k, v in arguments.items() if k != "note"} - self.results.append(await client.call_tool(name, arguments)) - run.trace.content = "done" - - -@asynccontextmanager -async def _served(sim: RobotBridge, control: DirectControl) -> AsyncIterator[Environment]: - """The docs' custom-bridge env with direct control attached; *sim* runs in this process.""" - await sim.start() - server = await sim.serve_control() - env = Environment("arm") - endpoint = RobotEndpoint.remote("127.0.0.1", server.sockets[0].getsockname()[1]).attach(env) - control.attach(endpoint) - - @env.initialize - async def _up() -> None: - await endpoint.start() - for cap in await endpoint.capabilities(): - env.add_capability(cap) - - @env.shutdown - async def _down() -> None: - await endpoint.stop() - - @env.template() - async def reach() -> AsyncGenerator[Any, Any]: - ep = await endpoint.reset() - yield {"prompt": ep["prompt"]} - yield await endpoint.result() - - try: - yield env - finally: - await env.stop() - server.close() - await sim.stop() - - -def _text(result: MCPToolResult) -> str: - return "\n".join(block.text for block in result.content if block.type == "text") - - -def _move(tool: str, **values: float) -> tuple[str, dict[str, Any]]: - return tool, {"targets": _Sparse(values), "note": "move toward the goal"} - - -async def test_move_to_interpolates_absolute_targets_until_the_sim_succeeds() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM( - _move("move_to", x=0.5), - _move("move_to", grip=1.0), - ) - - async with _served(sim, DirectControl(max_step={"grip": 2.0})) as env: - run = await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert agent.tools == {"move_to", "wait"} # the ee_abs contract picked the motion tool - schema = agent.schemas["move_to"] - assert set(schema["required"]) == {"targets", "note"} - # Every dimension is listed and nullable, so strict mode keeps the schema as it is. - assert schema["properties"]["targets"]["required"] == ["x", "grip"] - assert schema["properties"]["targets"]["properties"]["x"]["type"] == ["number", "null"] - assert schema["properties"]["targets"]["properties"]["grip"]["description"] == "[-1, 1]" - assert ensure_strict_json_schema(copy.deepcopy(schema)) == schema - assert run.reward == 1.0 # graded by the sim, not the tools - # 0.1 of x's range per second at 10 Hz: 0.01 per tick, grip held at its reference. - # The sim matches the command; the call holds until two 3-tick windows show it settled. - reach_x, held, close = np.array(sim.actions[:50]), sim.actions[50:56], sim.actions[56:] - np.testing.assert_allclose(reach_x[:, 0], np.linspace(0.01, 0.5, 50), rtol=1e-6) - np.testing.assert_allclose(reach_x[:, 1], 0.0) - np.testing.assert_allclose(held, [[0.5, 0.0]] * 6) - # max_step lets the gripper switch in one tick while x holds its commanded target; - # the sim succeeds on that tick, which ends the episode. - np.testing.assert_allclose(close, [[0.5, 1.0]]) - reached, closed = agent.results - assert [block.type for block in reached.content] == ["text", "text", "image"] - # The trace viewer parses this opening line to map tool calls to ticks. - assert _text(reached).startswith("Played 56 steps (5.6 s).") - assert "observation/state: x=0.5000, grip=0.0000" in _text(reached) - assert "pose: x=0.5000, grip=0.0000" in _text(reached) - assert "off: x=0.0000, grip=0.0000" in _text(reached) - assert "At the target. Remaining offsets: 0 mm on x." in _text(reached) - assert "The episode has ended" in _text(closed) - # Tolerance only picks the verdict; nothing the model sees mentions it. - seen = " ".join([str(agent.schemas), *agent.descriptions.values(), _text(reached)]) - for word in ("toleran", "within", "settle"): - assert word not in seen.lower() - - -@pytest.mark.parametrize(("state_type", "tool"), [(None, "move_by"), ("BASE_DEL_POSE", "walk_by")]) -async def test_a_displacement_kind_plays_in_steps_within_the_per_step_box( - state_type: str | None, tool: str -) -> None: - sim = _Arm(_contract("ee_del", [-0.1, -1.0], [0.1, 1.0])) - sim.contract["features"]["action"]["state_type"] = state_type # wins over the coarse type - agent = _ScriptedLLM(_move(tool, x=0.35)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert agent.tools == {tool, "wait"} - assert "action's own components" in agent.descriptions[tool] - # Four in-box steps; a displacement returns once played. - np.testing.assert_allclose(sim.actions, [[0.0875, 0.0]] * 4) - assert "observation/state: x=0.3500" in _text(agent.results[0]) - assert "At the target" not in _text(agent.results[0]) - - -@pytest.mark.parametrize( - ("call", "error"), - [ - (("move_to", {"targets": {"x": 0.1, "z": 0.1}, "note": "go"}), "'z' was unexpected"), - (("move_to", {"targets": {"x": None, "grip": 1.0}, "note": "go"}), "'grip' was unexpected"), - (("move_to", {"targets": {"grip": None}, "note": "go"}), "'x' is a required property"), - (("move_to", {"targets": {"x": "far"}, "note": "go"}), "'far' is not of type"), - (_move("move_to", x=1.5), "x=1.5 is outside [0, 1]"), - (("move_to", {"targets": {"x": None}, "note": "go"}), "set at least one"), - (("move_to", {"targets": {"x": 0.1}, "note": " "}), "note must say"), - (("wait", {"duration": None, "note": " "}), "note must say"), - (("wait", {"duration": 101, "note": "sleep"}), "duration must be within [0, 100]"), - ], -) -async def test_an_invalid_move_is_a_correctable_error_that_leaves_the_sim_still( - call: tuple[str, dict[str, Any]], error: str -) -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(call) - - async with _served(sim, DirectControl(dims=["x"], speed=0.05)) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - (result,) = agent.results - assert result.isError - assert error in _text(result) - assert sim.actions == [] - - -class _GraspTool(DirectControl): - """Stands in for an env that serves its own motion tool on this wire.""" - - def _bind_tools(self, server) -> None: - server.add_tool( - self._make_tool( - self.move, "move_eef", "grasp targets", {"targets": self._targets_schema()} - ) - ) - - -async def test_an_env_can_replace_the_contract_motion_tool() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_eef", x=0.5)) - - async with _served(sim, _GraspTool(max_step={"grip": 2.0})) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert agent.tools == {"move_eef", "wait"} - assert sim.actions # the replacement tool still plays through the shared wire - - -class _Pose(RobotBridge): - """Tracks an absolute action. ``lag`` of 1 snaps to the command; less lags behind.""" - - def __init__(self, contract: dict[str, Any], home: list[float], lag: float = 1.0) -> None: - super().__init__() - self.contract = contract - self.home = np.asarray(home, dtype=np.float64) - self.lag = lag - self.state = self.home.copy() - self.actions: list[NDArray[Any]] = [] - - def reset(self, **task_args: Any) -> str: - self.state = self.home.copy() - self.actions.clear() - return "move" - - def step(self, action: NDArray[Any]) -> None: - target = np.asarray(action[0], dtype=np.float64) - self.actions.append(target.copy()) - self.state = self.state + self.lag * (target - self.state) - self.success = False - - def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: - data = { - "observation/image": np.zeros((1, 8, 8, 3), dtype=np.uint8), - "observation/state": self.state[None].astype(np.float32), - } - return data, np.array([self.success]) - - -async def test_an_empty_target_list_is_rejected_and_does_not_move() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to")) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - (result,) = agent.results - assert result.isError - assert "wait" in _text(result) - assert sim.actions == [] - - -async def test_a_move_longer_than_ten_seconds_plays_until_the_arm_arrives() -> None: - """VLABench was rejected at 10.1 s. The cap is a 30 s safety timeout, not that reject.""" - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to", x=0.6)) - - async with _served(sim, DirectControl(speed=0.05, max_step={"grip": 2.0})) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - (result,) = agent.results - assert not result.isError - # 0.6 / 0.05 s = 12 s at 10 Hz, which used to exceed the 10 s cap. - assert len(sim.actions) >= 120 - assert sim.state[0] == pytest.approx(0.6) - assert "At the target." in _text(result) - - -async def test_the_safety_timeout_returns_the_pose_instead_of_rejecting_the_call() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to", x=1.0)) - - async with _served(sim, DirectControl(dims=["x"], speed=0.05, timeout=0.3)) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - (result,) = agent.results - assert not result.isError - assert len(sim.actions) == 3 # 0.3 s at 10 Hz - assert sim.state[0] < 0.05 # the prefix of the move, not a jump to the goal - assert ( - "Ran out of time before settling; the arm was still moving. Remaining offsets: 985 mm on x." - ) in _text(result) - assert "pose:" in _text(result) - assert "commanded:" in _text(result) - - -async def test_dimensions_with_a_shared_prefix_nest_as_a_group_the_model_can_null_whole() -> None: - names = ["left_arm.x", "left_arm.y", "right_arm.x", "grip"] - sim = _Pose(_contract("ee_abs", [-1.0] * 4, [1.0] * 4, names), [0.0] * 4) - agent = _ScriptedLLM(_move("move_to", **{"left_arm.x": 0.1})) # right_arm: null - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - schema = agent.schemas["move_to"] - targets = schema["properties"]["targets"] - assert targets["required"] == ["grip", "left_arm", "right_arm"] - group = targets["properties"]["left_arm"]["anyOf"] - assert list(group[0]["properties"]) == ["x", "y"] - assert group[1] == {"type": "null"} - assert ensure_strict_json_schema(copy.deepcopy(schema)) == schema - np.testing.assert_allclose(sim.actions[-1], [0.1, 0.0, 0.0, 0.0], atol=1e-6) - - -async def test_without_annotation_the_tools_take_no_note_and_never_refuse_an_empty_one() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM( - ("move_to", {"targets": {"x": 0.1, "grip": None}}), ("wait", {"duration": None}) - ) - - async with _served(sim, DirectControl(use_annotation=False)) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - for schema in agent.schemas.values(): - assert "note" not in schema["properties"] - assert "note" not in schema["required"] - assert not any(result.isError for result in agent.results) - assert sim.actions - - -async def test_wait_holds_the_commanded_pose_and_a_null_duration_observes_for_one_tick() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM( - _move("move_to", x=0.2), - ("wait", {"duration": None, "note": "look"}), - ("wait", {"duration": 1.5, "note": "let it settle"}), - ) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - _, looked, waited = (_text(result) for result in agent.results) - np.testing.assert_allclose(sim.actions[26:], [[0.2, 0.0]] * 16) # held at the setpoint - assert looked.startswith("Played 1 steps (0.1 s).") - assert waited.startswith("Played 15 steps (1.5 s).") - assert "commanded: x=0.2000, grip=0.0000" in looked - assert "At the target" not in looked - - -async def test_every_reply_reports_the_time_left_and_the_tools_state_the_limit() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to", x=0.2), ("wait", {"duration": 1.0, "note": "look"})) - - async with _served(sim, DirectControl(time_limit=10.0)) as env: - run = await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert run.prompt == "move x to 0.5, then close the gripper" # VLAs see the sim's own - for tool in ("move_to", "wait"): - assert "You have 10 seconds of simulated time." in agent.descriptions[tool] - moved, waited = (_text(result) for result in agent.results) - assert moved.startswith("Played 26 steps (2.6 s). Time remaining: 7.4 s.") - assert waited.startswith("Played 10 steps (1.0 s). Time remaining: 6.4 s.") - - -async def test_a_move_crossing_the_time_limit_is_cut_there_and_ends_the_episode() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to", x=0.5), _move("move_to", x=0.1)) - - async with _served(sim, DirectControl(time_limit=2.0)) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - cut, after = (_text(result) for result in agent.results) - assert len(sim.actions) == 20 # the move needed 50 ticks; 2 s at 10 Hz were left - assert sim.state[0] == pytest.approx(0.2) - assert cut.startswith("Played 20 steps (2.0 s). Time remaining: 0.0 s.") - assert "The time limit is reached." in cut - assert "The episode has ended; stop calling tools." in cut - assert "The episode ended before the arm settled. Remaining offsets: 300 mm on x." in cut - assert "Ran out of time" not in cut - assert after.startswith("Played 0 steps (0.0 s). Time remaining: 0.0 s.") - assert "The episode has ended; stop calling tools." in after - - -async def test_a_sim_that_terminates_mid_move_is_not_reported_as_running_out_of_time() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to", x=1.0, grip=1.0)) - - async with _served(sim, DirectControl(max_step={"grip": 2.0})) as env: - run = await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - (result,) = agent.results - assert run.reward == 1.0 - assert len(sim.actions) < 100 # the sim succeeded before the 100-tick move finished - assert "The episode ended before the arm settled." in _text(result) - assert "Ran out of time" not in _text(result) - assert "The episode has ended; stop calling tools." in _text(result) - - -async def test_an_arm_already_at_the_target_is_at_rest_at_a_low_control_rate() -> None: - sim = _Pose(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0]), [0.0, 0.0]) - sim.contract["control_rate"] = 1 - agent = _ScriptedLLM(_move("move_to", x=0.05)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert "At the target." in _text(agent.results[0]) - assert "Ran out of time" not in _text(agent.results[0]) - - -class _Jitter(_Pose): - """An arm that never moves, whose sensor reads a little high then low on alternate ticks.""" - - def step(self, action: NDArray[Any]) -> None: - super().step(action) - self.state = self.home + 2e-5 * (-1) ** len(self.actions) - - -@pytest.mark.parametrize("arm", [_Pose, _Jitter]) -async def test_an_arm_that_never_moves_gives_up_at_the_settle_cap_not_the_safety_timeout( - arm: type[_Pose], -) -> None: - sim = arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0]), [0.0, 0.0], lag=0.0) - agent = _ScriptedLLM(_move("move_to", x=0.5)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - # The 50-tick move plays out; an arm that never responded is held the whole 3 s cap. - assert len(sim.actions) == 80 - text = _text(agent.results[0]) - assert "off: x=-0.5000" in text - assert ( - "Stopped short of the target; the arm is at rest. Remaining offsets: 500 mm on x." in text - ) - - -async def test_a_lagging_arm_keeps_stepping_until_it_reaches_the_target() -> None: - sim = _Pose(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0]), [0.0, 0.0], lag=0.25) - agent = _ScriptedLLM(_move("move_to", x=0.5)) - - async with _served(sim, DirectControl(max_step={"grip": 2.0})) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert len(sim.actions) > 50 # interpolation, then a hold while the arm catches up - assert sim.state[0] == pytest.approx(0.5, abs=0.01) - assert "pose:" in _text(agent.results[0]) - assert "off:" in _text(agent.results[0]) - assert "At the target. Remaining offsets: " in _text(agent.results[0]) - - -class _Offset(_Pose): - """An arm that settles a fixed ``offset`` (in action columns) away from every command.""" - - def __init__(self, contract: dict[str, Any], home: list[float], offset: list[float]) -> None: - super().__init__(contract, home) - self.offset = np.asarray(offset, dtype=np.float64) - - def step(self, action: NDArray[Any]) -> None: - super().step(action) - self.state = self.state + self.offset - - -@pytest.mark.parametrize( - ("tolerance", "verdict", "ticks"), - [ - ( - None, - "Stopped short of the target; the arm is at rest. Remaining offsets: 12 mm on x.", - 50 + 21, - ), - ({"position": 0.02}, "At the target. Remaining offsets: 12 mm on x.", 56), - ], -) -async def test_an_arm_at_rest_outside_its_tolerance_is_held_for_the_grace_period_before_stopping( - tolerance: dict[str, float] | None, verdict: str, ticks: int -) -> None: - sim = _Offset(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0]), [0.0, 0.0], [-0.012, 0.0]) - agent = _ScriptedLLM(_move("move_to", x=0.5)) - - async with _served(sim, DirectControl(tolerance=tolerance)) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert verdict in _text(agent.results[0]) - assert len(sim.actions) == ticks # tolerance only lengthens the wait, never shortens it - - -class _Delayed(_Pose): - """An arm whose state follows each command ``delay`` ticks late.""" - - def __init__(self, contract: dict[str, Any], home: list[float], delay: int) -> None: - super().__init__(contract, home) - self.delay = delay - - def step(self, action: NDArray[Any]) -> None: - super().step(action) - late = len(self.actions) - 1 - self.delay - self.state = self.actions[late] if late >= 0 else self.home - - -async def test_a_move_waits_for_a_slow_arm_even_when_already_within_tolerance() -> None: - sim = _Delayed(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0]), [0.0, 0.0], delay=10) - agent = _ScriptedLLM(_move("move_to", x=8e-4)) - - async with _served(sim, DirectControl(tolerance={"position": 0.05})) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert sim.state[0] == pytest.approx(8e-4) - assert len(sim.actions) == 17 - - -class _Fingers(_Pose): - """A hand whose two fingers close 5 cm per tick, the left one jamming on an object at 0.3.""" - - def step(self, action: NDArray[Any]) -> None: - target = np.asarray(action[0], dtype=np.float64) - self.actions.append(target.copy()) - close = np.minimum(self.state[1:] + 0.05, target[1:]) - self.state = np.concatenate([self.state[:1], close]) - self.state[1] = min(self.state[1], 0.3) - self.success = False - - -async def test_a_gripper_call_waits_for_every_finger_and_not_only_the_first_to_stop() -> None: - names = ["x", "grip_left", "grip_right"] - contract = _contract("ee_abs", [0.0, 0.0, 0.0], [1.0, 1.0, 1.0], names) - sim = _Fingers(contract, [0.0, 0.0, 0.0]) - agent = _ScriptedLLM(_move("move_to", grip_left=1.0, grip_right=1.0)) - - async with _served(sim, DirectControl(max_step={"grip_left": 2.0, "grip_right": 2.0})) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - # The left finger jams after 6 ticks; the right one closes for 20, and the call outlasts it. - np.testing.assert_allclose(sim.state, [0.0, 0.3, 1.0]) - assert 20 + 3 <= len(sim.actions) < 20 + 30 - text = _text(agent.results[0]) - assert "fingers: grip_left=0.3000, grip_right=1.0000" in text # the jam reads as no success - assert "At the target" not in text - assert "Remaining offsets" not in text - - -class _OpenHand(_Pose): - """A lagging arm whose state always reads the gripper open, as a hand-only reference does.""" - - def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: - data, success = super().get_observation() - data["observation/state"] = data["observation/state"] * [1.0, 0.0] - [0.0, 1.0] - return data, success - - -@pytest.mark.parametrize("reads_hand", [True, False]) -async def test_gripper_offsets_are_reported_only_when_the_reference_reads_the_hand( - reads_hand: bool, -) -> None: - sim = _OpenHand(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0]), [0.0, -1.0], lag=0.25) - agent = _ScriptedLLM(_move("move_to", x=0.5, grip=1.0)) - reference = None if reads_hand else (lambda data: data["observation/state"]) - - async with _served(sim, DirectControl(max_step={"grip": 2.0}, reference=reference)) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - text = _text(agent.results[0]) - assert "At the target. Remaining offsets: " in text # the gripper never decides the verdict - assert ("2 on grip" in text) == reads_hand - - -def test_a_tolerance_key_that_is_not_a_quantity_is_refused() -> None: - with pytest.raises(ValueError, match="tolerance keys"): - DirectControl(tolerance={"grip": 0.1}) - - -def _quat_names(order: str) -> list[str]: - parts = ("qx", "qy", "qz", "qw") if order == "xyzw" else ("qw", "qx", "qy", "qz") - return [f"tcp_quat.{part}" for part in parts] - - -_EULER = ["ee.x", "ee_euler.x", "ee_euler.y", "ee_euler.z"] - - -@pytest.mark.parametrize("share", [0.9, 1.1]) -@pytest.mark.parametrize( - ("action_type", "names", "move", "column", "tolerance", "label"), - [ - # Each default moves the fingertips 1 mm: directly, over a 0.15 m gripper, over a - # 2.5 m lever of joints. - ("ee_abs", ["x", "grip"], {"x": 0.5}, 0, 1e-3, "x"), - ("ee_abs", _EULER, {"roll": 0.3}, 1, 1e-3 / 0.15, "orientation"), - ("joint_pos", ["joint1", "grip"], {"joint1": 0.3}, 0, 4e-4, "joint1"), - ], -) -async def test_each_quantity_defaults_to_the_angle_that_moves_the_fingertips_one_millimetre( - share: float, - action_type: str, - names: list[str], - move: dict[str, float], - column: int, - tolerance: float, - label: str, -) -> None: - offset = [0.0] * len(names) - offset[column] = share * tolerance - contract = _contract(action_type, [-1.0] * len(names), [1.0] * len(names), names) - sim = _Offset(contract, [0.0] * len(names), offset) - agent = _ScriptedLLM(_move("move_to" if action_type == "ee_abs" else "move_joints", **move)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - text = _text(agent.results[0]) - assert ("At the target. " in text) == (share < 1) - assert ("Stopped short of the target; the arm is at rest. " in text) == (share > 1) - assert f" on {label}." in text # every verdict names the dimension and how far off it is - - -@pytest.mark.parametrize( - ("action_type", "names", "targets"), - [ - ( # LIBERO - "ee_abs", - [f"target_eef_pos.{a}" for a in "xyz"] + [f"target_eef_axis_angle.r{a}" for a in "xyz"], - ["x", "y", "z", "roll", "pitch", "yaw"], - ), - ("ee_del", ["delta_eef_pos.dx", "delta_eef_rot.droll", "gripper"], ["x", "roll", "grip"]), - ("joint_pos", ["panda_joint1", "panda_joint2"], ["joint1", "joint2"]), - ("ee_abs", ["left_x", "right_x", "left_gripper"], ["left_arm", "right_arm"]), - ("ee_abs", ["left_x", "left_pitch_deg"], ["left_x", "left_pitch_deg"]), # no standard - ], -) -async def test_tools_use_standard_dimension_names_when_every_native_name_has_one( - action_type: str, names: list[str], targets: list[str] -) -> None: - contract = _contract(action_type, [-1.0] * len(names), [1.0] * len(names), names) - sim = _Pose(contract, [0.0] * len(names)) - agent = _ScriptedLLM() - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - (schema,) = (s for tool, s in agent.schemas.items() if tool != "wait") - assert list(schema["properties"]["targets"]["properties"]) == targets - - -async def test_without_wait_only_the_motion_tool_is_served_and_its_texts_drop_waiting() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to")) - - async with _served(sim, DirectControl(use_wait=False, time_limit=10.0)) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert agent.tools == {"move_to"} - assert "Time passes only while the robot moves;" in agent.descriptions["move_to"] - assert "wait" not in _text(agent.results[0]) - - -async def test_a_yaw_slerps_an_xyzw_quaternion_instead_of_lerping_components() -> None: - names = _quat_names("xyzw") - sim = _Pose(_contract("ee_abs", [-1.0] * 4, [1.0] * 4, names), [0.0, 0.0, 0.0, 1.0]) - agent = _ScriptedLLM(_move("move_to", yaw=math.pi / 2)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - actions = np.array(sim.actions) - norms = np.linalg.norm(actions, axis=1) - np.testing.assert_allclose(norms, 1.0, atol=1e-5) - np.testing.assert_allclose( - actions[-1], [0.0, 0.0, math.sin(math.pi / 4), math.cos(math.pi / 4)], atol=1e-5 - ) - # Component lerp of the endpoints is not unit length; slerp stays on the arc. - assert len(actions) > 2 - - -async def test_a_roll_writes_a_wxyz_quaternion() -> None: - names = _quat_names("wxyz") - sim = _Pose(_contract("ee_abs", [-1.0] * 4, [1.0] * 4, names), [1.0, 0.0, 0.0, 0.0]) - agent = _ScriptedLLM(_move("move_to", roll=math.pi / 2)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - np.testing.assert_allclose( - sim.actions[-1], - [np.cos(np.pi / 4), np.sin(np.pi / 4), 0.0, 0.0], - atol=1e-5, - ) - - -async def test_an_unnamed_quaternion_is_copied_unchanged() -> None: - names = ["x", "tcp_quat.qx", "tcp_quat.qy", "tcp_quat.qz", "tcp_quat.qw"] - home = [0.0, 0.0, 0.2, 0.0, 0.9] # not a unit quaternion - sim = _Pose(_contract("ee_abs", [0.0, -1, -1, -1, -1], [1, 1, 1, 1, 1], names), home) - agent = _ScriptedLLM(_move("move_to", x=0.2)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - actions = np.array(sim.actions) - assert len(actions) > 1 - held = np.broadcast_to(actions[0, 1:], actions[:, 1:].shape) - np.testing.assert_array_equal(actions[:, 1:], held) - assert abs(np.linalg.norm(actions[0, 1:]) - 1.0) > 1e-3 - - -async def test_euler_commands_slerp_onto_axis_angle() -> None: - names = [ - "target_eef_axis_angle.rx", - "target_eef_axis_angle.ry", - "target_eef_axis_angle.rz", - ] - sim = _Pose(_contract("ee_abs", [-np.pi] * 3, [np.pi] * 3, names), [0.0, 0.0, 0.0]) - agent = _ScriptedLLM(_move("move_to", roll=math.pi / 2, yaw=math.pi / 2)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - # Intrinsic XYZ of roll=yaw=pi/2 is the quaternion (0.5, 0.5, 0.5, 0.5), angle 2pi/3. - component = (2.0 * np.pi / 3.0) / np.sqrt(3.0) - np.testing.assert_allclose(sim.actions[-1], [component, component, component], atol=1e-4) - # 2pi/3 at a tenth of a turn per second, 10 Hz; the hold after it repeats the goal. - ticks = math.ceil((2 * np.pi / 3) / (0.1 * 2 * np.pi / 10)) - angles = np.linalg.norm(sim.actions[:ticks], axis=1) - gaps = np.diff(angles) - assert gaps.min() > 0 - np.testing.assert_allclose(gaps, gaps[0], rtol=0.05) - - -async def test_an_euler_contract_is_addressed_as_roll_pitch_yaw() -> None: - names = [ - "ee_pos.x", - "ee_pos.y", - "ee_pos.z", - "ee_euler.x", - "ee_euler.y", - "ee_euler.z", - "gripper", - ] - low = [-1.0, -1.0, 0.0, -np.pi, -np.pi, -np.pi, 0.0] - high = [1.0, 1.0, 2.0, np.pi, np.pi, np.pi, 0.04] - sim = _Pose(_contract("ee_abs", low, high, names), [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.04]) - agent = _ScriptedLLM(_move("move_to", z=1.2, yaw=0.4)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - final = sim.actions[-1] - assert final[2] == pytest.approx(1.2, abs=1e-4) - assert final[5] == pytest.approx(0.4, abs=1e-4) - np.testing.assert_allclose(final[3:5], 0.0, atol=1e-4) - assert final[6] == pytest.approx(0.04) - - -async def test_a_contract_without_a_motion_type_is_refused_at_start() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - sim.contract["features"]["action"]["type"] = "joint_vel" - - with pytest.raises(ValueError, match="direct control needs an action state_type"): - async with _served(sim, DirectControl()) as env: - await env.start() - - -@pytest.mark.parametrize(("control_rate", "tick_seconds"), [(10, 0.1), (0.4, 1.0)]) -async def test_every_played_tick_streams_to_the_trace_as_state_and_one_video_per_camera( - monkeypatch: pytest.MonkeyPatch, control_rate: float, tick_seconds: float -) -> None: - spans: list[dict[str, Any]] = [] - monkeypatch.setattr("hud.types.queue_span", spans.append) - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - sim.contract["control_rate"] = control_rate - agent = _ScriptedLLM(_move("move_to", x=0.3), _move("move_to", x=0.5)) - - async with _served(sim, DirectControl()) as env: - run = await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - payloads = [ - span["attributes"][PAYLOAD_ATTRIBUTE] - for span in spans - if span["attributes"][TASK_RUN_ID_ATTRIBUTE] == run.trace_id - ] - segments = [p for p in payloads if p.get("source") == "video_segment"] - assert {segment["camera"] for segment in segments} == {"observation/image"} - mp4 = b"".join(base64.b64decode(segment["segment"]["data"]) for segment in segments) - with av.open(io.BytesIO(mp4), mode="r") as container: - frames = sum(1 for _ in container.decode(video=0)) - assert frames == 1 + len(sim.actions) # the opening scene, then every tick of both moves - observations = [p for p in payloads if p.get("source") == "observation"] - assert [obs["tick"] for obs in observations] == list(range(frames)) - assert observations[-1]["state"] - # The viewer's clock is the stamps' span: the video's length, not the calls' latency. - starts = [datetime.fromisoformat(obs["started_at"]) for obs in observations] - assert [(b - a).total_seconds() for a, b in itertools.pairwise(starts)] == pytest.approx( - [tick_seconds] * (frames - 1) - ) - - -async def test_ending_the_episode_mid_move_closes_the_recording_after_the_tick_in_flight( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr("hud.types.queue_span", lambda _span: None) - events: list[str] = [] - ends: list[asyncio.Task[None]] = [] - control = DirectControl() - - class _Recorder(TraceRecorder): - def record_observation(self, data: dict[str, Any], *, tick: int) -> None: - if tick == 2: # mid-move, the episode is ended (as a cancel would) - loop.call_soon_threadsafe( - lambda: ends.append(loop.create_task(control.end_episode())) - ) - time.sleep(0.2) - super().record_observation(data, tick=tick) - events.append("record") - - def close(self) -> None: - events.append("close") - super().close() - - monkeypatch.setattr("hud.environment.robot.control.TraceRecorder", _Recorder) - loop = asyncio.get_running_loop() - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to", x=0.5)) - - async with _served(sim, control) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - await asyncio.gather(*ends) - - assert events == ["record"] * 3 + ["close"] # ticks 0-2, then nothing after the close - - -async def test_reported_angles_wrap_into_pi_and_a_command_past_pi_is_accepted() -> None: - names = ["ee.x", "ee_euler.x", "ee_euler.y", "ee_euler.z"] - home = [0.0, 3.2, 0.0, 0.0] # a sim that reports roll unwrapped, just past pi - sim = _Pose(_contract("ee_abs", [-1.0, -4.0, -4.0, -4.0], [1.0, 4.0, 4.0, 4.0], names), home) - agent = _ScriptedLLM( - ("wait", {"duration": None, "note": "look"}), - _move("move_to", roll=3.2), - _move("move_to", roll=-math.pi), - ) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - looked, past, edge = agent.results - wrapped = 3.2 - 2 * math.pi - for result in (looked, past): - assert not result.isError - assert f"ee_euler.x={wrapped:.4f}" in _text(result) - assert f"pose: x=0.0000, roll={wrapped:.4f}" in _text(result) - assert "At the target. Remaining offsets: " in _text(past) - assert not edge.isError - assert "pi and -pi are the same orientation" in agent.descriptions["move_to"] diff --git a/hud/environment/tests/test_robot_regressions.py b/hud/environment/tests/test_robot_regressions.py deleted file mode 100644 index 824e2ab6b..000000000 --- a/hud/environment/tests/test_robot_regressions.py +++ /dev/null @@ -1,642 +0,0 @@ -"""Unit regressions for robot bridge, endpoint claims, gym shapes, and spawn state.""" - -from __future__ import annotations - -import asyncio -import contextlib -import math -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, cast -from unittest.mock import AsyncMock - -import numpy as np -import pytest - -from hud.environment.env import current_session_id -from hud.environment.robot import DirectControl -from hud.environment.robot.bridge import _HUD_STATE, RobotBridge, _apply_declaration_state -from hud.environment.robot.endpoint import RobotEndpoint, _bridge_init_kwargs -from hud.environment.robot.gym import GymBridge, action_dim_of -from hud.environment.robot.orientation import angular_distance, euler_to_xyzw, xyzw_to_euler -from hud.telemetry.robot.recorder import JobRecorder - -if TYPE_CHECKING: - from numpy.typing import NDArray - -# --- spawn state (declaration → child) ------------------------------------------------- - - -class _CustomBridge(RobotBridge): - """Subclass with a real ctor param — contract is assigned after init (docs style).""" - - def __init__(self, *, use_delta: bool = False) -> None: - super().__init__() - self.use_delta = use_delta - - def reset(self, **kwargs: Any) -> str: - return "p" - - def step(self, action: Any) -> None: - return None - - def get_observation(self) -> tuple[dict[str, Any], Any] | None: - return None - - -def test_bridge_init_kwargs_packs_declaration_contract_and_ctor_params() -> None: - bridge = _CustomBridge(use_delta=True) - bridge.contract = { - "control_rate": 10, - "features": {"action": {"role": "action", "names": ["a"]}}, - } - bridge.num_envs = 4 - bridge.metadata = {"backend": "test"} - - kwargs = _bridge_init_kwargs(bridge) - assert kwargs["use_delta"] is True - assert kwargs[_HUD_STATE]["contract"] == bridge.contract - assert kwargs[_HUD_STATE]["num_envs"] == 4 - assert kwargs[_HUD_STATE]["metadata"] == {"backend": "test"} - # Bind address stays with the child — never forwarded. - assert "host" not in kwargs and "port" not in kwargs - - -def test_bridge_init_kwargs_skips_empty_default_state() -> None: - bridge = _CustomBridge() - kwargs = _bridge_init_kwargs(bridge) - assert kwargs == {"use_delta": False} - assert _HUD_STATE not in kwargs - - -def test_apply_declaration_state_sets_contract_after_ctor() -> None: - """Child reconstructs with ctor kwargs only; state is applied afterward.""" - bridge = _CustomBridge(use_delta=True) - assert bridge.contract == {} - _apply_declaration_state( - bridge, - { - "contract": {"features": {"action": {"role": "action", "names": ["a"]}}}, - "num_envs": 2, - "metadata": {"k": 1}, - }, - ) - assert bridge.contract["features"]["action"]["names"] == ["a"] - assert bridge.num_envs == 2 - assert len(bridge._registry.slots) == 2 - assert bridge.metadata == {"k": 1} - - -# --- gym shapes ------------------------------------------------------------------------ - - -@pytest.mark.parametrize( - ("single_shape",), - [ - pytest.param((2, 2), id="keeps_single_action_space_shape"), - pytest.param(None, id="strips_batched_action_space_without_single"), - ], -) -def test_action_dim_of_batched(single_shape: tuple[int, ...] | None) -> None: - env = SimpleNamespace( - single_action_space=None if single_shape is None else SimpleNamespace(shape=single_shape), - action_space=SimpleNamespace(shape=(4, 2, 2)), - ) - assert action_dim_of(env, batched=True) == 4 - - -def test_var_keyword_factory_defaults_are_build_params() -> None: - def make_env(**kwargs: Any) -> Any: - return kwargs - - bridge = GymBridge(make_env, contract=None, num_envs=8) - assert "num_envs" in bridge._build_params - - -def test_step_reshapes_float_box_action() -> None: - class _FakeEnv: - action_space = SimpleNamespace(shape=(2, 2), dtype=np.float32) - - def __init__(self) -> None: - self.last_action: Any = None - - def step(self, action: Any) -> tuple[Any, ...]: - self.last_action = action - return np.zeros(3, dtype=np.float32), 0.0, False, False, {} - - bridge = GymBridge(lambda: None, contract=None) - bridge.env = _FakeEnv() - bridge.batched = False - bridge.num_envs = 1 - bridge._done = np.zeros(1, dtype=bool) - bridge._success = np.zeros(1, dtype=bool) - bridge._acc_reward = np.zeros(1) - bridge._step_reward = np.zeros(1) - bridge.step(np.arange(4, dtype=np.float32).reshape(1, 4)) - assert bridge.env.last_action.shape == (2, 2) - - -def test_step_reshapes_batched_float_box_action() -> None: - class _FakeEnv: - action_space = SimpleNamespace(shape=(2, 2, 2), dtype=np.float32) - - def __init__(self) -> None: - self.last_action: Any = None - - def step(self, action: Any) -> tuple[Any, ...]: - self.last_action = action - return ( - np.zeros((2, 3), dtype=np.float32), - np.zeros(2), - np.zeros(2, dtype=bool), - np.zeros(2, dtype=bool), - {}, - ) - - bridge = GymBridge(lambda: None, contract=None) - bridge.env = _FakeEnv() - bridge.batched = True - bridge.num_envs = 2 - bridge._done = np.zeros(2, dtype=bool) - bridge._success = np.zeros(2, dtype=bool) - bridge._acc_reward = np.zeros(2) - bridge._step_reward = np.zeros(2) - bridge.step(np.arange(8, dtype=np.float32).reshape(2, 4)) - assert bridge.env.last_action.shape == (2, 2, 2) - - -def test_plain_env_observation_always_gets_batch_axis() -> None: - bridge = GymBridge(lambda: None, contract=None) - bridge.env = object() - bridge.batched = False - bridge.num_envs = 1 - bridge._done = np.zeros(1, dtype=bool) - bridge._step_reward = np.array([0.5], dtype=np.float32) - bridge._obs = { - "state": np.array([1.0], dtype=np.float32), - "camera": np.zeros((1, 4, 3), dtype=np.uint8), - } - observation = bridge.get_observation() - assert observation is not None - data, terminated = observation - assert data is not None - assert data["state"].shape == (1, 1) - assert data["camera"].shape == (1, 1, 4, 3) - assert data["reward"].shape == (1,) - assert data["reward"][0] == pytest.approx(0.5) - assert terminated.shape == (1,) - - -def test_job_recorder_canonicalizes_shared_job_id(monkeypatch: pytest.MonkeyPatch) -> None: - from hud.settings import settings - - compact_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - canonical_id = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" - reports: list[tuple[str, dict[str, Any]]] = [] - monkeypatch.setattr(settings, "hud_web_url", "https://hud.test") - monkeypatch.setattr( - "hud.telemetry.robot.recorder._report_sync", - lambda path, payload: reports.append((path, payload)), - ) - - recorder = JobRecorder("test", 1, record_indices=[], job_id=compact_id) - - assert recorder.job_id == canonical_id - assert recorder.job_url == f"https://hud.test/jobs/{canonical_id}" - assert reports == [(f"/trace/job/{canonical_id}/enter", {"name": "test", "group": 1})] - - -def test_seeded_recording_preserves_trace_id_for_each_job_id_spelling( - monkeypatch: pytest.MonkeyPatch, -) -> None: - compact_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - canonical_id = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" - legacy_trace_ids = { - compact_id: "e9266339c6fa58b08102ce7f41c4f372", - canonical_id: "7b3269f689b65c5887542a9bc1ffb5c8", - } - reports: list[tuple[str, dict[str, Any]]] = [] - - def capture_report(path: str, payload: dict[str, Any]) -> None: - reports.append((path, payload)) - - monkeypatch.setattr("hud.telemetry.robot.recorder._report_sync", capture_report) - - for job_id, expected_id in legacy_trace_ids.items(): - reports.clear() - recorder = JobRecorder("test", 1, record_indices=[0], seed=7, job_id=job_id) - recorder.record(done=np.array([True])) - - assert ( - f"/trace/{expected_id}/enter", - {"job_id": canonical_id, "group_id": None, "model": None}, - ) in reports - - -def test_job_recorder_rejects_non_uuid_job_id() -> None: - with pytest.raises(ValueError, match="job_id must be a UUID"): - JobRecorder("test", 1, record_indices=[], job_id="shared-suite") - - -# --- endpoint session claims ----------------------------------------------------------- - - -@pytest.fixture -def endpoint() -> RobotEndpoint: - ep = RobotEndpoint.remote("127.0.0.1", 9) - setattr(ep, "_call", AsyncMock()) - return ep - - -@pytest.mark.asyncio -async def test_release_claim_frees_current_session_slot(endpoint: RobotEndpoint) -> None: - call = cast("AsyncMock", endpoint._call) - call.return_value = {"prompt": "p", "token": "slot-0-abcd"} - token = current_session_id.set("sess-a") - try: - await endpoint.reset() - assert endpoint._claims["sess-a"] == "slot-0-abcd" - call.return_value = {"score": 0.0} - await endpoint.release_claim() - assert "sess-a" not in endpoint._claims - call.assert_called_with("result", {"token": "slot-0-abcd"}) - finally: - current_session_id.reset(token) - - -@pytest.mark.asyncio -async def test_release_claim_ends_the_direct_control_episode( - endpoint: RobotEndpoint, monkeypatch: pytest.MonkeyPatch -) -> None: - control = DirectControl().attach(endpoint) - end_episode = AsyncMock() - monkeypatch.setattr(control, "end_episode", end_episode) - cast("AsyncMock", endpoint._call).return_value = {"prompt": "p", "token": "slot-0-abcd"} - token = current_session_id.set("sess-a") - try: - await endpoint.reset() - end_episode.reset_mock() - await endpoint.release_claim() - end_episode.assert_awaited_once() - finally: - current_session_id.reset(token) - - -@pytest.mark.asyncio -async def test_release_claim_on_shutdown_frees_each_session(endpoint: RobotEndpoint) -> None: - call = cast("AsyncMock", endpoint._call) - endpoint._claims["sess-a"] = "tok-a" - endpoint._claims["sess-b"] = "tok-b" - call.return_value = {"score": 0.0} - - for sid in ("sess-a", "sess-b"): - token = current_session_id.set(sid) - try: - await endpoint.release_claim() - finally: - current_session_id.reset(token) - - assert endpoint._claims == {} - tokens = [c.args[1]["token"] for c in call.call_args_list] - assert sorted(tokens) == ["tok-a", "tok-b"] - - -@pytest.mark.asyncio -async def test_result_marks_claim_freed_so_disconnect_does_not_re_result( - endpoint: RobotEndpoint, -) -> None: - call = cast("AsyncMock", endpoint._call) - token = current_session_id.set("sess-a") - try: - endpoint._claims["sess-a"] = "tok-a" - call.return_value = {"score": 1.0, "success": True, "total_reward": 1.0} - await endpoint.result(token="tok-a") - assert endpoint._claims["sess-a"] == "" - call.reset_mock() - await endpoint.release_claim() - call.assert_not_called() - finally: - current_session_id.reset(token) - - -@pytest.mark.asyncio -async def test_failed_release_rpc_retries_then_succeeds(endpoint: RobotEndpoint) -> None: - call = cast("AsyncMock", endpoint._call) - token = current_session_id.set("sess-a") - try: - endpoint._claims["sess-a"] = "tok-a" - call.side_effect = [ - ConnectionError("sim down"), - {"score": 0.0}, - ] - await endpoint.release_claim() - assert "sess-a" not in endpoint._claims - assert call.call_count == 2 - finally: - current_session_id.reset(token) - - -@pytest.mark.asyncio -async def test_stop_drains_claims_that_cancel_failed_to_free(endpoint: RobotEndpoint) -> None: - call = cast("AsyncMock", endpoint._call) - endpoint._claims["sess-a"] = "tok-a" - call.side_effect = [ - ConnectionError("sim down"), - ConnectionError("sim down"), - ConnectionError("sim down"), - ] - token = current_session_id.set("sess-a") - try: - await endpoint.release_claim() # retries exhausted — claim kept - assert endpoint._claims["sess-a"] == "tok-a" - finally: - current_session_id.reset(token) - - call.side_effect = None - call.return_value = {"score": 0.0} - await endpoint.stop() # last chance while the control link is up - assert endpoint._claims == {} - - -# --- bridge barrier / claim ------------------------------------------------------------ - - -class _StubBridge(RobotBridge): - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self.steps: list[NDArray[Any]] = [] - self.contract = {"features": {"action": {"role": "action", "names": ["a"]}}} - - def reset(self, **kwargs: Any) -> str: - return f"prompt:{kwargs!r}" - - def step(self, action: NDArray[Any]) -> None: - self.steps.append(np.asarray(action)) - - def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]] | None: - return {"x": np.zeros((self.num_envs, 1), dtype=np.float32)}, np.zeros( - self.num_envs, dtype=bool - ) - - -class _FakeWS: - async def send(self, _data: Any) -> None: - return None - - -@pytest.mark.asyncio -async def test_claim_awaits_legacy_async_reset() -> None: - bridge = _StubBridge() - - async def async_reset(**kwargs: Any) -> str: - await asyncio.sleep(0) - return "async-prompt" - - setattr(bridge, "reset", async_reset) - ep = await bridge._claim_episode() - assert ep["prompt"] == "async-prompt" - - -@pytest.mark.asyncio -async def test_claim_rejects_empty_kwargs_after_nonempty_batch() -> None: - bridge = _StubBridge() - bridge.num_envs = 2 - first = await bridge._claim_episode(task="A") - assert first["token"] - with pytest.raises(ValueError, match="identical args"): - await bridge._claim_episode() - - -@pytest.mark.asyncio -async def test_release_uses_overridden_result() -> None: - class _CustomGrade(_StubBridge): - def result(self) -> dict[str, Any]: - return {"score": 0.42, "success": True, "total_reward": 3.0, "detail": "custom"} - - bridge = _CustomGrade() - ep = await bridge._claim_episode() - grade = await bridge._release_episode(ep["token"]) - assert grade == {"score": 0.42, "success": True, "total_reward": 3.0, "detail": "custom"} - - -@pytest.mark.asyncio -async def test_tick_loop_does_not_hold_spin_when_all_claimed_idle() -> None: - """Terminated slots may keep WS open until close(); barrier must not step.""" - bridge = _StubBridge() - bridge.num_envs = 1 - bridge._registry.configure(1) - slot = bridge._registry.slots[0] - bridge._registry.claim(slot) - slot.ws = _FakeWS() # still connected - slot.idle = True # terminated - slot.action = None - - task = asyncio.create_task(bridge._tick_loop()) - bridge._action_event.set() - await asyncio.sleep(0.05) - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - assert bridge.steps == [] - - -@pytest.mark.asyncio -async def test_tick_loop_stops_after_terminal_obs_while_ws_still_open() -> None: - """Terminate wakes the barrier; idle+open WS must not cascade hold-steps. - - The e2e gate ``sim.tick == EPISODE_TICKS`` fails if each terminal fan-out - re-arms the loop into a hold-spin before the agent closes the socket. - """ - - class _Terminating(_StubBridge): - def __init__(self) -> None: - super().__init__() - self.tick = 0 - - def step(self, action: NDArray[Any]) -> None: - self.tick += 1 - super().step(action) - - def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]] | None: - data = {"x": np.zeros((self.num_envs, 1), dtype=np.float32)} - return data, np.array([self.tick >= 3]) - - bridge = _Terminating() - bridge.num_envs = 1 - bridge._registry.configure(1) - slot = bridge._registry.slots[0] - bridge._registry.claim(slot) - slot.ws = _FakeWS() - task = asyncio.create_task(bridge._tick_loop()) - try: - for expected in (1, 2, 3): - slot.idle = False - slot.action = np.array([1.0], dtype=np.float32) - bridge._action_event.set() - for _ in range(50): - if bridge.tick >= expected: - break - await asyncio.sleep(0.01) - assert bridge.tick == expected - assert slot.idle # terminal obs dropped the slot out of the barrier - for _ in range(20): # spam the wake that terminal fan-out itself sets - bridge._action_event.set() - await asyncio.sleep(0.005) - assert bridge.tick == 3 - assert len(bridge.steps) == 3 - finally: - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - - -@pytest.mark.asyncio -async def test_claim_raises_when_batch_full() -> None: - bridge = _StubBridge() - bridge.num_envs = 1 - bridge._registry.configure(1) - first = await bridge._claim_episode(goal="a") - with pytest.raises(RuntimeError, match="slots are claimed"): - await bridge._claim_episode(goal="a") - await bridge._release_episode(first["token"]) - second = await bridge._claim_episode(goal="a") - assert second["token"] - - -@pytest.mark.asyncio -async def test_endpoint_reset_retries_until_peer_result_frees_slot() -> None: - """Shared width+1: reset retries outside the RPC lock so result is not deadlocked.""" - bridge = _StubBridge() - bridge.num_envs = 1 - server = await bridge.serve_control("127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - ep = RobotEndpoint.remote("127.0.0.1", port) - await ep.start() - try: - first = await ep.reset(goal="a") - waiting = asyncio.create_task(ep.reset(goal="a")) - await asyncio.sleep(0.05) - assert not waiting.done() - await asyncio.wait_for(ep.result(token=first["token"]), timeout=1.0) - second = await asyncio.wait_for(waiting, timeout=1.0) - assert second["token"] != first["token"] - finally: - await ep.stop() - server.close() - await server.wait_closed() - - -@pytest.mark.asyncio -async def test_tick_loop_times_out_silent_live_slot() -> None: - """A connected agent that never sends must not stall the barrier forever.""" - bridge = _StubBridge() - bridge.step_timeout = 0.05 - bridge.num_envs = 2 - bridge._registry.configure(2) - a, b = bridge._registry.slots - bridge._registry.claim(a) - bridge._registry.claim(b) - a.ws, b.ws = _FakeWS(), _FakeWS() - a.idle = b.idle = False - a.action = np.array([1.0], dtype=np.float32) # ready - b.action = None # silent - - task = asyncio.create_task(bridge._tick_loop()) - bridge._action_event.set() - for _ in range(50): - if bridge.steps: - break - await asyncio.sleep(0.02) - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - assert len(bridge.steps) == 1 - assert b.idle # timed out - - -@pytest.mark.asyncio -async def test_tick_loop_does_not_hold_step_undialed_claimed_slot() -> None: - """A peer that has claimed but not WS-connected must not be hold-advanced.""" - bridge = _StubBridge() - bridge.step_timeout = 0.05 - bridge.num_envs = 2 - bridge._registry.configure(2) - a, b = bridge._registry.slots - bridge._registry.claim(a) - bridge._registry.claim(b) - a.ws = _FakeWS() - a.idle = b.idle = False - a.action = np.array([1.0], dtype=np.float32) - # b: claimed, still dialing (ws is None) - - task = asyncio.create_task(bridge._tick_loop()) - bridge._action_event.set() - await asyncio.sleep(0.15) - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - assert bridge.steps == [] - assert not b.idle # still waiting to dial — not timed out into hold - - -def test_both_pitch_singularities_keep_roll_through_a_partial_yaw_merge() -> None: - """A wrist at ±pi/2 reports its twist as roll, so naming only yaw rebuilds that start.""" - roll = 0.4 - yaw = -0.25 - for pitch in (math.pi / 2, -math.pi / 2): - got_roll, got_pitch, got_yaw = xyzw_to_euler(euler_to_xyzw(roll, pitch, 0.0)) - assert got_roll == pytest.approx(roll) - assert got_pitch == pytest.approx(pitch) - assert got_yaw == pytest.approx(0.0, abs=1e-8) - merged = euler_to_xyzw(got_roll, got_pitch, yaw) - assert angular_distance(merged, euler_to_xyzw(roll, pitch, yaw)) == pytest.approx( - 0.0, abs=1e-8 - ) - - -class _VectorAutoReset: - """Two lockstep envs that reset inside ``step``; env 1 finishes on the first step. - - ``per_env`` reports the final observation as gymnasium vector envs do (one obs per env), - otherwise as Isaac Lab does (an obs-shaped batch). - """ - - num_envs = 2 - action_space = SimpleNamespace(shape=(2, 1), dtype=np.float32) - - def __init__(self, per_env: bool) -> None: - self.per_env = per_env - self.ticks = np.zeros(2) - - def _observe(self) -> dict[str, NDArray[Any]]: - return {"state": self.ticks[:, None].astype(np.float32)} - - def reset(self, seed: int | None = None, options: Any = None) -> tuple[Any, dict[str, Any]]: - self.ticks[:] = 0 - return self._observe(), {} - - def step(self, action: NDArray[Any]) -> tuple[Any, ...]: - self.ticks += 1 - final = self._observe() - done = np.array([False, True]) - self.ticks[done] = 0 - if self.per_env: - final = np.array([None, {"state": final["state"][1]}], dtype=object) - return self._observe(), np.zeros(2), np.zeros(2, dtype=bool), done, {"final_obs": final} - - def close(self) -> None: - pass - - -@pytest.mark.parametrize("per_env", [False, True]) -def test_a_finished_env_is_observed_on_the_frame_it_ended_on(per_env: bool) -> None: - bridge = GymBridge(lambda: _VectorAutoReset(per_env), contract=None) - bridge.reset() - bridge.step(np.zeros((2, 1), dtype=np.float32)) - - observation = bridge.get_observation() - - assert observation is not None - data, terminated = observation - assert terminated.tolist() == [False, True] - # Env 1 was reset after its first tick; its observation is still that tick. - assert data["state"].ravel().tolist() == [1.0, 1.0] diff --git a/hud/environment/tests/test_server.py b/hud/environment/tests/test_server.py deleted file mode 100644 index 3dc34e1b5..000000000 --- a/hud/environment/tests/test_server.py +++ /dev/null @@ -1,248 +0,0 @@ -"""The wire grade contract: ``tasks.grade`` frames carry a numeric ``score``. - -The server normalizes every grade yield to the canonical frame and fails -loudly on authoring bugs (a grade that is neither a number, a ``.reward`` -object, nor a ``{"score": ...}`` dict) instead of silently grading 0.0. -""" - -from __future__ import annotations - -import asyncio -import contextlib -from typing import Literal -from urllib.parse import urlsplit - -import pytest -from pydantic import BaseModel - -from hud.clients import HudProtocolError, connect -from hud.environment import Answer, Environment -from hud.environment.utils import FrameTooLargeError, encode_frame, read_frame, send_frame -from hud.eval import LocalRuntime, Run, Task -from hud.graders import EvaluationResult, SubScore - -from .conftest import served - - -class _Payload(BaseModel): - text: str - - -_Mode = Literal["upper", "lower"] - - -async def test_dict_grade_without_numeric_score_errors_loudly() -> None: - env = Environment("badgrade") - - @env.template() - async def reward_keyed(): - yield "go" - yield {"reward": 1.0} # wrong key: the wire grade frame is {"score": ...} - - async with served(env) as client: - with pytest.raises(HudProtocolError, match="score"): - async with Run(client, "reward_keyed", {}) as run: - run.trace.content = "x" - - -async def test_non_numeric_grade_errors_loudly() -> None: - env = Environment("badgrade") - - @env.template() - async def stringy(): - yield "go" - yield "great job" - - async with served(env) as client: - with pytest.raises(HudProtocolError, match="yield a number"): - async with Run(client, "stringy", {}) as run: - run.trace.content = "x" - - -async def test_score_dict_passes_through_with_extra_keys() -> None: - env = Environment("richgrade") - - @env.template() - async def rich(): - yield "go" - yield {"score": 0.5, "info": {"detail": "partial credit"}} - - async with served(env) as client: - async with Run(client, "rich", {}) as run: - run.trace.content = "x" - assert run.reward == 0.5 - assert run.grade.info == {"detail": "partial credit"} - - -async def test_evaluation_result_info_reaches_evaluate_step() -> None: - env = Environment("modelgrade") - - @env.template() - async def graded(): - yield "go" - yield EvaluationResult( - reward=0.75, - content="nice", - info={"max_tile": 256}, - subscores=[ - SubScore( - name="judge", - value=0.75, - children=[ - SubScore( - name="criterion", - value=1.0, - info={"reason": "because"}, - ) - ], - info={"model": "judge-model"}, - ) - ], - ) - - async with served(env) as client: - async with Run(client, "graded", {}) as run: - run.trace.content = "x" - assert run.reward == 0.75 - assert run.grade.info == {"max_tile": 256} - assert "info" not in run.evaluation["subscores"][0] - assert "info" not in run.evaluation["subscores"][0]["children"][0] - evaluate_step = run.trace.steps[-1] - assert evaluate_step.task_call is not None - assert evaluate_step.task_call.phase == "evaluate" - assert evaluate_step.task_call.result == { - "score": 0.75, - "done": True, - "content": "nice", - "info": {"max_tile": 256}, - "isError": False, - "subscores": [ - { - "name": "judge", - "weight": 1.0, - "value": 0.75, - "children": [ - { - "name": "criterion", - "weight": 1.0, - "value": 1.0, - "children": None, - "info": {"reason": "because"}, - } - ], - "info": {"model": "judge-model"}, - } - ], - } - - -def test_answer_holds_parsed_content_and_raw_string() -> None: - answer = Answer(content={"final": "42"}, raw='{"final": "42"}') - assert answer.content == {"final": "42"} - assert answer.raw == '{"final": "42"}' - - -async def test_start_coerces_postponed_rich_annotations() -> None: - env = Environment("coerce") - - @env.template() - async def typed(mode: _Mode, payload: _Payload, retries: int | None = None): - if mode == "upper": - prompt = payload.text.upper() - elif mode == "lower": - prompt = payload.text.lower() - else: - raise ValueError(f"unexpected mode: {mode!r}") - if retries is not None: - prompt += "!" * retries - yield prompt - yield 1.0 - - assert callable(typed) - async with served(env) as client: - async with Run( - client, - "typed", - { - "mode": '"upper"', - "payload": '{"text":"hello"}', - "retries": "3", - }, - ) as run: - run.trace.content = "x" - assert run.prompt == "HELLO!!!" - - -@pytest.mark.parametrize("frame_size", [72483, 1024 * 1024, 16777216, 16777217]) -async def test_task_frame_limit_and_retry(frame_size: int) -> None: - env = Environment("frame-limit") - - @env.template() - async def task(data: str): - yield data - yield {"score": 1.0, "detail": data} - - frame = { - "jsonrpc": "2.0", - "id": 2, - "method": "tasks.start", - "params": {"id": "task", "args": {"data": ""}}, - } - data = "x" * (frame_size - (len(encode_frame(frame)) - 1)) - async with served(env) as client: - if frame_size > 16777216: - with pytest.raises(FrameTooLargeError, match="16777217 bytes; limit is 16777216 bytes"): - await client.start_task("task", {"data": data}) - data = "small retry" - assert (await client.start_task("task", {"data": data}))["prompt"] == data - assert (await client.grade({"answer": "done"}))["detail"] == data - - -@pytest.mark.parametrize("result_kind", ["prompt", "grade", "error"]) -async def test_oversized_response_releases_task_and_reports_error(result_kind: str) -> None: - env = Environment("large-response") - released = asyncio.Event() - - @env.template() - async def task(): - try: - if result_kind == "error": - raise ValueError("x" * (16 * 1024 * 1024)) - yield "x" * (16 * 1024 * 1024) if result_kind == "prompt" else "ready" - yield {"score": 1.0, "detail": "x" * (16 * 1024 * 1024)} - finally: - released.set() - - async with LocalRuntime(env)(Task(env=env.name, id="task")) as runtime: - async with connect(runtime) as client: - with pytest.raises(HudProtocolError, match=r"response is .*limit is 16777216 bytes"): - await client.start_task("task") - await client.grade({"answer": "done"}) - assert released.is_set() - assert (await client.list_tasks())[0]["id"] == "task" - async with connect(runtime) as resumed: - with pytest.raises(HudProtocolError, match="no task in progress"): - await resumed.grade({"answer": "done"}) - - -@pytest.mark.parametrize("hello_first,large_id", [(False, False), (True, False), (False, True)]) -async def test_wire_size_errors_are_bounded(hello_first: bool, large_id: bool) -> None: - frame = {"jsonrpc": "2.0", "id": "", "method": "nope"} - field = "id" if large_id else "padding" - frame[field] = "x" * (16777216 - (len(encode_frame(frame)) - 1) if large_id else 16777216) - env = Environment("wire-limit") - async with LocalRuntime(env)(Task(env=env.name, id="unused")) as runtime: - address = urlsplit(runtime.url) - reader, writer = await asyncio.open_connection(address.hostname, address.port) - try: - if hello_first: - await send_frame(writer, {"jsonrpc": "2.0", "id": 1, "method": "hello"}) - assert await read_frame(reader) is not None - writer.write(encode_frame(frame)) - response = await asyncio.wait_for(read_frame(reader), timeout=5.0) - assert response is not None and response["id"] is None - assert "16777216 bytes" in response["error"]["message"] - finally: - writer.close() - with contextlib.suppress(ConnectionError): - await writer.wait_closed() diff --git a/hud/environment/tests/test_sessions.py b/hud/environment/tests/test_sessions.py deleted file mode 100644 index d5f660dee..000000000 --- a/hud/environment/tests/test_sessions.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Control-channel sessions: each connection drives its own task. - -Suspended tasks are keyed by session id: concurrent connections start and -grade independently, a dropped connection parks its task, and a later -connection grades a parked task by resuming the session (``hello`` with its -id) or — only when exactly one is parked — by a plain ``tasks.grade`` (the -split ``hud task start`` / ``hud task grade`` flow). -""" - -from __future__ import annotations - -import asyncio -from typing import TYPE_CHECKING - -import pytest - -from hud.capabilities import Connection -from hud.clients import HudProtocolError, connect -from hud.environment import Environment, Workspace, WorkspaceRoute -from hud.eval import LocalRuntime, Task - -if TYPE_CHECKING: - from pathlib import Path - -_SESSION = Task(env="sessions", id="echo") - - -def _env() -> Environment: - env = Environment("sessions") - - @env.template() - async def echo(tag: str): - yield f"go {tag}" - yield {"score": 1.0, "tag": tag} - - return env - - -async def test_concurrent_sessions_grade_their_own_tasks() -> None: - async with ( - LocalRuntime(_env())(_SESSION) as runtime, - connect(runtime) as a, - connect(runtime) as b, - ): - await a.start_task("echo", {"tag": "a"}) - await b.start_task("echo", {"tag": "b"}) # must not disturb a's task - assert (await a.grade({"answer": "x"}))["tag"] == "a" - assert (await b.grade({"answer": "x"}))["tag"] == "b" - - -async def test_restart_replaces_only_the_sessions_own_task() -> None: - async with ( - LocalRuntime(_env())(_SESSION) as runtime, - connect(runtime) as a, - connect(runtime) as b, - ): - await b.start_task("echo", {"tag": "b"}) - await a.start_task("echo", {"tag": "first"}) - await a.start_task("echo", {"tag": "second"}) - assert (await a.grade({"answer": "x"}))["tag"] == "second" - assert (await b.grade({"answer": "x"}))["tag"] == "b" - - -@pytest.mark.parametrize("write_error", [None, ConnectionError, asyncio.CancelledError]) -async def test_disconnect_parks_the_task_for_a_later_connection( - monkeypatch: pytest.MonkeyPatch, write_error: type[BaseException] | None -) -> None: - real_write = asyncio.StreamWriter.write - - def write(writer: asyncio.StreamWriter, data: bytes) -> None: - if write_error is not None and b'"prompt":' in data: - writer.close() - raise write_error() - real_write(writer, data) - - monkeypatch.setattr(asyncio.StreamWriter, "write", write) - async with LocalRuntime(_env())(_SESSION) as runtime: - async with connect(runtime) as first: - assert first.manifest is not None - session_id = first.manifest.session_id - if write_error is None: - await first.start_task("echo", {"tag": "parked"}) - else: - with pytest.raises(EOFError): - await first.start_task("echo", {"tag": "parked"}) - async with connect(runtime) as later: - await later.hello(session_id=session_id) - assert (await later.grade({"answer": "x"}))["tag"] == "parked" - - -async def test_grade_with_multiple_parked_sessions_errors_loudly() -> None: - async with LocalRuntime(_env())(_SESSION) as runtime: - for tag in ("one", "two"): - async with connect(runtime) as client: - await client.start_task("echo", {"tag": tag}) - async with connect(runtime) as later: - with pytest.raises(HudProtocolError, match="parked sessions"): - await later.grade({"answer": "x"}) - - -async def test_hello_resumes_a_parked_session_by_id() -> None: - async with LocalRuntime(_env())(_SESSION) as runtime: - ids: dict[str, str] = {} - for tag in ("one", "two"): - async with connect(runtime) as client: - assert client.manifest is not None - ids[tag] = client.manifest.session_id - await client.start_task("echo", {"tag": tag}) - async with connect(runtime) as later: - await later.hello(session_id=ids["two"]) - assert (await later.grade({"answer": "x"}))["tag"] == "two" - - -async def test_hello_with_an_unknown_session_id_errors() -> None: - async with LocalRuntime(_env())(_SESSION) as runtime, connect(runtime) as client: - with pytest.raises(HudProtocolError, match="unknown session"): - await client.hello(session_id="sess-nope") - - -async def test_hello_cannot_resume_a_live_session() -> None: - async with ( - LocalRuntime(_env())(_SESSION) as runtime, - connect(runtime) as a, - connect(runtime) as b, - ): - assert a.manifest is not None - await a.start_task("echo", {"tag": "a"}) - with pytest.raises(HudProtocolError, match="live connection"): - await b.hello(session_id=a.manifest.session_id) - - -async def test_a_finished_session_releases_its_connection_for_the_next_rollout( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(Workspace, "supports_process_connections", property(lambda _self: True)) - env = _env() - workspace = env.workspace(tmp_path / "root") - first, second = ( - Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": f"Bearer {token}"}, - ) - for token in ("first-rollout", "rotated") - ) - - async with LocalRuntime(env)(_SESSION) as runtime: - async with connect(runtime, connections=[first]) as a: - await a.start_task("echo", {"tag": "a"}) - assert (await a.grade({"answer": "x"}))["tag"] == "a" - async with connect(runtime, connections=[second]) as b: - await b.start_task("echo", {"tag": "b"}) - assert [peer.name for peer in workspace.peers].count(second.host) == 1 - assert (await b.grade({"answer": "x"}))["tag"] == "b" - - -async def test_a_live_session_keeps_its_connection_name_from_other_sessions( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(Workspace, "supports_process_connections", property(lambda _self: True)) - env = _env() - env.workspace(tmp_path / "root") - first, second = ( - Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": f"Bearer {token}"}, - ) - for token in ("first", "second") - ) - - async with LocalRuntime(env)(_SESSION) as runtime, connect(runtime, connections=[first]): - with pytest.raises(HudProtocolError, match="bound to another control session"): - async with connect(runtime, connections=[second]): - pass - - -async def test_a_finished_session_takes_its_workspace_route_with_it( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(Workspace, "bwrap_available", property(lambda _self: True)) - env = _env() - workspace = env.workspace(tmp_path / "root", network=False) - route = WorkspaceRoute("ssh", "inference.hud.so", 443) - - async with LocalRuntime(env)(_SESSION) as runtime: - async with connect(runtime, workspace_routes=[route]) as a: - await a.start_task("echo", {"tag": "a"}) - assert [peer.name for peer in workspace.peers] == ["inference.hud.so"] - await a.grade({"answer": "x"}) - async with connect(runtime) as later: - await later.start_task("echo", {"tag": "later"}) - assert workspace.peers == () - await later.grade({"answer": "x"}) diff --git a/hud/environment/tests/test_tunnel.py b/hud/environment/tests/test_tunnel.py deleted file mode 100644 index aef7b8e86..000000000 --- a/hud/environment/tests/test_tunnel.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Capability tunneling: environment daemons reached through the control port. - -A capability address belongs to the environment's network. The raw manifest -retains that address for processes executing there, while the client binding -points at a local forwarder. Each connection to it becomes one fresh connection -to the control port, opened with a ``tunnel.open`` preface frame and spliced raw -to the daemon. The preface is transport-level routing — a connection is a stream -or a control session from its first frame, never upgraded mid-session. These -tests drive that path end to end against a served env fronting a real TCP echo -server. -""" - -from __future__ import annotations - -import asyncio -from typing import TYPE_CHECKING -from unittest.mock import Mock -from urllib.parse import urlsplit - -import pytest - -from hud.capabilities import Capability -from hud.environment import Environment -from hud.environment.utils import encode_frame, read_frame, send_frame - -from .conftest import served - -if TYPE_CHECKING: - from collections.abc import AsyncIterator - - -@pytest.fixture -async def echo_port() -> AsyncIterator[int]: - """A substrate-side TCP daemon: echoes every byte back.""" - - async def echo(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - while data := await reader.read(1024): - writer.write(data) - await writer.drain() - finally: - writer.close() - - server = await asyncio.start_server(echo, "127.0.0.1", 0) - yield server.sockets[0].getsockname()[1] - server.close() - await server.wait_closed() - - -def _echo_env(port: int) -> Environment: - cap = Capability(name="echo", protocol="rfb/3.8", url=f"rfb://127.0.0.1:{port}", params={}) - return Environment("echo-env", capabilities=[cap]) - - -def test_mcp_capabilities_materialize_default_tunnel_ports() -> None: - assert Capability.mcp(url="http://tools.example/mcp").url == "http://tools.example:80/mcp" - assert Capability.mcp(url="https://tools.example/mcp").url == "https://tools.example:443/mcp" - assert Capability.mcp(url="ws://tools.example/mcp").url == "ws://tools.example:80/mcp" - assert Capability.mcp(url="wss://tools.example/mcp").url == "wss://tools.example:443/mcp" - assert ( - Capability.mcp(url="https://user:secret@tools.example/mcp").url - == "https://user:secret@tools.example:443/mcp" - ) - - -async def test_bytes_round_trip_through_the_forwarded_binding(echo_port: int) -> None: - async with served(_echo_env(echo_port)) as client: - assert client.manifest is not None - assert urlsplit(client.manifest.bindings[0].url).port == echo_port - parts = urlsplit(client.binding("echo").url) - assert parts.port != echo_port # the binding points at the forwarder - - reader, writer = await asyncio.open_connection(parts.hostname, parts.port) - writer.write(b"ping through the tunnel") - await writer.drain() - assert await reader.readexactly(23) == b"ping through the tunnel" - writer.close() - await writer.wait_closed() - - -async def test_concurrent_tunnel_streams_do_not_interleave(echo_port: int) -> None: - async with served(_echo_env(echo_port)) as client: - parts = urlsplit(client.binding("echo").url) - - async def stream(payload: bytes) -> bytes: - reader, writer = await asyncio.open_connection(parts.hostname, parts.port) - writer.write(payload) - await writer.drain() - data = await reader.readexactly(len(payload)) - writer.close() - await writer.wait_closed() - return data - - payloads = [f"stream-{i}".encode() * 100 for i in range(8)] - assert await asyncio.gather(*(stream(p) for p in payloads)) == payloads - - -async def test_tunnel_open_for_an_unknown_capability_returns_an_error_frame( - echo_port: int, -) -> None: - async with served(_echo_env(echo_port)) as client: - assert client._endpoint is not None - reader, writer = await asyncio.open_connection(*client._endpoint) - await send_frame( - writer, - {"jsonrpc": "2.0", "id": 1, "method": "tunnel.open", "params": {"capability": "nope"}}, - ) - opened = await read_frame(reader) - assert opened is not None and "error" in opened - assert "nope" in opened["error"]["message"] - writer.close() - await writer.wait_closed() - - -async def test_tunnel_open_mid_session_is_not_a_method(echo_port: int) -> None: - """A control session never mutates into a stream: tunnel.open is a preface only.""" - async with served(_echo_env(echo_port)) as client: - assert client._endpoint is not None - reader, writer = await asyncio.open_connection(*client._endpoint) - await send_frame(writer, {"jsonrpc": "2.0", "id": 1, "method": "tasks.list"}) - assert await read_frame(reader) is not None # a control session is established - await send_frame( - writer, - {"jsonrpc": "2.0", "id": 2, "method": "tunnel.open", "params": {"capability": "echo"}}, - ) - opened = await read_frame(reader) - assert opened is not None and "error" in opened - assert opened["error"]["code"] == -32601 # method not found - writer.close() - await writer.wait_closed() - - -async def test_closing_the_client_tears_down_its_forwarders(echo_port: int) -> None: - async with served(_echo_env(echo_port)) as client: - parts = urlsplit(client.binding("echo").url) - - with pytest.raises(OSError): - _, writer = await asyncio.open_connection(parts.hostname, parts.port) - writer.close() - - -@pytest.mark.parametrize("frame_size", [70000, 16 * 1024 * 1024]) -async def test_buffered_frame_preserves_newline_boundary_and_backpressure(frame_size: int) -> None: - reader = asyncio.StreamReader() - transport = Mock(spec=asyncio.Transport) - reader.set_transport(transport) - frame = {"padding": ""} - frame["padding"] = "x" * (frame_size - (len(encode_frame(frame)) - 1)) - reader.feed_data(encode_frame(frame) + b"raw bytes") - assert await asyncio.wait_for(read_frame(reader, max_bytes=16 * 1024 * 1024), 1) == frame - assert await reader.readexactly(9) == b"raw bytes" - - transport.pause_reading.reset_mock() - reader.feed_data(b"x" * (128 * 1024 + 1)) - transport.pause_reading.assert_called_once() diff --git a/hud/environment/tests/test_workspace.py b/hud/environment/tests/test_workspace.py deleted file mode 100644 index e1ec28834..000000000 --- a/hud/environment/tests/test_workspace.py +++ /dev/null @@ -1,2656 +0,0 @@ -"""Workspace contract tests: credential placement and the shell_uid wall.""" - -from __future__ import annotations - -import asyncio -import base64 -import contextlib -import itertools -import json -import os -import shutil -import socket -import struct -import subprocess -import sys -import tempfile -import threading -import time -import urllib.parse -from pathlib import Path -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, cast -from unittest import mock -from unittest.mock import AsyncMock, Mock - -import asyncssh -import pytest - -from hud.capabilities import Connection, SSHClient -from hud.capabilities.ssh import PROCESS_CONNECTIONS_REQUEST -from hud.environment import namespace as namespace_mod -from hud.environment import process_guard as process_guard_mod -from hud.environment import workspace as workspace_mod -from hud.environment.egress import ( - ConnectionRelay, - Peer, - WorkspaceRoute, - _field, - _Unrelayable, -) -from hud.environment.workspace import Bubblewrap, Mount, Workspace -from hud.utils.process import ProcessGroup, ProcessResult - -if TYPE_CHECKING: - from collections.abc import AsyncIterator, Mapping - -pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX workspace semantics") - - -async def _connect(ws: Workspace) -> asyncssh.SSHClientConnection: - host, port = ws.ssh_url.removeprefix("ssh://").split(":") - key_path = ws.ssh_client_key_path - assert key_path is not None - return await asyncssh.connect( - host, - int(port), - username=ws.ssh_user, - client_keys=[str(key_path)], - known_hosts=None, - ) - - -@contextlib.asynccontextmanager -async def _connected_namespace_host( - monkeypatch: pytest.MonkeyPatch, -) -> AsyncIterator[namespace_mod.NamespaceHost]: - socket_dir = tempfile.TemporaryDirectory(prefix="hud-namespace-", dir="/tmp") - socket_path = Path(socket_dir.name) / "host.sock" - server_host = namespace_mod._NamespaceHost( - socket_path, - setup_loopback=False, - holder_argv=[], - bwrap="", - launcher_depth=0, - map_identities=False, - ports=frozenset(), - ) - holders = [SimpleNamespace(terminate=AsyncMock()) for _ in range(2)] - monkeypatch.setattr( - server_host, - "_start_holder", - AsyncMock(side_effect=[(holder, os.getpid()) for holder in holders]), - ) - - create_process_group_exec = namespace_mod.create_process_group_exec - - async def spawn_without_nsenter(*argv: str, **kwargs: Any) -> ProcessGroup: - command = argv[argv.index("--") + 1 :] - return await create_process_group_exec(*command, **kwargs) - - monkeypatch.setattr(namespace_mod, "create_process_group_exec", spawn_without_nsenter) - listen = asyncssh.listen - listening = asyncio.Event() - - async def listen_and_signal(*args: Any, **kwargs: Any) -> asyncssh.SSHAcceptor: - server = await listen(*args, **kwargs) - listening.set() - return server - - monkeypatch.setattr(asyncssh, "listen", listen_and_signal) - serve_task = asyncio.create_task(server_host.serve()) - client: namespace_mod.NamespaceHost | None = None - try: - await asyncio.wait_for(listening.wait(), 1.0) - client = namespace_mod.NamespaceHost(socket_path) - await client.connect() - yield client - finally: - if client is not None: - await client.close() - serve_task.cancel() - await asyncio.gather(serve_task, return_exceptions=True) - socket_dir.cleanup() - - -def _wait_for_path(path: Path, timeout: float = 5.0) -> None: - deadline = time.monotonic() + timeout - while not path.exists(): - if time.monotonic() >= deadline: - raise TimeoutError(f"timed out waiting for {path}") - time.sleep(0.01) - - -@pytest.mark.asyncio -async def test_start_waits_until_the_ssh_acceptor_is_ready( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - listen_started = asyncio.Event() - allow_listen = asyncio.Event() - listen = asyncssh.listen - - async def delayed_listen(*args: Any, **kwargs: Any) -> asyncssh.SSHAcceptor: - listen_started.set() - await allow_listen.wait() - return await listen(*args, **kwargs) - - monkeypatch.setattr(asyncssh, "listen", delayed_listen) - ws = Workspace(tmp_path / "root", track_files=False) - start = asyncio.create_task(ws.start()) - await asyncio.wait_for(listen_started.wait(), 1.0) - - assert not start.done() - - allow_listen.set() - await start - try: - async with await _connect(ws) as conn: - assert (await conn.run("echo ready")).stdout == "ready\n" - finally: - await ws.stop() - - -@pytest.mark.asyncio -async def test_credentials_live_outside_the_served_root(tmp_path: Path) -> None: - """The agent's shell root must not contain its SSH key material.""" - ws = Workspace(tmp_path / "root") - await ws.start() - try: - key_path = ws.ssh_client_key_path - assert key_path is not None - assert not key_path.is_relative_to(ws.root) - assert not (ws.root / ".hud").exists() - # The daemon still works from the external credentials. - async with await _connect(ws) as conn: - result = await conn.run("echo ok") - stdout = result.stdout - assert isinstance(stdout, str) and "ok" in stdout - finally: - await ws.stop() - assert not key_path.exists() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("guest_path", ["/workspace", "/tmp/hud-coding/123/workspace"]) -async def test_shell_user_can_access_guest_workspace_paths(tmp_path: Path, guest_path: str) -> None: - if sys.platform != "linux" or os.geteuid() != 0 or workspace_mod.usable_bwrap() is None: - pytest.skip("requires Linux, root, and usable bubblewrap") - root = tmp_path / "root" - root.mkdir() - (root / "input.txt").write_text("hello\n") - ws = Workspace( - root, guest_path=guest_path, network=False, shell_uid=1000, require_isolation=True - ) - await ws.start() - try: - command = f"id -u; cat {guest_path}/input.txt; echo written > {guest_path}/output.txt" - async with await _connect(ws) as conn: - result = await conn.run(command) - assert result.exit_status == 0, result.stderr - assert result.stdout == "1000\nhello\n" - await ws.terminate_sessions() - captured = await ws.run(["sh", "-c", command], scope="environment") - assert captured.returncode == 0, captured.stderr - assert captured.stdout == b"1000\nhello\n" - assert (root / "output.txt").read_text() == "written\n" - finally: - await ws.stop() - - -@pytest.mark.asyncio -async def test_sftp_subsystem_is_not_served(tmp_path: Path) -> None: - ws = Workspace(tmp_path / "root") - await ws.start() - try: - async with await _connect(ws) as conn: - with pytest.raises(asyncssh.ChannelOpenError): - await conn.start_sftp_client() - finally: - await ws.stop() - - -@pytest.mark.asyncio -async def test_file_operations_use_the_exec_channel(tmp_path: Path) -> None: - ws = Workspace(tmp_path / "root") - await ws.start() - try: - async with await _connect(ws) as conn: - client = SSHClient(ws.capability(), conn) - await client.write_text("hello world.txt", "héllo\n") - assert await client.read_text("hello world.txt") == "héllo\n" - await client.write_text("empty.txt", "") - assert await client.read_text("empty.txt") == "" - assert await client.listdir(".") == ["empty.txt", "hello world.txt"] - # Absolute paths mean what they say in the session's namespace — - # never re-anchored under the workspace. - outside = tmp_path / "outside.txt" - await client.write_text(str(outside), "done") - assert outside.read_text() == "done" - assert await client.read_text(str(outside)) == "done" - assert not (tmp_path / "root" / str(outside).lstrip("/")).exists() - finally: - await ws.stop() - - -@pytest.mark.asyncio -async def test_output_arrives_while_the_command_is_still_running(tmp_path: Path) -> None: - """Held until exit, a long build tells the agent nothing while it runs and - a session that never exits says nothing at all.""" - ws = Workspace(tmp_path / "root") - await ws.start() - try: - async with await _connect(ws) as conn: - started = time.monotonic() - process = await conn.create_process("echo first; sleep 5; echo second") - first = await asyncio.wait_for(process.stdout.readline(), 10) - elapsed = time.monotonic() - started - process.channel.close() - finally: - await ws.stop() - - assert first.strip() == "first" - # Held until exit it would take the full sleep; the point is that it does not. - assert elapsed < 2.0, f"first line took {elapsed:.1f}s — output is not streaming" - - -@pytest.mark.parametrize( - ("command", "marker"), - [ - ( - "sh -c 'sleep .1; exec >/dev/null 2>&1; : > silent-done' & printf started", - "silent-done", - ), - ( - "sh -c \"sleep .1; trap '' PIPE; printf late-out || :; " - "printf late-err >&2 || :; exec >/dev/null 2>&1; " - ': > late-done" & printf started', - "late-done", - ), - ( - "cd . && nohup sh -c 'sleep .1' " - "stdin.log 2>&1 && exec >/dev/null 2>&1 && " - ": > stdin-done & printf started", - "stdin-done", - ), - ( - "(cd . && exec sh -c 'sleep .1; : > redirected-done') " - "redirected.log 2>&1 & printf started", - "redirected-done", - ), - ], - ids=("inherited-eof", "late-output", "stdin-detached", "fully-redirected"), -) -@pytest.mark.asyncio -async def test_background_completion_preserves_subsequent_namespace_commands( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - command: str, - marker: str, -) -> None: - shell = shutil.which("bash") or "sh" - async with _connected_namespace_host(monkeypatch) as namespace: - launched = await namespace.spawn( - [shell, "-c", command], - cwd=tmp_path, - env=dict(os.environ), - persistent=True, - ) - launch_result = await asyncio.wait_for(launched.complete(), 5.0) - - assert launch_result.returncode == 0 - assert launch_result.stdout == b"started" - - await asyncio.to_thread(_wait_for_path, tmp_path / marker) - await asyncio.sleep(0) - - probe = await namespace.spawn( - [shell, "-c", "printf healthy"], - cwd=tmp_path, - env=dict(os.environ), - persistent=True, - ) - probe_result = await asyncio.wait_for(probe.complete(), 5.0) - - assert probe_result.returncode == 0 - assert probe_result.stdout == b"healthy" - - -@pytest.mark.asyncio -async def test_namespace_process_forwards_standard_streams( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - shell = shutil.which("bash") or "sh" - async with _connected_namespace_host(monkeypatch) as namespace: - process = await namespace.spawn( - [ - shell, - "-c", - "IFS= read -r line; printf 'out:%s' \"$line\"; printf err >&2", - ], - cwd=tmp_path, - env=dict(os.environ), - persistent=True, - ) - process.stdin.write(b"input\n") - process.stdin.write_eof() - result = await asyncio.wait_for(process.complete(), 5.0) - - assert result.returncode == 0 - assert result.stdout == b"out:input" - assert result.stderr == b"err" - - -@pytest.mark.asyncio -async def test_a_session_that_asks_for_a_terminal_gets_one(tmp_path: Path) -> None: - """Programs branch on isatty: without a pty they take their batch path, so - a terminal task is graded on behaviour a terminal would never produce.""" - ws = Workspace(tmp_path / "root") - await ws.start() - try: - async with await _connect(ws) as conn: - with_pty = await conn.run( - "test -t 0 && test -t 1 && echo TTY || echo NOT_TTY; tput cols 2>/dev/null", - term_type="xterm-256color", - term_size=(120, 40), - check=False, - ) - without = await conn.run("test -t 1 && echo TTY || echo NOT_TTY", check=False) - finally: - await ws.stop() - - assert "TTY" in str(with_pty.stdout) and "NOT_TTY" not in str(with_pty.stdout) - # The size the client asked for reaches the terminal, not a default. - assert "120" in str(with_pty.stdout) - assert "NOT_TTY" in str(without.stdout) - - -@pytest.mark.asyncio -async def test_a_resize_does_not_cost_the_session_its_keyboard(tmp_path: Path) -> None: - """asyncssh delivers a resize as an exception on the stdin read, and it is - not an asyncssh.Error — unhandled it escapes the relay and input stops.""" - ws = Workspace(tmp_path / "root") - await ws.start() - try: - async with await _connect(ws) as conn: - process = await conn.create_process( - "cat", term_type="xterm-256color", term_size=(80, 24) - ) - process.channel.change_terminal_size(132, 43) - await asyncio.sleep(0.2) - # Input still reaches the shell after the resize. - process.stdin.write("still-listening\n") - echoed = await asyncio.wait_for(process.stdout.readline(), 5) - process.channel.close() - finally: - await ws.stop() - - assert "still-listening" in echoed - - -@pytest.mark.asyncio -async def test_session_setup_failure_is_reported_to_the_client( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root") - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(side_effect=RuntimeError("map failed"))) - await ws.start() - try: - async with await _connect(ws) as conn: - result = await conn.run("echo unreachable", check=False) - finally: - await ws.stop() - - assert result.exit_status == 1 - assert "workspace: cannot prepare shell: map failed" in str(result.stderr) - - -def _wall(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(Workspace, "_drops_privileges", lambda self: True) - monkeypatch.setattr(Workspace, "_setpriv", lambda self: "/usr/bin/setpriv") - - -@pytest.mark.asyncio -async def test_dropped_session_env_excludes_server_secrets( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A dropped shell must not inherit the server's environment (secrets).""" - _wall(monkeypatch) - monkeypatch.setenv("HUD_API_KEY", "super-secret") - - ws = Workspace(tmp_path / "root", shell_uid=1000, env={"CUSTOM": "1"}) - session_env = ws._session_env() - assert session_env is not None - assert "HUD_API_KEY" not in session_env - assert session_env["CUSTOM"] == "1" - assert "PATH" in session_env - # The server's HOME (/root) is unreadable by the dropped uid. - assert session_env["HOME"] == ws._guest_path - - -def _sandbox_env(argv: list[str]) -> dict[str, str]: - """The environment the sandboxed payload starts from (its ``env -i`` set).""" - assignments = argv[argv.index("-i") + 1 :] - return dict(item.split("=", 1) for item in itertools.takewhile(lambda a: "=" in a, assignments)) - - -def test_bwrap_drops_host_env_when_walled(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """The bwrap path must not re-inject host secrets, while per-call env - overrides still reach the sandbox.""" - monkeypatch.setenv("HUD_API_KEY", "super-secret") - _wall(monkeypatch) - - ws = Workspace(tmp_path / "root", shell_uid=1000, env={"CUSTOM": "1"}) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - argv = ws.shell_argv("echo hi", env={"PER_CALL": "1"}) - - sandbox_env = _sandbox_env(argv) - assert "HUD_API_KEY" not in sandbox_env - assert sandbox_env["CUSTOM"] == "1" - assert "PATH" in sandbox_env - assert sandbox_env["PER_CALL"] == "1" - - -@pytest.mark.parametrize("tmpfs_modes", [True, False]) -def test_sandbox_tmp_is_world_writable_where_bwrap_can_set_modes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, tmpfs_modes: bool -) -> None: - ws = Workspace(tmp_path / "root", shell_uid=1000) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap", tmpfs_modes=tmpfs_modes)) - - argv = ws.bwrap_argv(["true"]) - - tmp = next(i for i in range(len(argv) - 1) if argv[i : i + 2] == ["--tmpfs", "/tmp"]) - if tmpfs_modes: - assert argv[tmp - 2 : tmp] == ["--perms", "1777"] - else: - assert "--perms" not in argv - - -def test_bwrap_inherits_host_env_when_not_walled( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("SENTINEL", "visible") - ws = Workspace(tmp_path / "root") - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - argv = ws.bwrap_argv(["bash", "-lc", "true"]) - assert _sandbox_env(argv)["SENTINEL"] == "visible" - - -def test_the_harness_own_configuration_never_reaches_a_session( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """HUD's variables in the serving process are a credential leak where they - hold a key, and a tell everywhere else — an agent that finds HUD_ anything - knows what is running it.""" - monkeypatch.setenv("HUD_API_KEY", "super-secret") - monkeypatch.setenv("HUD_SKIP_VERSION_CHECK", "1") - monkeypatch.setenv("ORDINARY", "kept") - # What the task itself declares is the task's, HUD-shaped name or not. - ws = Workspace(tmp_path / "root", env={"HUD_TASK_DECLARED": "mine"}) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - - for argv in ( - ws.shell_argv("echo hi"), - ws.bwrap_argv(["true"]), - ws.session_argv("echo hi"), - ): - session_env = _sandbox_env(argv) - assert "HUD_API_KEY" not in session_env - assert "HUD_SKIP_VERSION_CHECK" not in session_env - assert session_env["ORDINARY"] == "kept" - assert session_env["HUD_TASK_DECLARED"] == "mine" - - -#: Options bubblewrap gained after 0.4, the newest release on distros still in -#: use (debian bullseye ships 0.4.1). One of these in a session's argv aborts -#: every command on such a host with "Unknown option", which grades as a -#: legitimate zero rather than a broken environment. -_POST_0_4_BWRAP_OPTIONS = frozenset( - { - "--clearenv", # 0.5.0 - "--assert-userns-disabled", # 0.5.0 - "--overlay", # 0.8.0 - "--tmp-overlay", # 0.8.0 - "--ro-overlay", # 0.8.0 - "--overlay-src", # 0.8.0 - "--size", # 0.9.0 - "--chmod", # 0.9.0 - } -) - - -def test_session_argv_runs_on_bubblewrap_0_4( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Sessions must not pass an option an old-but-usable bwrap will reject.""" - ws = Workspace( - tmp_path / "root", - shell_uid=1000, - env={"CUSTOM": "1"}, - mounts=(Mount("tmpfs", dst="/tests"),), - ) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - _wall(monkeypatch) - - for argv in ( - ws.shell_argv("echo hi"), - ws.shell_argv(), - ws.bwrap_argv(["true"]), - ws.session_argv("echo hi"), - ): - assert not _POST_0_4_BWRAP_OPTIONS.intersection(argv) - - -def test_hosted_sessions_do_not_create_new_namespaces( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root", guest_path="/app", network=True) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - - first = ws.session_argv("start-a-server &") - second = ws.session_argv("curl localhost") - - assert first[0] == "/usr/bin/bwrap" and second[0] == "/usr/bin/bwrap" - for argv in (first, second): - assert "--unshare-user" not in argv - assert "--unshare-pid" not in argv - assert "--unshare-net" not in argv - assert "--die-with-parent" not in argv - dev = argv.index("--dev-bind") - assert argv[dev : dev + 3] == ["--dev-bind", "/dev", "/dev"] - assert argv.count("--cap-drop") == 3 - assert argv[argv.index("--chdir") + 1] == "/app" - - -@pytest.mark.asyncio -async def test_concurrent_sessions_share_one_sandbox( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """An agent issues parallel tool calls; if each started its own sandbox, - what one backgrounds would be invisible to the next.""" - ws = Workspace(tmp_path / "root") - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - spawned = 0 - - async def fake_spawn() -> int: - nonlocal spawned - spawned += 1 - await asyncio.sleep(0.01) # the real spawn awaits bwrap's readiness - ws._sandbox = cast("Any", SimpleNamespace(returncode=None)) - ws._sandbox_init = 4000 + spawned - return ws._sandbox_init - - monkeypatch.setattr(ws, "_start_sandbox", fake_spawn) - - pids = await asyncio.gather(*(ws.sandbox_pid() for _ in range(4))) - - assert spawned == 1 - assert set(pids) == {4001} - - -def test_the_sandbox_reports_readiness_before_sessions_join_it( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """bwrap names the child pid before that child has built its mount - namespace, so the pid alone is not proof the sandbox can run anything.""" - ws = Workspace(tmp_path / "root") - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - argv = ws.bwrap_argv(["sh", "-c", "echo ready"], info_fd=7) - - assert argv[argv.index("--info-fd") + 1] == "7" - # The signal comes from the payload, which bwrap runs only after setup. - assert argv[-1] == "echo ready" - - -def test_network_ownership_matches_bubblewrap_isolation( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - for allowed, network, owns in ( - ({"example.com"}, True, True), # a policy needs a network to apply to - (set(), True, True), # declared unreachable - (None, False, True), # no-network, however it was spelled - (None, True, False), # the substrate's network, as before - ): - ws = Workspace(tmp_path / "root", network=network, allowed_hosts=allowed) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - - assert ws.owns_netns is owns - assert ("--unshare-net" in ws.bwrap_argv(["true"])) is owns - - -@pytest.mark.asyncio -async def test_run_uses_fresh_mounts_and_shared_network_with_visitor_egress( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("PATH", "/task/bin:/usr/bin") - monkeypatch.setenv("HUD_API_KEY", "sk-secret") - ws = Workspace(tmp_path / "root", env={"AGENT_ONLY": "secret"}) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - - @contextlib.asynccontextmanager - async def visiting(allowed): - assert allowed == {"pypi.org"} - yield {"HTTPS_PROXY": "http://visitor"} - - complete = AsyncMock(return_value=ProcessResult(0, b"passed", b"")) - spawn = AsyncMock(return_value=SimpleNamespace(complete=complete)) - - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) - monkeypatch.setattr(ws, "visiting", visiting) - ws._namespace = cast("Any", SimpleNamespace(spawn=spawn)) - - result = await ws.run( - ["test.sh"], - env={"VERIFIER_ONLY": "yes"}, - identity=None, - inherit_workspace_env=False, - allowed_hosts={"pypi.org"}, - max_wait=12, - ) - - assert result.stdout == b"passed" - complete.assert_awaited_once_with(max_wait=12) - assert spawn.await_args is not None - (argv,), kwargs = spawn.await_args - assert argv[0] == "/usr/bin/bwrap" - assert "--unshare-user" not in argv - assert "--unshare-pid" not in argv - assert "HTTPS_PROXY=http://visitor" in argv - assert "PATH=/task/bin:/usr/bin" in argv - assert "VERIFIER_ONLY=yes" in argv - assert kwargs["mount_view"] == "host" - assert "AGENT_ONLY=secret" not in argv - assert not any(arg.startswith("HUD_API_KEY=") for arg in argv) - assert kwargs["env"]["HTTPS_PROXY"] == "http://visitor" - - -@pytest.mark.asyncio -async def test_run_uses_a_disposable_writable_hosts_file( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - source = tmp_path / "hosts" - source.write_text("127.0.0.1 localhost\n", encoding="utf-8") - ws = Workspace(tmp_path / "root") - ws._hosts_path = source - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) - - @contextlib.asynccontextmanager - async def visiting(allowed): - assert allowed == () - yield {} - - mounted: Path | None = None - - async def spawn(argv: list[str], **kwargs: Any) -> Any: - nonlocal mounted - del kwargs - hosts_index = argv.index("/etc/hosts") - assert argv[hosts_index - 2] == "--bind" - mounted = Path(argv[hosts_index - 1]) - assert mounted.read_text("utf-8") == source.read_text("utf-8") - assert mounted.parent.stat().st_mode & 0o777 == 0o711 - mounted.write_text("127.0.0.1 verifier-added\n", encoding="utf-8") - return SimpleNamespace(complete=AsyncMock(return_value=ProcessResult(0, b"passed", b""))) - - monkeypatch.setattr(ws, "visiting", visiting) - ws._namespace = cast("Any", SimpleNamespace(spawn=spawn)) - - result = await ws.run(["test.sh"], identity=None, writable_hosts=True) - - assert result.stdout == b"passed" - assert mounted is not None and not await asyncio.to_thread(mounted.exists) - assert not mounted.parent.exists() - assert await asyncio.to_thread(source.read_text, "utf-8") == "127.0.0.1 localhost\n" - - -def test_peer_forwarders_use_the_substrate_listen_backlog() -> None: - from hud.environment.egress import _UnixServer # unix-only, as this module is - - assert _UnixServer.request_queue_size == socket.SOMAXCONN - - -@pytest.mark.asyncio -async def test_visiting_none_uses_the_workspace_egress( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root") - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) - ws._egress = cast( - "Any", - SimpleNamespace(environment=Mock(return_value={"HTTPS_PROXY": "http://workspace"})), - ) - - async with ws.visiting(None) as environment: - assert environment == {"HTTPS_PROXY": "http://workspace"} - - ws.allowed_hosts = frozenset({"pypi.org"}) - async with ws.visiting({"pypi.org"}) as environment: - assert environment == {"HTTPS_PROXY": "http://workspace"} - - -@pytest.mark.asyncio -async def test_visiting_uses_the_reserved_workspace_bridge( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - credentials = Path(tempfile.mkdtemp(prefix="hud-visitor-", dir="/tmp")) - try: - ws = Workspace( - tmp_path / "root", - peers=[Peer("db", 5432)], - ports=[5432], - credentials_dir=credentials, - ) - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) - - async with ws.visiting({"pypi.org"}) as environment: - assert environment["HTTPS_PROXY"].endswith("127.0.0.1:3129") - assert "db" in environment["NO_PROXY"] - assert "127.0.0.2" in environment["NO_PROXY"] - assert (credentials / "visitor" / "egress.sock").is_socket() - - assert not (credentials / "visitor" / "egress.sock").exists() - finally: - shutil.rmtree(credentials) - - -@pytest.mark.asyncio -async def test_overlapping_runs_keep_their_visitor_policies_and_credentials( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - from hud.environment import egress as egress_mod - - credentials = Path(tempfile.mkdtemp(prefix="hud-visitors-", dir="/tmp")) - ws = Workspace(tmp_path / "root", credentials_dir=credentials) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) - - def unavailable(*_args: object, **_kwargs: object) -> socket.socket: - raise ConnectionRefusedError - - monkeypatch.setattr(egress_mod, "_connect_public", unavailable) - started = {host: asyncio.Event() for host in ("first.example", "second.example")} - release = {host: asyncio.Event() for host in started} - environments: dict[str, Mapping[str, str]] = {} - - class VisitorProcess: - def __init__(self, host: str, environment: Mapping[str, str]) -> None: - self.host = host - self.environment = environment - - async def complete(self, *, max_wait: float | None = None) -> ProcessResult: - environments[self.host] = self.environment - started[self.host].set() - await release[self.host].wait() - return ProcessResult(0, b"", b"") - - async def launch( - command: list[str], - *, - env: Mapping[str, str] | None = None, - **_kwargs: object, - ) -> VisitorProcess: - assert env is not None - return VisitorProcess(command[0], env) - - def proxy_response( - environment: Mapping[str, str], - host: str, - *, - credentials_from: Mapping[str, str] | None = None, - ) -> bytes: - proxy = urllib.parse.urlsplit((credentials_from or environment)["http_proxy"]) - assert proxy.username == "hud" and proxy.password is not None - token = urllib.parse.unquote(proxy.password) - authorization = base64.b64encode(f"hud:{token}".encode()).decode() - client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - client.settimeout(5) - client.connect(str(credentials / "visitor" / "egress.sock")) - try: - client.sendall( - ( - f"GET http://{host}/ HTTP/1.1\r\n" - f"Host: {host}\r\n" - f"Proxy-Authorization: Basic {authorization}\r\n\r\n" - ).encode() - ) - return client.recv(4096) - finally: - client.close() - - monkeypatch.setattr(ws, "launch", launch) - tasks = [ - asyncio.create_task( - ws.run(["first.example"], allowed_hosts={"first.example"}, identity=None) - ) - ] - try: - await asyncio.wait_for(started["first.example"].wait(), 1) - tasks.append( - asyncio.create_task( - ws.run(["second.example"], allowed_hosts={"second.example"}, identity=None) - ) - ) - await asyncio.sleep(0) - assert not started["second.example"].is_set() - - first = environments["first.example"] - assert b"502" in proxy_response(first, "first.example") - assert b"403" in proxy_response(first, "second.example") - release["first.example"].set() - assert (await tasks[0]).returncode == 0 - - await asyncio.wait_for(started["second.example"].wait(), 1) - second = environments["second.example"] - assert first["http_proxy"] != second["http_proxy"] - assert b"502" in proxy_response(second, "second.example") - assert b"403" in proxy_response(second, "first.example") - assert b"407" in proxy_response(second, "second.example", credentials_from=first) - finally: - for event in release.values(): - event.set() - await asyncio.gather(*tasks, return_exceptions=True) - shutil.rmtree(credentials, ignore_errors=True) - - -@pytest.mark.asyncio -async def test_staged_verifier_is_launched_by_the_namespace_host( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root") - monkeypatch.setattr( - ws, - "_bwrap", - Bubblewrap("/usr/bin/bwrap", pid_unshare="/usr/bin/unshare"), - ) - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=121)) - complete = AsyncMock(return_value=ProcessResult(0, b"passed", b"")) - spawn = AsyncMock(return_value=SimpleNamespace(complete=complete)) - ws._namespace = cast("Any", SimpleNamespace(spawn=spawn)) - - await ws.run(["test.sh"], identity=None) - - assert spawn.await_args is not None - (argv,), kwargs = spawn.await_args - assert argv[0] == "/usr/bin/bwrap" - assert "CAP_SYS_ADMIN" in argv - assert "/usr/bin/unshare" not in argv - assert kwargs["mount_view"] == "host" - - -@pytest.mark.asyncio -async def test_launch_can_keep_infrastructure_outside_workload_sessions( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root") - _wall(monkeypatch) - monkeypatch.setattr(workspace_mod, "_is_root", lambda: True) - monkeypatch.setattr(workspace_mod.sys, "platform", "linux") - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=121)) - launched = SimpleNamespace(returncode=None) - spawn = AsyncMock(return_value=launched) - ws._namespace = cast("Any", SimpleNamespace(spawn=spawn)) - - process = await ws.launch( - ["start-environment", "sleep", "infinity"], - cwd="/app", - identity=(1000, 2000), - no_new_privs=False, - persistent=True, - scope="environment", - ) - - assert process is launched - assert spawn.await_args is not None - (argv,), kwargs = spawn.await_args - assert argv[0] == "/usr/bin/bwrap" - assert argv[argv.index("--uid") + 1] == "1000" - assert argv[argv.index("--gid") + 1] == "2000" - assert "--unshare-user-try" in argv - assert "--dev-bind" in argv - assert "--reuid" not in argv - assert argv[argv.index("--chdir") + 1] == "/app" - assert kwargs["mount_view"] == "host" - assert kwargs["identity"] == (1000, 2000) - assert kwargs["persistent"] is True - assert kwargs["scope"] == "environment" - - -@pytest.mark.asyncio -async def test_terminate_sessions_preserves_the_namespace_host(tmp_path: Path) -> None: - ws = Workspace(tmp_path / "root") - terminate_sessions = AsyncMock() - ws._namespace = cast("Any", SimpleNamespace(terminate_sessions=terminate_sessions)) - - await ws.terminate_sessions() - - terminate_sessions.assert_awaited_once_with() - assert ws._namespace is not None - - -@pytest.mark.asyncio -async def test_namespace_management_does_not_share_process_connections( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - host = namespace_mod.NamespaceHost(tmp_path / "namespace.sock") - process_connection = SimpleNamespace(run=AsyncMock()) - host._connection = cast("Any", process_connection) - management = SimpleNamespace( - run=AsyncMock(return_value=SimpleNamespace(returncode=0)), - close=Mock(), - wait_closed=AsyncMock(), - ) - open_connection = AsyncMock(return_value=management) - monkeypatch.setattr(host, "_open_connection", open_connection) - - await host.terminate_sessions() - - open_connection.assert_awaited_once_with() - process_connection.run.assert_not_awaited() - management.close.assert_called_once_with() - management.wait_closed.assert_awaited_once_with() - - -def test_process_guard_executes_projected_file_without_module_reentry() -> None: - guard = cast( - "Any", - SimpleNamespace( - backend="ptrace", - sandbox_helper="/tmp/.hud-process-connection/process_guard.py", - sandbox_socket="/tmp/.hud-process-connection/control.sock", - ), - ) - argv = workspace_mod._guarded_process_argv( - "/usr/bin/python3", - guard, - ["bash", "-lc", "true"], - ) - - assert argv == [ - "/usr/bin/python3", - "/tmp/.hud-process-connection/process_guard.py", - "--backend", - "ptrace", - "/tmp/.hud-process-connection/control.sock", - "--", - "bash", - "-lc", - "true", - ] - result = subprocess.run( - [sys.executable, str(Path(workspace_mod.__file__).with_name("process_guard.py")), "--help"], - check=False, - capture_output=True, - text=True, - ) - assert result.returncode == 0 - assert result.stderr == "" - - -def test_process_guard_selects_probed_ptrace_backend(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(process_guard_mod, "_backend", None) - monkeypatch.setattr(process_guard_mod, "_backend_probed", False) - monkeypatch.setattr(process_guard_mod.sys, "platform", "linux") - run = Mock(return_value=SimpleNamespace(returncode=0, stdout="ptrace\n")) - monkeypatch.setattr(process_guard_mod.subprocess, "run", run) - - assert process_guard_mod._detected_backend() == "ptrace" - assert process_guard_mod._detected_backend() == "ptrace" - run.assert_called_once() - - -@pytest.mark.asyncio -async def test_namespace_host_terminates_session_processes_without_the_holder( - tmp_path: Path, -) -> None: - host = namespace_mod._NamespaceHost( - tmp_path / "namespace.sock", - setup_loopback=False, - holder_argv=[], - bwrap="bwrap", - launcher_depth=0, - map_identities=False, - ports=frozenset(), - ) - holder = AsyncMock() - host.holders["session"] = (holder, 7) - first = AsyncMock() - second = AsyncMock() - host.sessions.extend((first, second)) - - await host._terminate_sessions() - - first.terminate.assert_awaited_once_with() - second.terminate.assert_awaited_once_with() - holder.terminate.assert_not_awaited() - assert host.sessions == [] - - -@pytest.mark.asyncio -async def test_run_can_use_a_fresh_no_network_sandbox( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root") - monkeypatch.setattr( - ws, - "_bwrap", - Bubblewrap("/usr/bin/bwrap", pid_unshare="/usr/bin/unshare"), - ) - install_identity_map = AsyncMock(return_value=9) - complete = AsyncMock(return_value=ProcessResult(0, b"isolated", b"")) - spawn = AsyncMock(return_value=SimpleNamespace(complete=complete)) - - monkeypatch.setattr(workspace_mod, "install_identity_map", install_identity_map) - monkeypatch.setattr(workspace_mod, "create_process_group_exec", spawn) - - result = await ws.run( - ["test.sh"], - isolated=True, - identity=None, - max_wait=5, - ) - - assert result.stdout == b"isolated" - complete.assert_awaited_once_with(max_wait=5) - install_identity_map.assert_awaited_once() - assert spawn.await_args is not None - argv, kwargs = spawn.await_args - assert argv[0] == "/usr/bin/bwrap" - assert "/usr/bin/unshare" not in argv - assert "--unshare-net" in argv - assert len(kwargs["pass_fds"]) == 2 - - -@pytest.mark.asyncio -async def test_an_isolated_command_keeps_the_image_environment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A command must not lose the image's PATH — the interpreters and tools - the task installed — merely because it asked for an isolated sandbox. - Both branches of run() give it the serving environment, less HUD's own.""" - monkeypatch.setenv("PATH", "/task/bin:/usr/bin") - monkeypatch.setenv("HUD_API_KEY", "sk-secret") - ws = Workspace(tmp_path / "root") - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - monkeypatch.setattr(workspace_mod, "install_identity_map", AsyncMock(return_value=9)) - complete = AsyncMock(return_value=ProcessResult(0, b"", b"")) - spawn = AsyncMock(return_value=SimpleNamespace(complete=complete)) - monkeypatch.setattr(workspace_mod, "create_process_group_exec", spawn) - - await ws.run(["test.sh"], isolated=True, identity=None) - - assert spawn.await_args is not None - argv, _ = spawn.await_args - assert "PATH=/task/bin:/usr/bin" in argv - assert not any(arg.startswith("HUD_API_KEY=") for arg in argv) - - -@pytest.mark.asyncio -async def test_failed_isolated_run_terminates_its_sandbox( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root") - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - process = SimpleNamespace(terminate=AsyncMock(), complete=AsyncMock()) - monkeypatch.setattr( - workspace_mod, - "create_process_group_exec", - AsyncMock(return_value=process), - ) - monkeypatch.setattr( - workspace_mod, - "install_identity_map", - AsyncMock(side_effect=RuntimeError("map failed")), - ) - - with pytest.raises(RuntimeError, match="map failed"): - await ws.run(["test.sh"], isolated=True, identity=None) - - process.terminate.assert_awaited_once() - process.complete.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_failed_sandbox_start_discards_the_holder( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root", network=True) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - failed_holder = SimpleNamespace( - returncode=None, - stdout=SimpleNamespace(readline=AsyncMock(return_value=b"ready\n")), - stderr=SimpleNamespace(read=AsyncMock(return_value=b"")), - kill=Mock(), - wait=AsyncMock(return_value=0), - ) - monkeypatch.setattr( - asyncio, - "create_subprocess_exec", - AsyncMock(return_value=failed_holder), - ) - monkeypatch.setattr( - workspace_mod, - "install_identity_map", - AsyncMock(side_effect=RuntimeError("map failed")), - ) - - with pytest.raises(RuntimeError, match="map failed"): - await ws.sandbox_pid() - - failed_holder.kill.assert_called_once() - - -@pytest.mark.asyncio -async def test_staged_sandbox_hosts_the_namespace_server_outside_bwrap( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root", network=False) - monkeypatch.setattr( - ws, - "_bwrap", - Bubblewrap("/usr/bin/bwrap", pid_unshare="/usr/bin/unshare"), - ) - stdin = SimpleNamespace(write=Mock(), drain=AsyncMock(), close=Mock()) - sandbox = SimpleNamespace( - returncode=None, - stdin=stdin, - stdout=SimpleNamespace(readline=AsyncMock(return_value=b'{"holder_pid": 121}\n')), - stderr=SimpleNamespace(read=AsyncMock(return_value=b"")), - ) - spawn = AsyncMock(return_value=sandbox) - monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn) - bridge = SimpleNamespace( - stdin=SimpleNamespace(write=Mock(), drain=AsyncMock()), - stdout=SimpleNamespace(readline=AsyncMock(return_value=b"ready\n")), - stderr=SimpleNamespace(read=AsyncMock(return_value=b"")), - ) - namespace = SimpleNamespace( - connect=AsyncMock(), - forward=AsyncMock(), - spawn=AsyncMock(return_value=bridge), - ) - monkeypatch.setattr(workspace_mod, "NamespaceHost", Mock(return_value=namespace)) - - assert await ws.sandbox_pid() == 121 - - assert spawn.await_args is not None - argv, _ = spawn.await_args - assert argv[:2] == ("/usr/bin/unshare", "--net") - assert "/usr/bin/bwrap" not in argv - config = json.loads(stdin.write.call_args.args[0]) - assert config["holder_argv"][:5] == [ - "/usr/bin/unshare", - "--kill-child=KILL", - "--pid", - "--mount-proc", - "/usr/bin/bwrap", - ] - holder_dev = config["holder_argv"].index("--dev-bind") - assert config["holder_argv"][holder_dev : holder_dev + 3] == [ - "--dev-bind", - "/dev", - "/dev", - ] - assert config["map_identities"] is True - assert config["launcher_depth"] == 2 - - -def test_a_peer_answers_at_the_address_the_task_expects() -> None: - """A task that names a service says where it expects to find it. Placed - anywhere else, the task's own client configuration points at nothing.""" - from hud.environment.egress import Peer, bind_addresses - - # One peer per port is the ordinary case: it is at localhost, which is - # what a task saying "localhost:5432" or "http://localhost:8080" means. - single = bind_addresses([Peer("db", 5432), Peer("api", 8080)]) - assert single == {"db": "127.0.0.1", "api": "127.0.0.1"} - - # Two services cannot both hold one port there, so the second moves — and - # is still reached by its name, which is how a task addresses two anyway. - both = bind_addresses([Peer("primary", 5432), Peer("replica", 5432)]) - assert both == {"primary": "127.0.0.1", "replica": "127.0.0.2"} - - reserved = bind_addresses([Peer("api", 8080)], reserved_ports={8080}) - assert reserved == {"api": "127.0.0.2"} - - proxies = bind_addresses([Peer("agent", 3128), Peer("verifier", 3129)]) - assert proxies == {"agent": "127.0.0.2", "verifier": "127.0.0.2"} - - service = bind_addresses([Peer("db", 5432), Peer("db", 6379)]) - assert service == {"db": "127.0.0.1"} - - service_with_reserved_port = bind_addresses( - [Peer("db", 5432), Peer("db", 6379)], - reserved_ports={5432}, - ) - assert service_with_reserved_port == {"db": "127.0.0.2"} - - with pytest.raises(ValueError, match="declares port 5432 twice"): - bind_addresses([Peer("db", 5432), Peer("db", 5432)]) - - -async def test_workspace_route_is_bound_once_and_removed_on_stop(tmp_path: Path) -> None: - from hud.environment import Environment - - env = Environment() - workspace = env.workspace(tmp_path / "root", track_files=False) - workspace._bwrap = cast("Any", object()) - env._started = True - route = WorkspaceRoute("ssh", "inference.hud.so", 443) - - env.bind_workspace_routes([route, route], session="a") - env.bind_workspace_routes([route], session="a") - - assert workspace.peers == (Peer("inference.hud.so", 443, target=("inference.hud.so", 443)),) - await env.stop() - assert workspace.peers == () - - -def test_workspace_route_rejects_ip_literals() -> None: - with pytest.raises(ValueError, match="hostname"): - WorkspaceRoute("ssh", "127.0.0.1", 443) - - -def test_workspace_names_are_added_to_the_substrates_hosts_rather_than_replacing_it() -> None: - """Dropping the substrate's entries would cost the workspace localhost.""" - from hud.environment.egress import Peer, hosts_text - - text = hosts_text( - [Peer("db", 5432)], - "127.0.0.1\tlocalhost\n::1\tip6-localhost\n", - local_aliases=["main"], - ) - - assert "127.0.0.1\tlocalhost" in text - assert "::1\tip6-localhost" in text - assert "127.0.0.1\tmain" in text - assert text.endswith("127.0.0.1\tdb\n") - - same_service = hosts_text( - [Peer("db", 5432), Peer("db", 6379)], - "", - ) - assert same_service == "127.0.0.1\tdb\n" - - -@pytest.mark.asyncio -async def test_a_declared_peer_is_a_name_sessions_resolve( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A peer reached only by port is not at the address the task expects, so - the workspace carries its own hosts file — and only when it has a network - of its own, since otherwise the service is already at its real address.""" - from hud.environment.egress import Peer - - ws = Workspace( - tmp_path / "root", - peers=[Peer("db", 5432)], - local_aliases=["main"], - ports=[5432], - allowed_hosts={"pypi.org"}, - credentials_dir=tmp_path / "protected" / "session-keys", - hosts_path=tmp_path / "runtime" / "hosts", - ) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - ws._prepare_runtime() - - argv = ws.bwrap_argv(["true"]) - hosts = Path(argv[argv.index("/etc/hosts") - 1]) - assert hosts == tmp_path / "runtime" / "hosts" - assert argv[argv.index("/etc/hosts") - 2] == "--ro-bind" - hosts_content = await asyncio.to_thread(hosts.read_text) - assert "127.0.0.1\tmain\n" in hosts_content - assert "127.0.0.2\tdb\n" in hosts_content - # Bound over /etc/hosts, so every session reads it whatever its identity. - assert (await asyncio.to_thread(hosts.stat)).st_mode & 0o044 - - sharing = Workspace(tmp_path / "shared", peers=[Peer("db", 5432)], network=True) - monkeypatch.setattr(sharing, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - sharing._prepare_runtime() - assert "/etc/hosts" not in sharing.bwrap_argv(["true"]) - - await ws.stop() - await sharing.stop() - - -def test_private_workspace_preserves_localhost_without_peers( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - hosts = tmp_path / "hosts" - ws = Workspace( - tmp_path / "root", - allowed_hosts=set(), - hosts_path=hosts, - ) - monkeypatch.setattr(ws, "_bwrap", Bubblewrap("/usr/bin/bwrap")) - - ws._prepare_runtime() - - assert "/etc/hosts" in ws.bwrap_argv(["true"]) - assert "localhost" in hosts.read_text("utf-8") - - -def test_a_peer_is_reached_directly_rather_than_through_the_proxy() -> None: - """The proxy resolves names out on the substrate, where a peer's name - means nothing and its address is something else entirely.""" - from hud.environment.egress import Egress, Peer - - egress = Egress( - "/tmp/unused", - {"pypi.org"}, - [Peer("db", 5432), Peer("replica", 5432)], - local_aliases=["main"], - ) - bypass = egress.environment()["no_proxy"].split(",") - - assert "main" in bypass - assert "db" in bypass and "replica" in bypass - assert "127.0.0.1" in bypass and "127.0.0.2" in bypass - - -def test_bridge_socket_paths_are_configuration_not_process_arguments() -> None: - from hud.environment.egress import Egress, Peer - - egress = Egress( - "/media/hud/session-keys", - {"pypi.org"}, - [Peer("db", 5432)], - reserved_ports={5432}, - ) - - visitor = Path("/media/hud/session-keys/visitor/egress.sock") - argv, config = egress.bridge_command(visitor_socket=visitor) - - assert "/media/hud/session-keys" not in " ".join(argv) - assert "/media/hud/session-keys" in config.decode() - routes = json.loads(config) - assert ["127.0.0.1", 3129, str(visitor)] in routes - assert ["127.0.0.2", 5432, "/media/hud/session-keys/peer-0.sock"] in routes - - -def test_bridge_ignores_workspace_imports_and_closes_reset_connections(tmp_path: Path) -> None: - from hud.environment.egress import Egress - - workspace = tmp_path / "workspace" - package = workspace / "hud" / "environment" - package.mkdir(parents=True) - (workspace / "hud" / "__init__.py").write_text("") - (package / "__init__.py").write_text("") - marker = workspace / "imported" - (package / "utils.py").write_text( - f"from pathlib import Path\nPath({str(marker)!r}).write_text('imported')\n" - ) - - sockets = Path(tempfile.mkdtemp(dir="/tmp")) - egress = Egress(sockets, {"example.com"}) - upstream_listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - upstream_listener.bind(str(egress.socket_path)) - upstream_listener.listen() - upstream_listener.settimeout(5) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as port_holder: - port_holder.bind(("127.0.0.1", 0)) - port = port_holder.getsockname()[1] - argv, config = egress.bridge_command(port=port) - process = subprocess.Popen( - argv, - cwd=workspace, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - client: socket.socket | None = None - upstream: socket.socket | None = None - try: - assert process.stdin is not None and process.stdout is not None - process.stdin.write(config) - process.stdin.flush() - assert process.stdout.readline() == b"ready\n" - assert not marker.exists() - - client = socket.create_connection(("127.0.0.1", port), timeout=5) - upstream, _ = upstream_listener.accept() - upstream.settimeout(1) - client.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) - client.close() - client = None - - assert upstream.recv(1) == b"" - finally: - if client is not None: - client.close() - if upstream is not None: - upstream.close() - upstream_listener.close() - if process.poll() is None: - process.terminate() - process.wait(timeout=5) - shutil.rmtree(sockets, ignore_errors=True) - - -def test_bridge_drains_reverse_direction_after_half_close(tmp_path: Path) -> None: - from hud.environment.egress import Egress - - sockets = Path(tempfile.mkdtemp(dir="/tmp")) - egress = Egress(sockets, {"example.com"}) - upstream_listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - upstream_listener.bind(str(egress.socket_path)) - upstream_listener.listen() - upstream_listener.settimeout(5) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as port_holder: - port_holder.bind(("127.0.0.1", 0)) - port = port_holder.getsockname()[1] - argv, config = egress.bridge_command(port=port) - process = subprocess.Popen( - argv, - cwd=tmp_path, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - client: socket.socket | None = None - upstream: socket.socket | None = None - try: - assert process.stdin is not None and process.stdout is not None - process.stdin.write(config) - process.stdin.flush() - assert process.stdout.readline() == b"ready\n" - - client = socket.create_connection(("127.0.0.1", port), timeout=5) - client.settimeout(5) - upstream, _ = upstream_listener.accept() - upstream.settimeout(5) - - request = b"complete request" - client.sendall(request) - client.shutdown(socket.SHUT_WR) - assert upstream.recv(65536) == request - assert upstream.recv(1) == b"" - - response = b"complete response" * 4096 - upstream.sendall(response) - upstream.shutdown(socket.SHUT_WR) - received = bytearray() - while chunk := client.recv(65536): - received.extend(chunk) - assert received == response - finally: - if client is not None: - client.close() - if upstream is not None: - upstream.close() - upstream_listener.close() - if process.poll() is None: - process.terminate() - process.wait(timeout=5) - shutil.rmtree(sockets, ignore_errors=True) - - -def _half_close_backend( - response: bytes, -) -> tuple[socket.socket, threading.Thread, int, list[bytes]]: - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.bind(("127.0.0.1", 0)) - listener.listen() - listener.settimeout(5) - port = listener.getsockname()[1] - requests: list[bytes] = [] - - def serve() -> None: - with listener: - connection, _ = listener.accept() - with connection: - chunks: list[bytes] = [] - while chunk := connection.recv(65536): - chunks.append(chunk) - requests.append(b"".join(chunks)) - connection.sendall(response) - - thread = threading.Thread(target=serve) - thread.start() - return listener, thread, port, requests - - -def test_relay_drains_reverse_direction_when_write_shutdown_fails() -> None: - from hud.environment.egress import _relay - - class ShutdownErrorSocket(socket.socket): - def shutdown(self, how: int) -> None: - raise OSError("injected shutdown failure") - - one, one_peer = socket.socketpair() - other_socket, other_peer = socket.socketpair() - other = ShutdownErrorSocket(fileno=other_socket.detach()) - one_peer.settimeout(5) - relay_thread = threading.Thread(target=_relay, args=(one, other, 1.0)) - relay_thread.start() - try: - one_peer.shutdown(socket.SHUT_WR) - response = b"complete response" * 64 - other_peer.sendall(response) - other_peer.shutdown(socket.SHUT_WR) - - received = bytearray() - while len(received) < len(response): - received.extend(one_peer.recv(65536)) - assert received == response - finally: - one.close() - one_peer.close() - other.close() - other_peer.close() - relay_thread.join(5) - assert not relay_thread.is_alive() - - -def test_connect_tunnel_drains_response_after_client_half_close( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.environment import egress as egress_mod - from hud.environment.egress import Egress - - response = b"complete response" * 32768 - listener, thread, port, requests = _half_close_backend(response) - - def connect(_host: str, _port: int, timeout: float) -> socket.socket: - return socket.create_connection(("127.0.0.1", port), timeout) - - monkeypatch.setattr(egress_mod, "_connect_public", connect) - sockets = Path(tempfile.mkdtemp(dir="/tmp")) - egress = Egress(sockets, {"example.com"}) - egress.start() - try: - client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - client.settimeout(5) - client.connect(str(egress.socket_path)) - client.sendall(b"CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n") - established = b"" - while b"\r\n\r\n" not in established: - established += client.recv(4096) - assert b"200 Connection established" in established - - request = b"request body" - client.sendall(request) - client.shutdown(socket.SHUT_WR) - received = b"" - while chunk := client.recv(65536): - received += chunk - client.close() - finally: - egress.stop() - listener.close() - shutil.rmtree(sockets, ignore_errors=True) - - thread.join(5) - assert not thread.is_alive() - assert requests == [request] - assert received == response - - -def test_peer_forward_drains_response_after_client_half_close() -> None: - from hud.environment.egress import Egress - - response = b"complete response" * 32768 - listener, thread, port, requests = _half_close_backend(response) - sockets = Path(tempfile.mkdtemp(dir="/tmp")) - egress = Egress(sockets, set(), [Peer("service", port)]) - egress.start() - try: - client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - client.settimeout(5) - client.connect(str(sockets / "peer-0.sock")) - request = b"request body" - client.sendall(request) - client.shutdown(socket.SHUT_WR) - received = b"" - while chunk := client.recv(65536): - received += chunk - client.close() - finally: - egress.stop() - listener.close() - shutil.rmtree(sockets, ignore_errors=True) - - thread.join(5) - assert not thread.is_alive() - assert requests == [request] - assert received == response - - -def test_the_proxy_normalizes_http_framing_in_both_directions( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import socket as socket_mod - import threading - from http.server import BaseHTTPRequestHandler, HTTPServer - - from hud.environment import egress as egress_mod - from hud.environment.egress import Egress - - class Chunked(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - request_body = b"" - request_host: str | None = None - request_length: str | None = None - request_transfer: str | None = None - - def do_POST(self) -> None: - type(self).request_host = self.headers.get("Host") - type(self).request_length = self.headers.get("Content-Length") - type(self).request_transfer = self.headers.get("Transfer-Encoding") - type(self).request_body = self.rfile.read(int(type(self).request_length or 0)) - self.send_response(200) - self.send_header("Transfer-Encoding", "chunked") - self.end_headers() - self.wfile.write(b"5\r\nhello\r\n0\r\n\r\n") - - def log_message(self, format: str, *args: Any) -> None: - pass - - upstream = HTTPServer(("127.0.0.1", 0), Chunked) - threading.Thread(target=upstream.serve_forever, daemon=True).start() - port = upstream.server_address[1] - monkeypatch.setattr( - egress_mod, - "_connect_public", - lambda host, port, timeout: socket_mod.create_connection((host, port), timeout), - ) - # Not the pytest tmp dir: a unix socket path is capped near 104 bytes. - sockets = Path(tempfile.mkdtemp(dir="/tmp")) - egress = Egress(sockets, {"127.0.0.1"}) - egress.start() - try: - client = socket_mod.socket(socket_mod.AF_UNIX, socket_mod.SOCK_STREAM) - client.settimeout(10) - client.connect(str(egress.socket_path)) - client.sendall( - ( - f"POST http://127.0.0.1:{port}/ HTTP/1.1\r\n" - "Host: forbidden.example\r\n" - "Transfer-Encoding: chunked\r\n\r\n" - "5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n" - ).encode() - ) - received = b"" - while b"hello" not in received: - chunk = client.recv(4096) - if not chunk: - break - received += chunk - client.close() - finally: - egress.stop() - upstream.shutdown() - upstream.server_close() - shutil.rmtree(sockets, ignore_errors=True) - - headers, _, body = received.partition(b"\r\n\r\n") - assert b"200" in headers - assert b"transfer-encoding" not in headers.lower() - assert body == b"hello" - assert Chunked.request_body == b"hello world" - assert Chunked.request_host == f"127.0.0.1:{port}" - assert Chunked.request_length == "11" - assert Chunked.request_transfer is None - - -def test_connection_relay_replaces_credentials_and_preserves_streaming() -> None: - import http.client - from http.server import BaseHTTPRequestHandler, HTTPServer - - class Upstream(BaseHTTPRequestHandler): - authorization: str | None = None - api_key: str | None = None - body = b"" - - def do_POST(self) -> None: - type(self).authorization = self.headers.get("Authorization") - type(self).api_key = self.headers.get("X-Api-Key") - type(self).body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) - self.send_response(200) - self.send_header("Transfer-Encoding", "chunked") - self.end_headers() - self.wfile.write(b"5\r\nfirst\r\n6\r\nsecond\r\n0\r\n\r\n") - - def log_message(self, format: str, *args: Any) -> None: - pass - - upstream = HTTPServer(("127.0.0.1", 0), Upstream) - upstream_thread = threading.Thread(target=upstream.serve_forever, daemon=True) - upstream_thread.start() - token = "scoped-runtime-token" - connection = Connection( - name="inference", - capability="ssh", - url=f"http://127.0.0.1:{upstream.server_address[1]}/v1", - headers={"Authorization": f"Bearer {token}", "X-Api-Key": token}, - ) - relay = ConnectionRelay(connection) - relay.start() - try: - client = http.client.HTTPConnection("127.0.0.1", relay.port, timeout=5) - client.request( - "POST", - "/v1/messages", - body=b"request", - headers={ - "Authorization": "Bearer model-visible", - "X-Api-Key": "model-visible", - }, - ) - response = client.getresponse() - assert response.status == 200 - assert response.read() == b"firstsecond" - client.close() - finally: - relay.stop() - upstream.shutdown() - upstream.server_close() - upstream_thread.join() - - assert Upstream.authorization == f"Bearer {token}" - assert Upstream.api_key == token - assert Upstream.body == b"request" - - -def test_connection_relay_refuses_paths_that_could_leave_its_prefix() -> None: - import http.client - from http.server import BaseHTTPRequestHandler, HTTPServer - - forwarded: list[str] = [] - - class Upstream(BaseHTTPRequestHandler): - def do_GET(self) -> None: - forwarded.append(self.path) - self.send_response(204) - self.end_headers() - - def log_message(self, format: str, *args: Any) -> None: - pass - - upstream = HTTPServer(("127.0.0.1", 0), Upstream) - upstream_thread = threading.Thread(target=upstream.serve_forever, daemon=True) - upstream_thread.start() - connection = Connection( - name="inference", - capability="ssh", - url=f"http://127.0.0.1:{upstream.server_address[1]}/v1", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - relay = ConnectionRelay(connection) - relay.start() - refused: dict[str, tuple[int, str | None]] = {} - try: - for path in ( - "/v1/../admin", - "/v1/./models", - "/v1/%2e%2e/admin", - "/v1/%2E%2E/admin", - "/v1/..%2fadmin", - "/v1/..%5cadmin", - "/v1/models", - ): - client = http.client.HTTPConnection("127.0.0.1", relay.port, timeout=5) - client.request("GET", path) - response = client.getresponse() - response.read() - refused[path] = (response.status, response.getheader("X-Connection-Error")) - client.close() - finally: - relay.stop() - upstream.shutdown() - upstream.server_close() - upstream_thread.join() - - assert forwarded == ["/v1/models"] - assert refused.pop("/v1/models") == (204, None) - assert set(refused.values()) == {(400, "non-canonical-path")} - - -@pytest.mark.parametrize("placeholder", ["X-Api-Key", "Api-Key", "X-Goog-Api-Key"]) -def test_connection_relay_drops_process_credentials_the_connection_does_not_set( - placeholder: str, -) -> None: - import http.client - from http.server import BaseHTTPRequestHandler, HTTPServer - - received: dict[str, str] = {} - - class Upstream(BaseHTTPRequestHandler): - def do_POST(self) -> None: - received.update({key.casefold(): value for key, value in self.headers.items()}) - self.send_response(204) - self.end_headers() - - def log_message(self, format: str, *args: Any) -> None: - pass - - upstream = HTTPServer(("127.0.0.1", 0), Upstream) - upstream_thread = threading.Thread(target=upstream.serve_forever, daemon=True) - upstream_thread.start() - connection = Connection( - name="inference", - capability="ssh", - url=f"http://127.0.0.1:{upstream.server_address[1]}", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - relay = ConnectionRelay(connection) - relay.start() - try: - client = http.client.HTTPConnection("127.0.0.1", relay.port, timeout=5) - client.request("POST", "/v1/messages", body=b"{}", headers={placeholder: "model-visible"}) - assert client.getresponse().status == 204 - client.close() - finally: - relay.stop() - upstream.shutdown() - upstream.server_close() - upstream_thread.join() - - assert received["authorization"] == "Bearer scoped-runtime-token" - assert placeholder.casefold() not in received - assert "model-visible" not in received.values() - - -@pytest.mark.parametrize( - "payload", - [ - b"CONNECT 127.0.0.1:8765 HTTP/1.1\r\nHost: 127.0.0.1:8765\r\n\r\n", - b"GET http://internal.example:8765/ HTTP/1.1\r\nHost: internal.example\r\n\r\n", - ], -) -def test_public_egress_cannot_reach_substrate_addresses( - payload: bytes, monkeypatch: pytest.MonkeyPatch -) -> None: - import socket as socket_mod - - from hud.environment.egress import ANY_HOST, Egress - - monkeypatch.setattr( - socket_mod, - "getaddrinfo", - lambda *args, **kwargs: [ - ( - socket_mod.AF_INET, - socket_mod.SOCK_STREAM, - socket_mod.IPPROTO_TCP, - "", - ("127.0.0.1", 8765), - ) - ], - ) - sockets = Path(tempfile.mkdtemp(dir="/tmp")) - egress = Egress(sockets, {ANY_HOST}) - egress.start() - try: - client = socket_mod.socket(socket_mod.AF_UNIX, socket_mod.SOCK_STREAM) - client.settimeout(5) - client.connect(str(egress.socket_path)) - client.sendall(payload) - response = client.recv(4096) - client.close() - finally: - egress.stop() - shutil.rmtree(sockets, ignore_errors=True) - - assert b"403" in response - assert b"X-Proxy-Error: blocked-by-address" in response - - -def test_plain_http_upstream_failures_return_a_proxy_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.environment import egress as egress_mod - from hud.environment.egress import Egress - - def unavailable(*_args: object, **_kwargs: object) -> socket.socket: - raise ConnectionRefusedError - - monkeypatch.setattr(egress_mod, "_connect_public", unavailable) - sockets = Path(tempfile.mkdtemp(dir="/tmp")) - egress = Egress(sockets, {"example.com"}) - egress.start() - try: - client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - client.settimeout(5) - client.connect(str(egress.socket_path)) - client.sendall(b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n") - response = client.recv(4096) - client.close() - finally: - egress.stop() - shutil.rmtree(sockets, ignore_errors=True) - - assert b"502" in response - assert b"X-Proxy-Error: upstream-failure" in response - - -def test_a_visitor_proxy_requires_its_ephemeral_credential() -> None: - import socket as socket_mod - - from hud.environment.egress import Egress - - sockets = Path(tempfile.mkdtemp(dir="/tmp")) - egress = Egress(sockets, {"example.com"}, token="secret") - egress.start() - try: - client = socket_mod.socket(socket_mod.AF_UNIX, socket_mod.SOCK_STREAM) - client.settimeout(5) - client.connect(str(egress.socket_path)) - client.sendall(b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n") - response = client.recv(4096) - client.close() - finally: - egress.stop() - shutil.rmtree(sockets, ignore_errors=True) - - assert b"407 Proxy Authentication Required" in response - assert egress.environment()["http_proxy"].startswith("http://hud:secret@") - - -def test_a_workspace_that_reaches_no_host_is_told_of_no_proxy() -> None: - """Pointing a client at a proxy that was never started turns "this task - has no network" into a connection failure on the first hop.""" - from hud.environment.egress import Egress, Peer - - assert Egress("/tmp/unused", set(), [Peer("db", 5432)]).environment() == {} - assert Egress("/tmp/unused", {"pypi.org"}).environment()["https_proxy"].endswith(":3128") - - -def test_a_host_is_permitted_by_name_or_as_a_subdomain() -> None: - from hud.environment.egress import ANY_HOST, permitted - - assert permitted("pypi.org", {"pypi.org"}) - assert permitted("files.pypi.org", {"pypi.org"}) # a subdomain of what was named - assert not permitted("notpypi.org", {"pypi.org"}) # not a subdomain, a different host - assert not permitted("pypi.org.evil.com", {"pypi.org"}) - assert not permitted("anything", set()) # declared unreachable - assert permitted("anything", {ANY_HOST}) - assert not permitted(None, {ANY_HOST}) - - -def test_shell_uid_wraps_sessions_in_setpriv( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _wall(monkeypatch) - ws = Workspace(tmp_path / "root", shell_uid=1000) - argv = ws.shell_argv("echo hi") - # Absolute path: a bare name would resolve through the session PATH, - # which the agent can influence — that lookup happens before the drop. - # --no-new-privs: a setuid binary must not let the shell regain root. - assert argv[:8] == [ - "/usr/bin/setpriv", - "--reuid", - "1000", - "--regid", - "1000", - "--clear-groups", - "--no-new-privs", - "--", - ] - # The session env rides `env -i` *inside* the setpriv wrapper, so it only - # takes effect after the drop. - assert argv[8].endswith("/env") and argv[9] == "-i" - assert "echo hi" in argv - - -def test_shell_identity_keeps_the_declared_primary_group( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _wall(monkeypatch) - ws = Workspace(tmp_path / "root", shell_uid=1000, shell_gid=2000) - - argv = ws.shell_argv("id") - - assert argv[argv.index("--reuid") + 1] == "1000" - assert argv[argv.index("--regid") + 1] == "2000" - - -def test_caller_env_is_injected_only_after_the_drop( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """An agent-influenced var like LD_PRELOAD must not be in the environment - of the root-run setpriv; it may only reach the post-drop shell.""" - _wall(monkeypatch) - ws = Workspace(tmp_path / "root", shell_uid=1000, env={"LD_PRELOAD": "/workspace/evil.so"}) - argv = ws.shell_argv("echo hi") - assert argv[0] == "/usr/bin/setpriv" - assert "LD_PRELOAD=/workspace/evil.so" in argv[argv.index("-i") :] - - -@pytest.mark.asyncio -async def test_wall_handoff_is_top_level_only_and_never_walks_the_tree( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """The handoff must be O(1): only the workspace dir is chowned. Recursing - over baked content (node_modules, a venv) on the serving path would delay - the control-port bind past the deploy readiness probe.""" - _wall(monkeypatch) - handed: list[str] = [] - monkeypatch.setattr(os, "lchown", lambda p, u, g: handed.append(os.fsdecode(p))) - monkeypatch.setattr(os, "walk", lambda *a, **k: pytest.fail("handoff must not walk the tree")) - monkeypatch.setattr(os, "fwalk", lambda *a, **k: pytest.fail("handoff must not walk the tree")) - - root = tmp_path / "root" - (root / "pkg").mkdir(parents=True) - (root / "pkg" / "mod.py").write_text("x = 1\n") - - ws = Workspace(root, shell_uid=1000) - await ws.start() - try: - assert [Path(p).name for p in handed] == ["root"] - finally: - await ws.stop() - - -@pytest.mark.asyncio -async def test_wall_handoff_failure_refuses_to_serve( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A failed ownership handoff leaves a workspace the agent can't write; - the server must fail loudly instead of serving it.""" - _wall(monkeypatch) - - def deny(p: object, u: int, g: int) -> None: - raise PermissionError("operation not permitted") - - monkeypatch.setattr(os, "lchown", deny) - ws = Workspace(tmp_path / "root", shell_uid=1000) - with pytest.raises(PermissionError): - await ws.start() - - -def test_shell_uid_is_a_noop_off_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(Workspace, "_drops_privileges", lambda self: False) - ws = Workspace(tmp_path / "root", shell_uid=1000) - assert "setpriv" not in ws.shell_argv("echo hi") - assert ws._session_env() is None - - -def test_without_shell_uid_argv_is_unchanged(tmp_path: Path) -> None: - ws = Workspace(tmp_path / "root") - assert "setpriv" not in ws.shell_argv("echo hi") - - -@pytest.mark.asyncio -async def test_session_wrapper_environment_contains_no_server_secrets( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - class SpawnCaptured(Exception): - def __init__(self, env: dict[str, str] | None) -> None: - self.env = env - - async def capture_spawn(*_args: str, **kwargs: Any) -> None: - raise SpawnCaptured(kwargs.get("env")) - - monkeypatch.setenv("HUD_API_KEY", "super-secret") - ws = Workspace(tmp_path / "root") - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) - ws._namespace = cast("Any", SimpleNamespace(spawn=capture_spawn)) - process = SimpleNamespace(term_type=None, command="true", env={}) - - with pytest.raises(SpawnCaptured) as captured: - await ws._handle_process(cast("Any", process)) - - assert captured.value.env == {"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin")} - - -@pytest.mark.asyncio -async def test_process_guard_preserves_session_proxy_environment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - captured: list[dict[str, Any]] = [] - - class Guard: - sandbox_helper = "/run/hud/process_guard.py" - sandbox_socket = "/run/hud/process_guard.sock" - backend = "notify" - - def __init__(self, directory: Path, *_args: object) -> None: - self.directory = directory - - def start(self) -> None: - pass - - def close(self) -> None: - pass - - def capture_bwrap(*_args: object, **kwargs: Any) -> list[str]: - captured.append(kwargs) - raise RuntimeError("captured") - - proxy = { - "http_proxy": "http://127.0.0.1:3128", - "https_proxy": "http://127.0.0.1:3128", - "no_proxy": "127.0.0.1,localhost,main", - } - monkeypatch.setenv("HUD_API_KEY", "server-secret") - monkeypatch.setattr(workspace_mod, "_process_guard_interpreter", lambda: "/usr/bin/python3") - monkeypatch.setattr(workspace_mod, "ProcessConnectionGuard", Guard) - monkeypatch.setattr(Workspace, "supports_process_connections", True) - ws = Workspace(tmp_path / "root", env={"WORKSPACE_ENV": "present"}) - ws._process_connections["inference"] = cast("Any", object()) - ws._egress = cast("Any", SimpleNamespace(environment=lambda: proxy)) - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) - monkeypatch.setattr(ws, "_process_connection_targets", lambda _names: frozenset()) - monkeypatch.setattr(ws, "bwrap_argv", capture_bwrap) - process = SimpleNamespace( - term_type="xterm-256color", - command="true", - env={PROCESS_CONNECTIONS_REQUEST: '["inference"]'}, - channel=SimpleNamespace(is_closing=Mock(return_value=False)), - stderr=SimpleNamespace(write=Mock()), - exit=Mock(), - ) - - await ws._handle_process(cast("Any", process)) - - assert len(captured) == 1 - session_env = captured[0]["env"] - assert {name: session_env[name] for name in proxy} == proxy - assert session_env["WORKSPACE_ENV"] == "present" - assert session_env["TERM"] == "xterm-256color" - assert "HUD_API_KEY" not in session_env - assert captured[0]["inherit_host_env"] is False - assert captured[0]["inherit_workspace_env"] is False - - -@pytest.mark.asyncio -async def test_namespace_wait_status_is_forwarded_to_ssh_client( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - async def wait_until_cancelled() -> None: - await asyncio.Event().wait() - - def empty_reader() -> asyncio.StreamReader: - reader = asyncio.StreamReader() - reader.feed_eof() - return reader - - child_channel = SimpleNamespace(close=Mock(), wait_closed=AsyncMock()) - child_process = SimpleNamespace( - stdin=SimpleNamespace(write_eof=Mock()), - stdout=empty_reader(), - stderr=empty_reader(), - wait_closed=AsyncMock(), - returncode=None, - channel=child_channel, - ) - child = namespace_mod.NamespaceProcess(cast("Any", child_process)) - namespace = SimpleNamespace(spawn=AsyncMock(return_value=child)) - channel = SimpleNamespace( - wait_closed=wait_until_cancelled, - is_closing=Mock(return_value=False), - ) - process = SimpleNamespace( - term_type=None, - command="true", - stdin=SimpleNamespace(read=AsyncMock(return_value=b"")), - stdout=SimpleNamespace(write=Mock(), drain=AsyncMock()), - stderr=SimpleNamespace(write=Mock(), drain=AsyncMock()), - channel=channel, - exit=Mock(), - ) - ws = Workspace(tmp_path / "root") - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=7)) - ws._namespace = cast("Any", namespace) - - await ws._handle_process(cast("Any", process)) - - process.exit.assert_called_once_with(255) - child_channel.close.assert_not_called() - - -@pytest.mark.asyncio -async def test_root_without_working_drop_fails_closed( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Serving as root while unable to drop must refuse rather than run agents - as root.""" - monkeypatch.setattr("hud.environment.workspace._is_root", lambda: True) - monkeypatch.setattr(Workspace, "_drops_privileges", lambda self: False) - ws = Workspace(tmp_path / "root", shell_uid=1000) - with pytest.raises(RuntimeError, match="privileges cannot be dropped"): - await ws.start() - - -def test_credentials_dir_is_private_and_unpredictable(tmp_path: Path) -> None: - ws = Workspace(tmp_path / "root") - creds = ws._credentials_dir() - assert creds.is_relative_to(Path(tempfile.gettempdir())) - assert not creds.is_relative_to(ws.root) - # mkdtemp yields 0700 and a fresh name each call (no shared parent to hijack). - assert (creds.stat().st_mode & 0o777) == 0o700 - assert ws._credentials_dir() == creds # cached per instance - - -def test_usable_bwrap_reports_unusable_installs(monkeypatch) -> None: - """An installed bwrap that cannot create namespaces must not be used.""" - import subprocess - - from hud.environment import workspace as ws - - monkeypatch.setattr(ws, "_bwrap_usable", None) - monkeypatch.setattr(ws.shutil, "which", lambda _name: "/usr/bin/bwrap") - monkeypatch.setattr( - ws.subprocess, - "run", - lambda *a, **k: subprocess.CompletedProcess(a[0], 1, b"", b"No permissions"), - ) - - assert ws.usable_bwrap() is None - - -def test_windows_job_owns_the_process_tree() -> None: - def first_thread(_snapshot: int, entry_pointer: Any) -> bool: - entry_pointer._obj.owner_process_id = 123 - entry_pointer._obj.thread_id = 456 - return True - - kernel32 = SimpleNamespace( - CreateJobObjectW=Mock(return_value=42), - SetInformationJobObject=Mock(return_value=True), - AssignProcessToJobObject=Mock(return_value=True), - CreateToolhelp32Snapshot=Mock(return_value=43), - Thread32First=Mock(side_effect=first_thread), - Thread32Next=Mock(return_value=False), - OpenThread=Mock(return_value=44), - ResumeThread=Mock(return_value=1), - CloseHandle=Mock(return_value=True), - GetLastError=Mock(return_value=0), - ) - job = workspace_mod._WindowsJob(kernel32) - child = SimpleNamespace(_handle=99, pid=123) - - job.assign(cast("Any", child)) - job.resume(cast("Any", child)) - job.close() - job.close() - - set_limits = kernel32.SetInformationJobObject.call_args.args - assert set_limits[1] == workspace_mod._JOB_OBJECT_EXTENDED_LIMIT_INFORMATION - assert ( - set_limits[2]._obj.basic_limit_information.limit_flags - == workspace_mod._JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE - ) - kernel32.AssignProcessToJobObject.assert_called_once_with(42, 99) - kernel32.CreateToolhelp32Snapshot.assert_called_once_with(workspace_mod._TH32CS_SNAPTHREAD, 0) - kernel32.OpenThread.assert_called_once_with(workspace_mod._THREAD_SUSPEND_RESUME, False, 456) - kernel32.ResumeThread.assert_called_once_with(44) - assert [call.args[0] for call in kernel32.CloseHandle.call_args_list] == [44, 43, 42] - - -@pytest.mark.asyncio -async def test_windows_channel_close_terminates_the_process_job( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - exited = threading.Event() - - class Child: - _handle = 99 - stdout = None - stderr = None - returncode: int | None = None - - def communicate(self) -> tuple[bytes, bytes]: - assert exited.wait(1) - self.returncode = -1 - return b"", b"" - - def kill(self) -> None: - self.returncode = -9 - exited.set() - - child = Child() - job = SimpleNamespace( - assign=Mock(), - resume=Mock(), - close=Mock(side_effect=exited.set), - ) - channel = SimpleNamespace( - wait_closed=AsyncMock(), - is_closing=Mock(return_value=True), - ) - process = SimpleNamespace( - term_type=None, - command="cmd /c echo hello", - channel=channel, - stdout=SimpleNamespace(write=Mock()), - stderr=SimpleNamespace(write=Mock()), - exit=Mock(), - ) - ws = Workspace(tmp_path / "root") - popen = Mock(return_value=child) - monkeypatch.setattr(workspace_mod.sys, "platform", "win32") - monkeypatch.setattr(workspace_mod, "_WindowsJob", Mock(return_value=job)) - monkeypatch.setattr(workspace_mod.subprocess, "Popen", popen) - monkeypatch.setattr(ws, "sandbox_pid", AsyncMock(return_value=None)) - - await ws._handle_process(cast("Any", process)) - - job.assign.assert_called_once_with(child) - job.resume.assert_called_once_with(child) - job.close.assert_called_once() - assert popen.call_args.kwargs["creationflags"] == workspace_mod._CREATE_SUSPENDED - process.exit.assert_not_called() - - -def test_usable_bwrap_stages_pid_creation_when_direct_mounting_is_blocked( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import subprocess - - from hud.environment import workspace as ws - - monkeypatch.setattr(ws, "_bwrap_usable", None) - binaries = { - "bwrap": "/usr/bin/bwrap", - "unshare": "/usr/bin/unshare", - "true": "/usr/bin/true", - } - monkeypatch.setattr(ws.shutil, "which", binaries.get) - calls: list[list[str]] = [] - - def run(argv: list[str], **_: object) -> subprocess.CompletedProcess[bytes]: - calls.append(argv) - return subprocess.CompletedProcess(argv, int(len(calls) == 1), b"", b"blocked") - - monkeypatch.setattr(ws.subprocess, "run", run) - - assert ws.usable_bwrap() == Bubblewrap( - "/usr/bin/bwrap", pid_unshare="/usr/bin/unshare", tmpfs_modes=True - ) - assert "--unshare-pid" in calls[0] - assert calls[1][:4] == [ - "/usr/bin/unshare", - "--kill-child=KILL", - "--pid", - "--mount-proc", - ] - assert "--unshare-pid" not in calls[1] - assert calls[2][:4] == calls[1][:4] - assert calls[2][-6:-2] == ["--perms", "1777", "--tmpfs", "/tmp"] - - -def test_staged_bwrap_keeps_user_isolation_and_uses_the_staged_proc( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - ws = Workspace(tmp_path / "root") - monkeypatch.setattr( - ws, - "_bwrap", - Bubblewrap("/usr/bin/bwrap", pid_unshare="/usr/bin/unshare"), - ) - - argv = ws.bwrap_argv(["true"]) - - assert argv[:5] == [ - "/usr/bin/unshare", - "--kill-child=KILL", - "--pid", - "--mount-proc", - "/usr/bin/bwrap", - ] - assert "--unshare-user-try" in argv - assert "--unshare-pid" not in argv - assert "--proc" in argv - - joined = ws.bwrap_argv(["true"], isolate_processes=False, isolate_users=False) - assert joined[0] == "/usr/bin/bwrap" - assert "/usr/bin/unshare" not in joined - assert "--proc" in joined - dev = joined.index("--dev-bind") - assert joined[dev : dev + 3] == ["--dev-bind", "/dev", "/dev"] - - -async def test_required_isolation_refuses_to_start_when_unavailable(monkeypatch, tmp_path) -> None: - from hud.environment import workspace as ws - - monkeypatch.setattr(ws, "usable_bwrap", lambda: None) - workspace = ws.Workspace(tmp_path, require_isolation=True) - - with pytest.raises(RuntimeError, match="isolation was required"): - await workspace.start() - - -@pytest.mark.asyncio -async def test_identity_map_reads_a_chunked_info_document() -> None: - """bwrap's info JSON arrives in as many chunks as the pipe delivers; a - reader that stops at the first chunk parses a truncated document and the - sandbox never starts (observed as every session failing on a live box).""" - info_read, info_write = os.pipe() - block_read, block_write = os.pipe() - document = b'{\n "child-pid": 4242,\n "other": "field"\n}\n' - - def write_in_chunks() -> None: - os.write(info_write, document[:21]) # cut mid-document, line 2 - time.sleep(0.05) - os.write(info_write, document[21:]) - os.close(info_write) - - writer = threading.Thread(target=write_in_chunks) - writer.start() - try: - with mock.patch.object(namespace_mod, "_map_identities") as mapped: - pid = await namespace_mod.install_identity_map(info_read, block_write) - assert pid == 4242 - mapped.assert_called_once_with(4242) - assert os.read(block_read, 1) == b"\n" # the sandbox was released - finally: - writer.join() - for fd in (info_read, block_read, block_write): - with contextlib.suppress(OSError): - os.close(fd) - - -@pytest.mark.asyncio -async def test_identity_map_resolves_a_staged_sandbox_to_its_parent_pid( - monkeypatch: pytest.MonkeyPatch, -) -> None: - info_read, info_write = os.pipe() - block_read, block_write = os.pipe() - os.write(info_write, b'{"child-pid": 14}') - os.close(info_write) - children = { - "/proc/100/task/100/children": "102 ", - "/proc/102/task/102/children": "121 ", - } - - def read_text(path: Path, *args: object, **kwargs: object) -> str: - return children[str(path)] - - monkeypatch.setattr(Path, "read_text", read_text) - try: - with mock.patch.object(namespace_mod, "_map_identities") as mapped: - pid = await namespace_mod.install_identity_map( - info_read, - block_write, - launcher_pid=100, - launcher_depth=2, - ) - assert pid == 121 - mapped.assert_called_once_with(121) - assert os.read(block_read, 1) == b"\n" - finally: - for fd in (info_read, block_read, block_write): - with contextlib.suppress(OSError): - os.close(fd) - - -def test_identity_map_preserves_every_id_available_to_the_container( - monkeypatch: pytest.MonkeyPatch, -) -> None: - maps = { - "/proc/self/uid_map": "0 100000 131072\n", - "/proc/self/gid_map": "0 200000 65536\n70000 300000 1000\n", - } - writes: dict[str, str] = {} - - def read_text(path: Path) -> str: - return maps[str(path)] - - def write_text(path: Path, value: str) -> int: - writes[str(path)] = value - return len(value) - - monkeypatch.setattr(Path, "read_text", read_text) - monkeypatch.setattr(Path, "write_text", write_text) - - namespace_mod._map_identities(42) - - assert writes["/proc/42/uid_map"] == "0 0 131072\n" - assert writes["/proc/42/gid_map"] == "0 0 65536\n70000 70000 1000\n" - - -def test_the_proxy_refuses_to_relay_a_header_it_cannot_represent() -> None: - """An upstream header is remote text, and a folded value keeps its CRLF - through http.client. Relayed verbatim it would carry headers of its own - into the response the workspace reads, so a field outside the grammar - fails the whole response rather than being quietly repaired.""" - assert _field("Content-Type", "text/plain") == ("Content-Type", "text/plain") - assert _field("X-Meta", "") == ("X-Meta", "") - for name, value in ( - ("X-Evil", "a\r\n b"), - ("X-Evil", "a\nX-Injected: 1"), - ("X-Evil", "a\x00b"), - ("Bad Name", "fine"), - ("X-Evil\r\nX-Injected", "fine"), - ): - with pytest.raises(_Unrelayable): - _field(name, value) - - -def test_private_workspace_maps_its_own_hostname_to_loopback( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(socket, "gethostname", lambda: "container-self") - ws = Workspace( - tmp_path / "root", - allowed_hosts=set(), - local_aliases=["main"], - credentials_dir=tmp_path / "keys", - hosts_path=tmp_path / "hosts", - ) - - generated = ws._write_hosts().read_text("utf-8") - - assert "127.0.0.1\tmain" in generated - assert "127.0.0.1\tcontainer-self" in generated - - -def test_child_pid_discovery_falls_back_to_proc_stat(monkeypatch: pytest.MonkeyPatch) -> None: - entries = [Path("/proc/101"), Path("/proc/102"), Path("/proc/self")] - - def read_text(path: Path, *args: object, **kwargs: object) -> str: - if path.name == "children": - raise OSError("unavailable") - if str(path) == "/proc/101/stat": - return "101 (child one) S 100 0 0 0" - if str(path) == "/proc/102/stat": - return "102 (other) S 99 0 0 0" - raise OSError("gone") - - monkeypatch.setattr(Path, "iterdir", lambda path: iter(entries)) - monkeypatch.setattr(Path, "read_text", read_text) - - assert namespace_mod._child_pids(100) == [101] - - -@pytest.mark.asyncio -async def test_namespace_preserves_large_streams_and_status( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - payload = b"0123456789abcdef" * 131072 + b"tail" - async with _connected_namespace_host(monkeypatch) as namespace: - process = await namespace.spawn( - ["sh", "-c", "cat; printf stderr-tail >&2"], - cwd=tmp_path, - env=dict(os.environ), - persistent=True, - ) - stdout = asyncio.create_task(process.stdout.read()) - stderr = asyncio.create_task(process.stderr.read()) - status = asyncio.create_task(process.wait_status()) - process.stdin.write(payload) - await process.stdin.drain() - process.stdin.write_eof() - async with asyncio.timeout(10): - assert await status == 0 - assert await stdout == payload - assert await stderr == b"stderr-tail" - - -@pytest.mark.asyncio -async def test_namespace_large_input_to_early_exit_does_not_hang( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - async with _connected_namespace_host(monkeypatch) as namespace: - process = await namespace.spawn( - ["sh", "-c", "printf done"], - cwd=tmp_path, - env=dict(os.environ), - persistent=True, - ) - process.stdin.write(b"x" * 500_000) - process.stdin.write_eof() - result = await asyncio.wait_for(process.complete(), 5) - assert result.returncode == 0 - assert result.stdout == b"done" - - -@pytest.mark.asyncio -async def test_namespace_terminal_output_survives_backpressure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - connect = asyncssh.connect - - async def small_window(*args: Any, **kwargs: Any) -> asyncssh.SSHClientConnection: - return await connect(*args, **kwargs, window=32768) - - monkeypatch.setattr(asyncssh, "connect", small_window) - async with _connected_namespace_host(monkeypatch) as namespace: - process = await namespace.spawn( - ["sh", "-c", "head -c 500000 /dev/zero | tr '\\000' x"], - cwd=tmp_path, - env=dict(os.environ), - persistent=True, - tty=True, - ) - result = await asyncio.wait_for(process.complete(), 5) - assert result.returncode == 0 - assert result.stdout == b"x" * 500000 - - -@pytest.mark.asyncio -async def test_command_timeout_keeps_connection_usable(tmp_path: Path) -> None: - ws = Workspace(tmp_path / "root") - await ws.start() - try: - async with await _connect(ws) as conn: - client = SSHClient(ws.capability(), conn) - with pytest.raises(TimeoutError): - await client.run("sleep 120", timeout=0.2) - result = await client.run("printf ready", check=True, timeout=5) - assert result.stdout == "ready" - finally: - await ws.stop() - - -@pytest.mark.asyncio -async def test_namespace_wait_drains_unread_output( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - async with _connected_namespace_host(monkeypatch) as namespace: - process = await namespace.spawn( - ["sh", "-c", "head -c 8388608 /dev/zero; head -c 8388608 /dev/zero >&2; exit 7"], - cwd=tmp_path, - env=dict(os.environ), - persistent=True, - ) - process.stdin.write_eof() - assert await asyncio.wait_for(process.wait(), 5) == 7 diff --git a/hud/environment/utils.py b/hud/environment/utils.py index 235f27f57..038c92261 100644 --- a/hud/environment/utils.py +++ b/hud/environment/utils.py @@ -94,11 +94,17 @@ def error(msg_id: int | str | None, code: int, message: str) -> dict[str, Any]: async def _pump(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: # Resets/aborts are a normal way for tunneled streams to end (an SSH # client hanging up, a container dying); they end the pump, not the world. - with contextlib.suppress(OSError): + # A reset carries no EOF, so close the far side instead: otherwise it + # waits for bytes that will never come. + try: while data := await reader.read(65536): writer.write(data) await writer.drain() - if writer.can_write_eof(): + except OSError: + writer.close() + return + if writer.can_write_eof(): + with contextlib.suppress(OSError): writer.write_eof() diff --git a/hud/eval/run.py b/hud/eval/run.py index 151f20fba..1e1b36d3b 100644 --- a/hud/eval/run.py +++ b/hud/eval/run.py @@ -107,28 +107,36 @@ def validate_rollout_timeouts( return agent_timeout -def _prompt_message(item: Any) -> mcp_types.PromptMessage: +def _prompt_messages(item: Any) -> list[mcp_types.PromptMessage]: """Coerce one wire prompt turn onto MCP's ``PromptMessage`` vocabulary. Turns are env-authored: chat-style dicts (plain-string content wrapped as text, roles outside MCP's user/assistant vocabulary such as ``system`` coerced to ``user``), already-built ``PromptMessage``s, or anything else - stringified. Coercion may be lossy — prompt context is what the agent is - given, and the verbatim payload stays on the setup ``task`` step's result. + stringified. A ``PromptMessage`` holds one content block, so a turn whose + content is a list of blocks becomes one message per block. Coercion may be + lossy — prompt context is what the agent is given, and the verbatim + payload stays on the setup ``task`` step's result. """ if isinstance(item, mcp_types.PromptMessage): - return item + return [item] if not isinstance(item, dict): item = {"content": str(item)} raw_role = item.get("role") role: Literal["user", "assistant"] = "assistant" if raw_role == "assistant" else "user" content = item.get("content") if isinstance(content, str): - return mcp_types.PromptMessage( - role=role, - content=mcp_types.TextContent(type="text", text=content), - ) - return mcp_types.PromptMessage.model_validate({**item, "role": role}) + return [ + mcp_types.PromptMessage( + role=role, + content=mcp_types.TextContent(type="text", text=content), + ) + ] + blocks = content if isinstance(content, list) else [content] + return [ + mcp_types.PromptMessage.model_validate({**item, "role": role, "content": block}) + for block in blocks + ] def _episode_bindings(started: dict[str, Any]) -> dict[str, dict[str, Any]]: @@ -280,11 +288,11 @@ def prompt_messages(self) -> list[mcp_types.PromptMessage]: The structured form agents consume and the opening ``user`` step records: a text prompt (or none) is one user turn; chat-style lists - map turn by turn. + map turn by turn, a turn of several blocks to one message per block. """ if self.prompt is None or isinstance(self.prompt, str): - return [_prompt_message({"content": self.prompt or ""})] - return [_prompt_message(item) for item in self.prompt] + return _prompt_messages({"content": self.prompt or ""}) + return [message for item in self.prompt for message in _prompt_messages(item)] @property def prompt_text(self) -> str: @@ -395,7 +403,8 @@ def failed(cls, error: str) -> Run: runtime, partial trace) with the error recorded on the trace. """ run = cls(None, "", {}) - run.trace = Trace(status="error", steps=[Step(source="system", error=error)]) + run.trace.status = "error" + run.record(Step(source="system", error=error)) return run @@ -693,7 +702,10 @@ async def close_actor() -> None: # handshake — where str(exc) would drop them. detail = "".join(traceback.format_exception_only(exc)).strip() if run is None: - logger.warning("rollout failed before launch (%s): %s", _phase, detail) + # No run exists yet, so this log is the only place the traceback survives. + logger.warning( + "rollout failed before launch (%s): %s", _phase, detail, exc_info=True + ) run = Run.failed(f"[{_phase}] {detail}") else: logger.warning("rollout failed mid-run (%s): %s", _phase, detail) diff --git a/hud/eval/runtime/compose.py b/hud/eval/runtime/compose.py index 331387dd1..44e1e8af7 100644 --- a/hud/eval/runtime/compose.py +++ b/hud/eval/runtime/compose.py @@ -39,6 +39,10 @@ class ComposeUnboundVariableError(ValueError): """A Compose variable depends on values outside the packaged project.""" +class ComposeUnsupportedError(ValueError): + """A Compose document uses a feature the serialized project contract excludes.""" + + def _interpolate_compose_value( value: str, environment: Mapping[str, str], @@ -357,7 +361,9 @@ def from_file(cls, path: Path, *, variables: Mapping[str, str] | None = None) -> isinstance(service, dict) and "extends" in service for service in services.values() ) ): - raise ValueError("remote adaptation does not support Compose include or extends") + raise ComposeUnsupportedError( + "remote adaptation does not support Compose include or extends" + ) return cls.model_validate(document) def with_project_directory(self, directory: str) -> ComposeConfig: diff --git a/hud/eval/runtime/hosted.py b/hud/eval/runtime/hosted.py index a98efe720..43db75bbe 100644 --- a/hud/eval/runtime/hosted.py +++ b/hud/eval/runtime/hosted.py @@ -211,7 +211,12 @@ def _fold(state: dict[str, Any], trace_id: str) -> Run: return run async def _await_terminal(self, platform: PlatformClient, trace_id: str) -> dict[str, Any]: + task = asyncio.current_task() while True: + # httpx can swallow a cancellation that lands mid-request, which + # would leave a timed-out or cancelled rollout polling forever. + if task is not None and task.cancelling(): + raise asyncio.CancelledError state: dict[str, Any] = await platform.aget(f"/trace/{trace_id}") if state.get("status") in _TERMINAL_TRACE_STATUSES: return state diff --git a/hud/eval/runtime/local.py b/hud/eval/runtime/local.py index a383a732d..090a41bef 100644 --- a/hud/eval/runtime/local.py +++ b/hud/eval/runtime/local.py @@ -13,6 +13,7 @@ from weakref import WeakValueDictionary from hud.utils.process import ( + OUTPUT_DRAIN_TIMEOUT, create_process_group_exec, finish_output, output_writer, @@ -219,38 +220,56 @@ def capture_error(text: str) -> None: try: from hud.environment.server import PORT_ANNOUNCEMENT - port = None - line_continues = False write_stdout, finish_stdout = output_writer( sys.stdout, capture=lambda text: output_tail.append(text.strip()[-1024:]), ) + + async def read_port() -> int | None: + """The port the child announces, or ``None`` once stdout ends without one.""" + line_continues = False + while True: + eof = False + try: + line = await output.readuntil() + except asyncio.LimitOverrunError as exc: + line = await output.read(exc.consumed) + line_continues = True + except asyncio.IncompleteReadError as exc: + line = exc.partial + eof = True + if not line: + return None + if not line_continues: + text = line.decode("utf-8", "replace").strip() + if text.startswith(PORT_ANNOUNCEMENT): + return int(text.removeprefix(PORT_ANNOUNCEMENT)) + write_stdout(line) + line_continues = not line.endswith(b"\n") + if eof: + return None + + reading = asyncio.create_task(read_port()) + exited = asyncio.create_task(proc.wait()) try: async with asyncio.timeout(self.ready_timeout): - while True: - eof = False - try: - line = await output.readuntil() - except asyncio.LimitOverrunError as exc: - line = await output.read(exc.consumed) - line_continues = True - except asyncio.IncompleteReadError as exc: - line = exc.partial - eof = True - if not line: - break - if not line_continues: - text = line.decode("utf-8", "replace").strip() - if text.startswith(PORT_ANNOUNCEMENT): - port = int(text.removeprefix(PORT_ANNOUNCEMENT)) - break - write_stdout(line) - line_continues = not line.endswith(b"\n") - if eof: - break + await asyncio.wait((reading, exited), return_when=asyncio.FIRST_COMPLETED) + if not reading.done(): + # The child exited without announcing a port, and a process + # it started may still hold stdout open: take what is there. + await asyncio.wait((reading,), timeout=OUTPUT_DRAIN_TIMEOUT) finally: + exited.cancel() + reading.cancel() + await asyncio.gather(reading, exited, return_exceptions=True) finish_stdout() + port = None if reading.cancelled() else reading.result() if port is None: + # Stdout ends as the child exits; give it a moment to be reaped + # so the error reports its exit code rather than a closed pipe. + with contextlib.suppress(TimeoutError): + async with asyncio.timeout(1.0): + await proc.wait() await finish_output(stderr_drain) tail = "\n".join((*output_tail, *error_tail)).strip() detail = f":\n{tail}" if tail else " (no output captured)" diff --git a/hud/eval/tests/test_chat.py b/hud/eval/tests/test_chat.py deleted file mode 100644 index c97e62f13..000000000 --- a/hud/eval/tests/test_chat.py +++ /dev/null @@ -1,129 +0,0 @@ -"""``Chat`` — multi-turn conversation runner over a task. - -Turn tests place each turn's rollout with ``runtime=SubprocessRuntime(env_file)`` — a pure-data -``Task`` row against a chat-style env served from a child process. -""" - -from __future__ import annotations - -import textwrap -from typing import TYPE_CHECKING, Any - -import pytest -from mcp.types import TextContent - -from hud.agents.base import Agent -from hud.eval import SubprocessRuntime, Task -from hud.eval.chat import Chat, _content_to_blocks - -if TYPE_CHECKING: - from pathlib import Path - - -class _EchoAgent(Agent): - """Replies with ``echo:`` read from the prompt.""" - - async def __call__(self, run: Any) -> None: - last = run.prompt[-1]["content"]["text"] - run.trace.content = f"echo:{last}" - - -@pytest.fixture() -def dummy_task() -> Any: - """Minimal Task for Chat construction.""" - return Task(env="chat", id="test_scenario") - - -class TestContentHelpers: - def test_content_to_blocks_string(self) -> None: - blocks = _content_to_blocks("hello") - assert len(blocks) == 1 - assert isinstance(blocks[0], TextContent) - assert blocks[0].text == "hello" - - def test_content_to_blocks_passthrough(self) -> None: - original = [TextContent(type="text", text="x")] - assert _content_to_blocks(original) is original - - -class TestChatConstruction: - def test_messages_start_empty_and_are_the_public_history(self, dummy_task: Any) -> None: - chat = Chat(dummy_task, _EchoAgent()) - assert chat.messages == [] - assert chat.job is None # the conversation's job starts on the first send - # History is the plain ``messages`` list: persist/restore it directly. - chat.messages = [{"role": "user", "content": {"type": "text", "text": "hi"}}] - assert Chat(dummy_task, _EchoAgent()).messages == [] - - -_CHAT_ENV = """\ -from hud import Environment - -env = Environment("chat") - - -@env.template() -async def assistant(messages: list): - _answer = yield messages - yield 1.0 -""" - - -@pytest.fixture(scope="module") -def chat_env_file(tmp_path_factory: pytest.TempPathFactory) -> Path: - path = tmp_path_factory.mktemp("chat") / "env.py" - path.write_text(textwrap.dedent(_CHAT_ENV), encoding="utf-8") - return path - - -def _chat_task() -> Task: - """A pure data row for the chat-style task the spawned file defines.""" - return Task(env="chat", id="assistant", args={"messages": []}) - - -class TestSend: - async def test_send_runs_a_turn_and_stores_prompt_message_format( - self, chat_env_file: Path - ) -> None: - chat = Chat(_chat_task(), _EchoAgent(), runtime=SubprocessRuntime(chat_env_file)) - - trace = await chat.send("hello") - - assert trace.content == "echo:hello" - assert len(chat.messages) == 2 - - user_msg = chat.messages[0] - assert user_msg["role"] == "user" - assert user_msg["content"]["type"] == "text" - assert user_msg["content"]["text"] == "hello" - - assistant_msg = chat.messages[1] - assert assistant_msg["role"] == "assistant" - assert assistant_msg["content"]["type"] == "text" - assert assistant_msg["content"]["text"] == "echo:hello" - - async def test_one_job_spans_the_conversation(self, chat_env_file: Path) -> None: - chat = Chat(_chat_task(), _EchoAgent(), runtime=SubprocessRuntime(chat_env_file)) - - await chat.send("hello") - await chat.send("again") - - job = chat.job - assert job is not None - assert len(job.runs) == 2 - # Every turn's trace reports under the conversation's job. - assert {run.job_id for run in job.runs} == {job.id} - - async def test_failed_turn_raises_and_records_no_assistant_message( - self, chat_env_file: Path - ) -> None: - class _Boom(Agent): - async def __call__(self, run: Any) -> None: - raise RuntimeError("agent exploded") - - chat = Chat(_chat_task(), _Boom(), runtime=SubprocessRuntime(chat_env_file)) - - with pytest.raises(RuntimeError, match="agent exploded"): - await chat.send("hello") - - assert [m["role"] for m in chat.messages] == ["user"] diff --git a/hud/eval/tests/test_docker_provider.py b/hud/eval/tests/test_docker_provider.py deleted file mode 100644 index a6e3b5f7f..000000000 --- a/hud/eval/tests/test_docker_provider.py +++ /dev/null @@ -1,2400 +0,0 @@ -"""``DockerRuntime()`` provider behavior, driven through a scripted docker CLI. - -No daemon needed: a fake ``docker`` executable on PATH records every -invocation and scripts the responses, so these tests pin the provider's -contract — command shape, runtime address, teardown — at the process -boundary. -""" - -from __future__ import annotations - -import asyncio -import hashlib -import json -import logging -import os -import sys -import tarfile -from dataclasses import dataclass -from pathlib import Path -from types import ModuleType, SimpleNamespace -from typing import TYPE_CHECKING, Any, cast - -import pytest - -import hud.eval.runtime.docker as runtime_module -import hud.utils.process as process_module -from hud.eval.runtime import ( - DaytonaRuntime, - DockerBindMount, - DockerRuntime, - ModalRuntime, - RuntimeConfig, - RuntimeGPU, - RuntimeLimits, - RuntimeResources, - Shared, -) -from hud.eval.runtime.compose import ( - ComposeConfig, - ComposeHealthcheck, - ComposeProject, - ComposeService, -) -from hud.eval.task import Task - -if TYPE_CHECKING: - from collections.abc import AsyncIterator - - from daytona import Image as DaytonaImage - -FAKE_DOCKER_SH = """\ -#!/bin/sh -echo "$@" >> "$DOCKER_LOG" -case "$1" in - run) echo cid-42 ;; - port) {port_behavior} ;; - exec) if [ "$3" = "df" ]; then - printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n' - printf 'overlay 4194304 0 4194304 0%% /\n' - fi ;; - logs) echo "ImportError: boom" ;; -esac -""" - -FAKE_DOCKER_CMD = """\ -@echo off -echo %*>>"%DOCKER_LOG%" -if "%1"=="run" ( - echo cid-42 - exit /b 0 -) -if "%1"=="port" ( - {port_behavior} - exit /b 0 -) -if "%1"=="logs" ( - echo ImportError: boom - exit /b 0 -) -exit /b 0 -""" - - -def _port_behavior_for_windows(port_behavior: str) -> str: - if port_behavior == "echo 127.0.0.1:43210": - return "echo 127.0.0.1:43210" - if port_behavior == ":": - return "rem noop" - raise ValueError(f"unsupported port_behavior: {port_behavior!r}") - - -async def _docker_via(fake_exe: Path, *args: str, check: bool = True) -> tuple[str, str]: - proc = await asyncio.create_subprocess_exec( - str(fake_exe), - *args, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - out, err = await proc.communicate() - if check and proc.returncode != 0: - detail = err.decode("utf-8", "replace").strip() or out.decode("utf-8", "replace").strip() - raise RuntimeError(f"docker {' '.join(args)} failed ({proc.returncode}): {detail}") - return out.decode("utf-8", "replace"), err.decode("utf-8", "replace") - - -@pytest.fixture -def docker_log(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - log = tmp_path / "docker.log" - log.touch() - monkeypatch.setenv("PATH", f"{tmp_path}{os.pathsep}{os.environ['PATH']}") - monkeypatch.setenv("DOCKER_LOG", str(log)) - return log - - -def _install_fake_docker( - tmp_path: Path, - *, - port_behavior: str, - monkeypatch: pytest.MonkeyPatch, -) -> None: - if sys.platform == "win32": - exe = tmp_path / "docker.cmd" - exe.write_text( - FAKE_DOCKER_CMD.format(port_behavior=_port_behavior_for_windows(port_behavior)) - ) - import hud.eval.runtime.docker as runtime_module - - async def _docker(*args: str, check: bool = True) -> tuple[str, str]: - return await _docker_via(exe, *args, check=check) - - monkeypatch.setattr(runtime_module, "_docker", _docker) - return - - exe = tmp_path / "docker" - exe.write_text(FAKE_DOCKER_SH.format(port_behavior=port_behavior)) - exe.chmod(0o755) - - -def _row() -> Task: - return Task(env="any-env", id="t") - - -async def _docker_calls(docker_log: Path) -> list[str]: - return (await asyncio.to_thread(docker_log.read_text)).splitlines() - - -def _docker_security_args() -> str: - return " ".join(runtime_module._DOCKER_SECURITY_ARGS) - - -def _session_mount_args(volume: str) -> str: - return ( - f"--mount type=volume,source={volume},target=/runtime/sessions " - f"--mount type=volume,source={volume},target=/media/hud/sessions" - ) - - -@dataclass(frozen=True) -class _ModalImageRef: - kind: str - name: str - env_vars: dict[str, str] | None = None - - def env(self, vars: dict[str, str]) -> _ModalImageRef: - return _ModalImageRef(self.kind, self.name, dict(vars)) - - -@dataclass(frozen=True) -class _ModalSecretRef: - name: str - - -class _FakeModalStream: - def __init__(self, calls: dict[str, Any], key: str) -> None: - self._calls = calls - self._key = key - - def __aiter__(self) -> AsyncIterator[str]: - return self._iterate() - - async def _iterate(self) -> AsyncIterator[str]: - output = cast("str", self._calls.get(self._key, "")) - if output: - yield output - wait = self._calls.get(f"{self._key}_wait") - if isinstance(wait, asyncio.Event): - await wait.wait() - - -class _FakeModalSandbox: - object_id = "sb-1" - - def __init__(self, calls: dict[str, Any], port: int) -> None: - self._calls = calls - self._port = port - self.wait_until_ready = SimpleNamespace(aio=self._wait_until_ready) - self.tunnels = SimpleNamespace(aio=self._tunnels) - self.terminate = SimpleNamespace(aio=self._terminate) - self.stdout = _FakeModalStream(calls, "sandbox_stdout") - self.stderr = _FakeModalStream(calls, "sandbox_stderr") - self.filesystem = SimpleNamespace( - copy_from_local=SimpleNamespace(aio=self._copy_from_local), - copy_to_local=SimpleNamespace(aio=self._copy_to_local), - ) - self.exec = SimpleNamespace(aio=self._exec) - - async def _wait_until_ready(self, **kwargs: object) -> None: - self._calls["ready_timeout"] = kwargs["timeout"] - - async def _tunnels(self) -> dict[int, SimpleNamespace]: - return {self._port: SimpleNamespace(tcp_socket=("modal.host", 4567))} - - async def _terminate(self) -> None: - self._calls["terminated"] = True - - async def _copy_from_local(self, source: Path, target: str) -> None: - uploads = self._calls.setdefault("uploads", []) - assert isinstance(uploads, list) - uploads.append((source.name, target)) - if source.name == "override.json": - content = await asyncio.to_thread(source.read_text, "utf-8") - self._calls["compose_override"] = json.loads(content) - - async def _copy_to_local(self, source: str, target: Path) -> None: - downloads = self._calls.setdefault("downloads", []) - assert isinstance(downloads, list) - downloads.append((source, target.name)) - - async def _exec(self, *args: str, **kwargs: object) -> SimpleNamespace: - commands = self._calls.setdefault("execs", []) - assert isinstance(commands, list) - commands.append((args, kwargs)) - - async def wait() -> int: - wait_event = self._calls.pop("exec_wait", None) - if isinstance(wait_event, asyncio.Event): - await wait_event.wait() - return 0 - - async def read_stderr() -> str: - return "" - - async def read_stdout() -> str: - command = args[-1] - return ( - "1" - if "sessions/sess-actor" in command - and ("test -d" in command or "if [ -d" in command) - else "" - ) - - if "logs" in args: - return SimpleNamespace( - wait=SimpleNamespace(aio=wait), - stdout=_FakeModalStream(self._calls, "compose_stdout"), - stderr=_FakeModalStream(self._calls, "compose_stderr"), - ) - - return SimpleNamespace( - wait=SimpleNamespace(aio=wait), - stdout=SimpleNamespace(read=SimpleNamespace(aio=read_stdout)), - stderr=SimpleNamespace(read=SimpleNamespace(aio=read_stderr)), - ) - - -def _install_fake_modal(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: - calls: dict[str, Any] = {} - modal = ModuleType("modal") - - class Image: - @staticmethod - def from_name(name: str) -> _ModalImageRef: - calls["image_name"] = name - return _ModalImageRef("name", name) - - @staticmethod - def from_registry(name: str, secret: object = None) -> _ModalImageRef: - calls["registry_image"] = name - calls["registry_secret"] = secret - return _ModalImageRef("registry", name) - - @staticmethod - def from_id(image_id: str) -> _ModalImageRef: - calls["modal_image_id"] = image_id - return _ModalImageRef("id", image_id) - - async def lookup(app_name: str, *, create_if_missing: bool) -> str: - calls["app_lookup"] = (app_name, create_if_missing) - return "app" - - async def create(*args: str, **kwargs: object) -> _FakeModalSandbox: - calls["sandbox_args"] = args - calls["sandbox_kwargs"] = kwargs - ports = kwargs["unencrypted_ports"] - assert isinstance(ports, list) - port = ports[0] - assert isinstance(port, int) - return _FakeModalSandbox(calls, port) - - setattr(modal, "Image", Image) - setattr(modal, "App", SimpleNamespace(lookup=SimpleNamespace(aio=lookup))) - setattr(modal, "Probe", SimpleNamespace(with_tcp=lambda port: ("tcp", port))) - setattr(modal, "Sandbox", SimpleNamespace(create=SimpleNamespace(aio=create))) - monkeypatch.setitem(sys.modules, "modal", modal) - return calls - - -@dataclass(frozen=True) -class _CreateSandboxFromSnapshotParams: - snapshot: str - ephemeral: bool - auto_stop_interval: int - name: str | None = None - env_vars: dict[str, str] | None = None - linked_sandbox: str | None = None - - -@dataclass(frozen=True) -class _CreateSnapshotParams: - name: str - image: object - resources: object | None = None - - -@dataclass(frozen=True) -class _CreateSandboxFromImageParams: - image: object - ephemeral: bool - auto_stop_interval: int - resources: object | None = None - name: str | None = None - env_vars: dict[str, str] | None = None - linked_sandbox: str | None = None - - -@dataclass(frozen=True) -class _DaytonaImage: - name: str - - -@dataclass(frozen=True) -class _DaytonaResources: - cpu: float | None = None - memory: int | None = None - disk: int | None = None - gpu: int | None = None - gpu_type: list[object] | None = None - - -@dataclass(frozen=True) -class _DaytonaGpuType: - value: str - - -@dataclass(frozen=True) -class _SessionExecuteRequest: - command: str - run_async: bool - - -class _FakeDaytonaProcess: - def __init__(self, calls: dict[str, Any], sandbox_id: str) -> None: - self._calls = calls - self._sandbox_id = sandbox_id - self.logs = SimpleNamespace( - stderr="ImportError: no module named bugs", output="", stdout="" - ) - - async def create_session(self, session: str) -> None: - self._calls["session"] = session - - async def execute_session_command(self, session: str, request: object) -> SimpleNamespace: - self._calls["execute"] = (session, request) - commands = self._calls.setdefault("session_commands", []) - assert isinstance(commands, list) - commands.append((self._sandbox_id, session, request)) - return SimpleNamespace(cmd_id="cmd-1") - - async def get_session_command_logs(self, session: str, cmd_id: str) -> SimpleNamespace: - self._calls["logs"] = (session, cmd_id) - return self.logs - - async def get_session_command_logs_async( - self, - session: str, - cmd_id: str, - on_stdout: Any, - on_stderr: Any, - ) -> None: - self._calls["stream_logs"] = (session, cmd_id) - on_stdout(cast("str", self._calls.get("daytona_stdout", ""))) - on_stderr(cast("str", self._calls.get("daytona_stderr", ""))) - wait = self._calls.get("daytona_log_wait") - if isinstance(wait, asyncio.Event): - await wait.wait() - - async def exec(self, command: str, **kwargs: object) -> SimpleNamespace: - commands = self._calls.setdefault("process_execs", []) - assert isinstance(commands, list) - commands.append((self._sandbox_id, command, kwargs)) - return SimpleNamespace(exit_code=0, result="") - - -class _FakeDaytonaSandbox: - def __init__(self, calls: dict[str, Any], sandbox_id: str) -> None: - self.id = sandbox_id - self._calls = calls - self.process = _FakeDaytonaProcess(calls, sandbox_id) - self.fs = SimpleNamespace(upload_file=self._upload_file) - - async def _upload_file(self, source: str, target: str) -> None: - uploads = self._calls.setdefault("uploads", []) - assert isinstance(uploads, list) - uploads.append((Path(source).name, target)) - - async def create_ssh_access(self, *, expires_in_minutes: int) -> SimpleNamespace: - self._calls["ssh_expires"] = expires_in_minutes - return SimpleNamespace(token="ssh-token") - - -def _tree_hash_sync(path_str: str) -> str: - """Deterministic fingerprint of a context tree: same tree, same hash.""" - digest = hashlib.md5(usedforsecurity=False) - root = Path(path_str) - for file in sorted(p for p in root.rglob("*") if p.is_file()): - digest.update(file.relative_to(root).as_posix().encode()) - digest.update(file.read_bytes()) - return digest.hexdigest() - - -async def _tree_hash(path_str: str) -> str: - return await asyncio.to_thread(_tree_hash_sync, path_str) - - -class _FakeObjectStorage: - """The SDK hasher the provider borrows to predict a context's upload hash.""" - - async def _compute_hash_for_path_md5( - self, path_str: str, archive_base_path: str | None = None - ) -> str: - return await _tree_hash(path_str) - - -class _FakeSnapshotApi: - """The registry a Daytona snapshot name resolves against: ``get`` 404s until - something with that exact name has been built, and each snapshot records what - it was built from (``image_name`` or ``build_info``), as the server does. - A deleted name stays taken for a couple more ``get``s — the live API frees - it asynchronously (~10s), and creating before then conflicts.""" - - def __init__(self) -> None: - self.snapshots: dict[str, SimpleNamespace] = {} - self.builds: list[str] = [] - self._deleting: dict[str, SimpleNamespace] = {} - self._deleting_gets: dict[str, int] = {} - - async def get(self, name: str) -> SimpleNamespace: - if name in self._deleting: - self._deleting_gets[name] -= 1 - if self._deleting_gets[name] >= 0: - return self._deleting[name] - del self._deleting[name], self._deleting_gets[name] - if name not in self.snapshots: - raise RuntimeError(f"snapshot {name} not found") - return self.snapshots[name] - - async def create(self, params: _CreateSnapshotParams) -> None: - if params.name in self.snapshots or params.name in self._deleting: - raise ValueError(f"snapshot with name {params.name} already exists") - image: Any = params.image - if isinstance(image, str): - record = SimpleNamespace(name=params.name, image_name=image, build_info=None) - else: - record = SimpleNamespace( - name=params.name, - image_name=None, - build_info=SimpleNamespace( - dockerfile_content=image.dockerfile(), - context_hashes=[ - await _tree_hash(entry.source_path) for entry in image._context_list - ], - ), - ) - self.snapshots[params.name] = record - self.builds.append(params.name) - - async def delete(self, snapshot: SimpleNamespace) -> None: - self._deleting[snapshot.name] = self.snapshots.pop(snapshot.name) - self._deleting_gets[snapshot.name] = 1 - - -class _FakeDaytonaClient: - def __init__(self, calls: dict[str, Any]) -> None: - self.calls = calls - self.sandbox = _FakeDaytonaSandbox(calls, "sandbox-1") - self.snapshot = _FakeSnapshotApi() - #: sandbox-create params in order; the last entry is the booted sandbox. - self.created: list[Any] = [] - self.delete_fails = False - - async def create(self, params: object, **kwargs: object) -> _FakeDaytonaSandbox: - self.calls["create"] = (params, kwargs["timeout"]) - self.created.append(params) - sandbox = ( - self.sandbox - if len(self.created) == 1 - else _FakeDaytonaSandbox(self.calls, f"sandbox-{len(self.created)}") - ) - return sandbox - - async def delete(self, sandbox: _FakeDaytonaSandbox) -> None: - if self.delete_fails: - raise RuntimeError("daytona API unreachable") - self.calls["delete"] = sandbox.id - deleted = self.calls.setdefault("deleted", []) - assert isinstance(deleted, list) - deleted.append(sandbox.id) - - -class _FakeSSHConnection: - def __init__(self, calls: dict[str, Any]) -> None: - self._calls = calls - - async def forward_local_port( - self, - listen_host: str, - listen_port: int, - dest_host: str, - dest_port: int, - ) -> SimpleNamespace: - self._calls["forward"] = (listen_host, listen_port, dest_host, dest_port) - return SimpleNamespace(get_port=lambda: 54321) - - -class _FakeSSHConnect: - def __init__( - self, - calls: dict[str, Any], - args: tuple[object, ...], - kwargs: dict[str, object], - ) -> None: - self._calls = calls - self._args = args - self._kwargs = kwargs - - async def __aenter__(self) -> _FakeSSHConnection: - self._calls["ssh_connect"] = (self._args, self._kwargs) - return _FakeSSHConnection(self._calls) - - async def __aexit__(self, *exc_info: object) -> None: - self._calls["ssh_closed"] = True - - -def _install_fake_daytona(monkeypatch: pytest.MonkeyPatch) -> _FakeDaytonaClient: - calls: dict[str, Any] = {} - client = _FakeDaytonaClient(calls) - daytona = ModuleType("daytona") - daytona_async = ModuleType("daytona._async") - object_storage = ModuleType("daytona._async.object_storage") - asyncssh = ModuleType("asyncssh") - - class AsyncDaytona: - async def __aenter__(self) -> _FakeDaytonaClient: - return client - - async def __aexit__(self, *exc_info: object) -> None: - calls["client_closed"] = True - - def connect(*args: object, **kwargs: object) -> _FakeSSHConnect: - return _FakeSSHConnect(calls, args, kwargs) - - setattr(daytona, "AsyncDaytona", AsyncDaytona) - setattr(daytona, "CreateSnapshotParams", _CreateSnapshotParams) - setattr(daytona, "CreateSandboxFromSnapshotParams", _CreateSandboxFromSnapshotParams) - setattr(daytona, "CreateSandboxFromImageParams", _CreateSandboxFromImageParams) - setattr(daytona, "DaytonaNotFoundError", RuntimeError) - setattr(daytona, "Image", SimpleNamespace(base=lambda name: _DaytonaImage(name))) - setattr(daytona, "Resources", _DaytonaResources) - setattr(daytona, "GpuType", _DaytonaGpuType) - setattr(daytona, "SessionExecuteRequest", _SessionExecuteRequest) - setattr(daytona, "_async", daytona_async) - setattr(daytona_async, "object_storage", object_storage) - setattr(object_storage, "AsyncObjectStorage", _FakeObjectStorage) - setattr(asyncssh, "connect", connect) - monkeypatch.setitem(sys.modules, "daytona", daytona) - monkeypatch.setitem(sys.modules, "daytona._async", daytona_async) - monkeypatch.setitem(sys.modules, "daytona._async.object_storage", object_storage) - monkeypatch.setitem(sys.modules, "asyncssh", asyncssh) - return client - - -async def test_acquisition_publishes_ephemeral_port_and_removes_container( - tmp_path: Path, - docker_log: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) - - provider = DockerRuntime("img:tag", run_args=("-e", "X=1")) - async with provider(_row()) as runtime: - assert runtime.url == "tcp://127.0.0.1:43210" - await asyncio.sleep(0.05) - calls = await _docker_calls(docker_log) - volume = calls[0].removeprefix("volume create ") - assert volume.startswith("hud-runtime-sessions-") - assert calls[1] == ( - f"run --detach -e X=1 {_docker_security_args()} {_session_mount_args(volume)} " - "--publish 127.0.0.1::8765 img:tag" - ) - assert calls[2] == "port cid-42 8765" - - assert (await _docker_calls(docker_log))[-2:] == [ - "rm --force cid-42", - f"volume rm --force {volume}", - ] - assert capsys.readouterr().out == "ImportError: boom\n" - - -async def test_docker_runtime_injects_provider_bind_mount( - tmp_path: Path, - docker_log: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) - bundle = tmp_path / "agents" / "codex" - bundle.mkdir(parents=True) - - provider = DockerRuntime( - "img:tag", - bind_mounts=(DockerBindMount(bundle, "/usr/local/lib/agents/codex"),), - ) - async with provider(_row()): - pass - - calls = await _docker_calls(docker_log) - volume = calls[0].removeprefix("volume create ") - assert calls[1] == ( - f"run --detach --mount type=bind,source={bundle}," - "target=/usr/local/lib/agents/codex,readonly " - f"{_docker_security_args()} {_session_mount_args(volume)} " - "--publish 127.0.0.1::8765 img:tag" - ) - - -@pytest.mark.parametrize( - ("source", "target", "message"), - [ - ("relative", "/opt/agents", "source must be absolute"), - ("/opt/agents", "relative", "target must be absolute"), - ("/opt/agents,old", "/opt/agents", "paths cannot contain commas"), - ], -) -def test_docker_bind_mount_rejects_ambiguous_paths( - source: str, - target: str, - message: str, -) -> None: - with pytest.raises(ValueError, match=message): - DockerBindMount(source, target) - - -async def test_docker_session_archives_inside_the_container( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: list[tuple[tuple[str, ...], bool]] = [] - - async def fake_docker(*args: str, check: bool = True) -> tuple[str, str]: - calls.append((args, check)) - if args[:3] == ("exec", "cid-42", "sh"): - return "1", "" - return "", "" - - monkeypatch.setattr(runtime_module, "_docker", fake_docker) - async with runtime_module.DockerEndpoint( - url="tcp://127.0.0.1:8765", - container="cid-42", - ).snapshot_session("sess-actor") as destination: - assert destination is not None - - probe, export, copy, cleanup = calls - assert probe[0][:3] == ("exec", "cid-42", "sh") - assert export[0][:6] == ("exec", "--user", "0", "cid-42", "python3", "-c") - assert "runtime session contains a symbolic link" in export[0][6] - assert export[0][7] == "/runtime/sessions/sess-actor" - archive = export[0][8] - assert copy == (("cp", f"cid-42:{archive}", str(destination)), True) - assert cleanup == ( - ("exec", "--user", "0", "cid-42", "rm", "-f", archive), - False, - ) - - -async def test_runtime_config_supplies_image_and_resources( - tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) - - task = Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig( - image="img:firefox", - resources=RuntimeResources(cpu=2, memory_mb=4096, gpu=RuntimeGPU()), - ), - ) - - async with DockerRuntime()(task) as runtime: - assert runtime.url == "tcp://127.0.0.1:43210" - assert runtime.config == task.runtime_config - - calls = await _docker_calls(docker_log) - volume = calls[0].removeprefix("volume create ") - assert calls[1] == ( - f"run --detach --cpus 2 --memory 4096m --gpus 1 {_docker_security_args()} " - f"{_session_mount_args(volume)} --publish 127.0.0.1::8765 img:firefox" - ) - - -async def test_task_runtime_config_overrides_default_image( - tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) - - task = Task(env="any-env", id="t", runtime_config=RuntimeConfig(image="img:task")) - - async with DockerRuntime( - "img:default", - runtime_config=RuntimeConfig( - resources=RuntimeResources(cpu=2, memory_mb=4096), - ), - )(task) as runtime: - assert runtime.config == RuntimeConfig( - image="img:task", - resources=RuntimeResources(cpu=2, memory_mb=4096), - ) - - calls = await _docker_calls(docker_log) - volume = calls[0].removeprefix("volume create ") - assert calls[1] == ( - f"run --detach --cpus 2 --memory 4096m {_docker_security_args()} " - f"{_session_mount_args(volume)} --publish 127.0.0.1::8765 img:task" - ) - - -def test_runtime_config_overrides_only_explicit_top_level_fields() -> None: - default = RuntimeConfig( - resources=RuntimeResources( - cpu=2, - memory_mb=4096, - gpu=RuntimeGPU(type=["A10G", "L4"], count=2), - ), - limits=RuntimeLimits(startup_timeout_s=30, run_timeout_s=120), - ) - - assert default.with_overrides(RuntimeConfig(image="img:task")) == RuntimeConfig( - image="img:task", - resources=RuntimeResources( - cpu=2, - memory_mb=4096, - gpu=RuntimeGPU(type=["A10G", "L4"], count=2), - ), - limits=RuntimeLimits(startup_timeout_s=30, run_timeout_s=120), - ) - assert default.with_overrides( - RuntimeConfig(resources=RuntimeResources(cpu=4)) - ) == RuntimeConfig( - resources=RuntimeResources(cpu=4), - limits=RuntimeLimits(startup_timeout_s=30, run_timeout_s=120), - ) - assert default.with_overrides(RuntimeConfig(resources=None)).resources is None - - -def test_runtime_config_source_override_is_mutually_exclusive(tmp_path: Path) -> None: - compose = tmp_path / "compose.yaml" - replacement = tmp_path / "replacement.yaml" - - assert RuntimeConfig(image="img:default").with_overrides( - RuntimeConfig(compose=ComposeProject(document=compose)) - ) == RuntimeConfig(compose=ComposeProject(document=compose)) - assert RuntimeConfig(compose=ComposeProject(document=compose)).with_overrides( - RuntimeConfig(image="img:task") - ) == RuntimeConfig(image="img:task") - assert RuntimeConfig(compose=ComposeProject(document=compose, root=tmp_path)).with_overrides( - RuntimeConfig(compose=ComposeProject(document=replacement)) - ) == RuntimeConfig(compose=ComposeProject(document=replacement)) - - -async def test_runtime_config_rejects_typed_docker_gpu() -> None: - with pytest.raises(ValueError, match="GPU"): - async with DockerRuntime()( - Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig( - image="img", - resources=RuntimeResources(gpu=RuntimeGPU(type="L40S")), - ), - ) - ): - pass - - -async def test_docker_runtime_starts_compose_with_a_main_service_override( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.delenv("DOCKER_HOST", raising=False) - calls: list[tuple[str, ...]] = [] - recipe: dict[str, Any] = {} - rendered: dict[str, Any] = {} - port_override = "" - compose = tmp_path / "compose.yaml" - compose.write_text( - "services:\n main:\n image: hud-env:one\n db:\n image: postgres:17\n", - encoding="utf-8", - ) - - async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: - nonlocal port_override - calls.append(args) - if args[:2] == ("context", "inspect"): - return "unix:///Users/test/.docker/run/docker.sock\n", "" - if args[-4:] == ("up", "--detach", "--build", "--remove-orphans"): - files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"] - compose_file, override, ports = files - recipe.update(json.loads(compose_file.read_text("utf-8"))) - rendered.update(json.loads(override.read_text("utf-8"))) - port_override = ports.read_text("utf-8") - if args[-3:] == ("port", "main", "8765"): - return "127.0.0.1:43210\n", "" - return "", "" - - monkeypatch.setattr(runtime_module, "_docker", fake_docker) - task = Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig( - compose=ComposeProject(document=compose, service_access=True), - resources=RuntimeResources(cpu=2, memory_mb=4096), - ), - ) - - async with DockerRuntime()(task) as runtime: - assert runtime.url == "tcp://127.0.0.1:43210" - assert runtime.config == task.runtime_config - - assert recipe["services"]["main"]["image"] == "hud-env:one" - assert rendered["services"]["main"] == { - "security_opt": [ - f"seccomp={runtime_module._DOCKER_SECCOMP_PROFILE}", - "systempaths=unconfined", - "apparmor=unconfined", - ], - "cpus": 2.0, - "mem_limit": "4096m", - "volumes": [ - { - "type": "volume", - "source": "hud-runtime-sessions", - "target": "/runtime/sessions", - }, - { - "type": "volume", - "source": "hud-runtime-sessions", - "target": "/media/hud/sessions", - }, - { - "type": "bind", - "source": "/Users/test/.docker/run/docker.sock", - "target": "/var/run/docker.sock", - }, - { - "type": "bind", - "source": "/Users/test/.docker/run/docker.sock", - "target": "/media/hud/docker.sock", - }, - ], - } - assert rendered["volumes"] == {"hud-runtime-sessions": {}} - assert 'ports: !override ["127.0.0.1::8765"]' in port_override - up = next( - call for call in calls if call[-4:] == ("up", "--detach", "--build", "--remove-orphans") - ) - assert up[up.index("--project-directory") + 1] == str(tmp_path) - assert str(compose) not in up - assert not any("--wait" in call for call in calls) - assert calls[-1][-3:] == ("down", "--volumes", "--remove-orphans") - - -async def test_docker_compose_startup_waits_for_default_services_with_healthchecks( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: list[tuple[str, ...]] = [] - compose = tmp_path / "compose.yaml" - compose.write_text( - """\ -services: - main: - image: hud-env:one - depends_on: - init: {condition: service_completed_successfully} - init: - image: alpine:3.20 - db: - image: postgres:17 - healthcheck: {test: [CMD, pg_isready]} - cache: - image: redis:7 - healthcheck: {test: [NONE]} - debug: - image: postgres:17 - profiles: [debug] - healthcheck: {test: [CMD, pg_isready]} - builder: - image: postgres:17 - scale: 0 - healthcheck: {test: [CMD, pg_isready]} -""", - encoding="utf-8", - ) - - async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: - calls.append(args) - if args[-3:] == ("port", "main", "8765"): - return "127.0.0.1:43210\n", "" - return "", "" - - monkeypatch.setattr(runtime_module, "_docker", fake_docker) - task = Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)), - ) - - async with DockerRuntime()(task): - pass - - subcommands = [call[call.index("up") :] for call in calls if "up" in call] - assert subcommands == [ - ("up", "--detach", "--build", "--remove-orphans"), - ("up", "--wait", "--no-deps", "--no-recreate", "--no-build", "db"), - ] - wait = next(index for index, call in enumerate(calls) if "--wait" in call) - port = next(index for index, call in enumerate(calls) if call[-3:] == ("port", "main", "8765")) - assert wait < port - - -async def test_docker_runtime_passes_env_vars_to_docker_run( - tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) - - provider = DockerRuntime("img:tag", env_vars={"OPENAI_API_KEY": "sk-test"}) - async with provider(_row()) as runtime: - assert runtime.url == "tcp://127.0.0.1:43210" - - calls = await _docker_calls(docker_log) - volume = calls[0].removeprefix("volume create ") - assert calls[1] == ( - f"run --detach --env OPENAI_API_KEY=sk-test {_docker_security_args()} " - f"{_session_mount_args(volume)} --publish 127.0.0.1::8765 img:tag" - ) - - -async def test_docker_runtime_stages_env_vars_into_the_compose_override( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - rendered: dict[str, Any] = {} - compose = tmp_path / "compose.yaml" - compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") - - async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: - if "up" in args: - files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"] - _, override, _ = files - rendered.update(json.loads(override.read_text("utf-8"))) - if args[-3:] == ("port", "main", "8765"): - return "127.0.0.1:43210\n", "" - return "", "" - - monkeypatch.setattr(runtime_module, "_docker", fake_docker) - task = Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)), - ) - provider = DockerRuntime(env_vars={"OPENAI_API_KEY": "sk-test"}) - - async with provider(task): - pass - - assert rendered["services"]["main"]["environment"] == {"OPENAI_API_KEY": "sk-test"} - - -async def test_docker_runtime_rejects_remote_compose_service_access( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") - monkeypatch.setenv("DOCKER_HOST", "tcp://docker.example:2376") - - with pytest.raises(ValueError, match="requires compose_service_socket"): - async with DockerRuntime()( - Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig( - compose=ComposeProject(document=compose, service_access=True), - ), - ) - ): - pass - - -async def test_docker_runtime_mounts_the_daemon_visible_socket( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") - monkeypatch.setenv("DOCKER_HOST", "tcp://docker.example:2376") - rendered: dict[str, Any] = {} - - async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: - if args[-4:] == ("up", "--detach", "--build", "--remove-orphans"): - files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"] - rendered.update(json.loads(files[1].read_text("utf-8"))) - if args[-3:] == ("port", "main", "8765"): - return "127.0.0.1:43210\n", "" - return "", "" - - monkeypatch.setattr(runtime_module, "_docker", fake_docker) - task = Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig(compose=ComposeProject(document=compose, service_access=True)), - ) - - async with DockerRuntime(compose_service_socket="/vm/run/docker.sock")(task): - pass - - assert rendered["services"]["main"]["volumes"] == [ - { - "type": "volume", - "source": "hud-runtime-sessions", - "target": "/runtime/sessions", - }, - { - "type": "volume", - "source": "hud-runtime-sessions", - "target": "/media/hud/sessions", - }, - { - "type": "bind", - "source": "/vm/run/docker.sock", - "target": "/var/run/docker.sock", - }, - { - "type": "bind", - "source": "/vm/run/docker.sock", - "target": "/media/hud/docker.sock", - }, - ] - - -async def test_docker_runtime_stages_provider_mount_with_service_socket( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") - bundle = tmp_path / "agents" / "claude" - bundle.mkdir(parents=True) - rendered: dict[str, Any] = {} - - async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: - if args[-4:] == ("up", "--detach", "--build", "--remove-orphans"): - files = [Path(args[index + 1]) for index, value in enumerate(args) if value == "--file"] - rendered.update(json.loads(files[1].read_text("utf-8"))) - if args[-3:] == ("port", "main", "8765"): - return "127.0.0.1:43210\n", "" - return "", "" - - monkeypatch.setattr(runtime_module, "_docker", fake_docker) - task = Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig(compose=ComposeProject(document=compose, service_access=True)), - ) - provider = DockerRuntime( - compose_service_socket="/vm/run/docker.sock", - bind_mounts=(DockerBindMount(bundle, "/usr/local/lib/agents/claude"),), - ) - - async with provider(task): - pass - - assert rendered["services"]["main"]["volumes"] == [ - { - "type": "volume", - "source": "hud-runtime-sessions", - "target": "/runtime/sessions", - }, - { - "type": "volume", - "source": "hud-runtime-sessions", - "target": "/media/hud/sessions", - }, - { - "type": "bind", - "source": str(bundle), - "target": "/usr/local/lib/agents/claude", - "read_only": True, - }, - { - "type": "bind", - "source": "/vm/run/docker.sock", - "target": "/var/run/docker.sock", - }, - { - "type": "bind", - "source": "/vm/run/docker.sock", - "target": "/media/hud/docker.sock", - }, - ] - - -def test_docker_runtime_accepts_only_one_environment_definition(tmp_path: Path) -> None: - with pytest.raises(ValueError, match="either image or compose"): - RuntimeConfig( - image="img:tag", - compose=ComposeProject(document=tmp_path / "compose.yaml"), - ) - - -def test_docker_runtime_accepts_runtime_config_defaults() -> None: - provider = DockerRuntime("img:tag") - assert provider.runtime_config == RuntimeConfig(image="img:tag") - - provider_with_resources = DockerRuntime( - "img:tag", - runtime_config=RuntimeConfig(resources=RuntimeResources(cpu=2)), - ) - assert provider_with_resources.runtime_config == RuntimeConfig( - image="img:tag", - resources=RuntimeResources(cpu=2), - ) - - provider = DockerRuntime("img:tag", runtime_config=RuntimeConfig(image="other:tag")) - assert provider.runtime_config == RuntimeConfig(image="other:tag") - - task = Task(env="any-env", id="t", runtime_config=RuntimeConfig(image="other:tag")) - assert provider_with_resources.runtime_config is not None - assert provider_with_resources.runtime_config.with_overrides( - task.runtime_config - ) == RuntimeConfig( - image="other:tag", - resources=RuntimeResources(cpu=2), - ) - - -async def test_modal_runtime_config_flows_into_modal_sdk( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - config = RuntimeConfig( - image="img:tag", - resources=RuntimeResources( - cpu=2, - memory_mb=4096, - gpu=RuntimeGPU(type="A10G", count=2), - ), - limits=RuntimeLimits(startup_timeout_s=30, run_timeout_s=120), - ) - provider = ModalRuntime(runtime_config=config) - - async with provider(_row()) as runtime: - assert runtime.url == "tcp://modal.host:4567" - assert runtime.params == {"provider": "modal", "instance_id": "sb-1"} - assert runtime.config == config - - assert calls["registry_image"] == "img:tag" - assert calls["app_lookup"] == ("hud-envs", True) - sandbox_args = calls["sandbox_args"] - assert sandbox_args[:2] == ("sh", "-c") - assert "ln -s /runtime/sessions /media/hud/sessions" in sandbox_args[2] - assert sandbox_args[-len(provider.command) :] == provider.command - assert calls["sandbox_kwargs"] == { - "app": "app", - "image": _ModalImageRef("registry", "img:tag"), - "workdir": None, - "unencrypted_ports": [8765], - "readiness_probe": ("tcp", 8765), - "timeout": 120, - "cpu": 2, - "memory": 4096, - "gpu": "A10G:2", - } - assert calls["ready_timeout"] == 30 - assert calls["terminated"] is True - - -async def test_modal_runtime_uses_caller_owned_app( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - app = object() - - async def build(*, app: object) -> None: - calls["image_build_app"] = app - - image = SimpleNamespace(build=SimpleNamespace(aio=build)) - provider = ModalRuntime(app=cast("Any", app), image=cast("Any", image)) - - async with provider(_row()): - pass - - assert "app_lookup" not in calls - assert calls["image_build_app"] is app - assert calls["sandbox_kwargs"]["app"] is app - - -def test_modal_runtime_rejects_app_and_app_name() -> None: - with pytest.raises(ValueError, match="either app or app_name"): - ModalRuntime(app=cast("Any", object()), app_name="hud-envs") - - -async def test_modal_runtime_rejects_gpu_alternatives() -> None: - provider = ModalRuntime( - runtime_config=RuntimeConfig( - image="img:tag", - resources=RuntimeResources(gpu=RuntimeGPU(type=["H100", "A100"])), - ) - ) - - with pytest.raises(ValueError, match="does not support alternative GPU types"): - async with provider(_row()): - pass - - -async def test_modal_runtime_runs_compose_inside_a_dind_vm( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - calls = _install_fake_modal(monkeypatch) - calls["compose_stdout"] = "main | server ready\n" - calls["compose_stderr"] = "grader | warning\n" - project = tmp_path / "artifact" - compose = project / "compose-project" / "compose.yaml" - compose.parent.mkdir(parents=True) - compose.write_text( - "services:\n main:\n image: hud-env:one\n" - " db:\n image: postgres:17\n healthcheck: {test: [CMD, pg_isready]}\n", - encoding="utf-8", - ) - - async with ModalRuntime( - runtime_config=RuntimeConfig( - compose=ComposeProject(document=compose, root=project, service_access=True), - limits=RuntimeLimits(startup_timeout_s=600), - ), - env_vars={"HUD_API_KEY": "secret"}, - )(_row()) as runtime: - assert runtime.url == "tcp://modal.host:4567" - assert runtime.params == {"provider": "modal", "instance_id": "sb-1", "ready_timeout": 600} - async with runtime.snapshot_session("sess-actor") as archive: - assert archive is not None - await runtime.restore_session("sess-verifier", archive) - - assert calls["registry_image"] == "docker:28.3.3-dind" - kwargs = calls["sandbox_kwargs"] - assert isinstance(kwargs, dict) - assert kwargs["experimental_options"] == {"vm_runtime": True} - assert kwargs["readiness_probe"] is None - assert kwargs["cpu"] == 4 - assert kwargs["memory"] == 8192 - assert kwargs["image"] == _ModalImageRef("registry", "docker:28.3.3-dind") - assert "env" not in kwargs - assert calls["uploads"] == [ - ("project.tar.gz", "/hud/project.tar.gz"), - ("override.json", "/hud/override.json"), - ("ports.yaml", "/hud/ports.yaml"), - ("docker-seccomp.json", "/hud/docker-seccomp.json"), - ("session.tar.gz", "/runtime/session.tar.gz"), - ] - override = calls["compose_override"] - assert isinstance(override, dict) - assert override["services"]["main"]["volumes"] == [ - { - "type": "volume", - "source": "hud-runtime-sessions", - "target": "/runtime/sessions", - }, - { - "type": "volume", - "source": "hud-runtime-sessions", - "target": "/media/hud/sessions", - }, - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - }, - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/media/hud/docker.sock", - }, - ] - assert override["volumes"] == {"hud-runtime-sessions": {}} - assert override["services"]["main"]["environment"] == {"HUD_API_KEY": "secret"} - execs = calls["execs"] - assert isinstance(execs, list) - startup = execs[0][0][-1] - assert "docker compose" in startup - assert "--project-directory /hud/project/compose-project" in startup - assert "--file /hud/project/compose-project/compose.yaml" in startup - assert "build.sh" not in startup - assert "up --detach --build --remove-orphans && " in startup - assert startup.endswith("up --wait --no-deps --no-recreate --no-build db") - compose_logs = execs[1][0] - assert compose_logs[-3:] == ("logs", "--follow", "--no-color") - session_commands = [ - call[0][-1] - for call in execs[1:-1] - if call[0][:2] == ("sh", "-c") and "docker compose" in call[0][-1] - ] - assert session_commands - assert all( - "--project-directory /hud/project/compose-project" in command - and "--file /hud/project/compose-project/compose.yaml" in command - for command in session_commands - ) - assert any("runtime session contains an unsupported entry" in call[0][-1] for call in execs) - teardown = execs[-1][0] - assert teardown[teardown.index("--project-directory") + 1] == ("/hud/project/compose-project") - assert teardown[teardown.index("--file") + 1] == ("/hud/project/compose-project/compose.yaml") - assert "down" in teardown - assert execs[0][1]["timeout"] == 600 - assert calls["downloads"] == [("/runtime/session.tar.gz", "session.tar.gz")] - captured = capsys.readouterr() - assert captured.out == "main | server ready\n" - assert captured.err == "grader | warning\n" - - -async def test_modal_runtime_bounds_compose_startup( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - calls["exec_wait"] = asyncio.Event() - compose = tmp_path / "compose.yaml" - compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") - - with pytest.raises(TimeoutError, match="Modal Compose startup timed out after 1 seconds"): - async with ModalRuntime( - runtime_config=RuntimeConfig( - compose=ComposeProject(document=compose), - limits=RuntimeLimits(startup_timeout_s=1), - ) - )(_row()): - pytest.fail("runtime should not become ready") - - assert calls["terminated"] is True - - -async def test_modal_runtime_rejects_gpu_inside_compose_dind(tmp_path: Path) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") - - provider = ModalRuntime( - runtime_config=RuntimeConfig( - compose=ComposeProject(document=compose), - resources=RuntimeResources(gpu=RuntimeGPU(type="H100")), - ) - ) - - with pytest.raises(ValueError, match=r"cannot attach GPUs.*Docker-in-Docker"): - async with provider(_row()): - pass - - -async def test_modal_runtime_accepts_modal_image_uri( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - config = RuntimeConfig(image="modal://im-built") - - async with ModalRuntime(runtime_config=config)(_row()) as runtime: - assert runtime.config == config - - assert calls["modal_image_id"] == "im-built" - assert calls["sandbox_kwargs"] == { - "app": "app", - "image": _ModalImageRef("id", "im-built"), - "workdir": None, - "unencrypted_ports": [8765], - "readiness_probe": ("tcp", 8765), - "timeout": 3600, - } - - -async def test_modal_runtime_outlives_the_task_agent_budget_without_a_declared_run_limit( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - task = Task( - env="any-env", - id="t", - agent_config={"timeout_seconds": 28800}, - runtime_config=RuntimeConfig(image="modal://im-built"), - ) - - async with ModalRuntime()(task): - pass - - assert calls["sandbox_kwargs"]["timeout"] == 28800 + 3600 - - declared = task.model_copy( - update={ - "runtime_config": RuntimeConfig( - image="modal://im-built", limits=RuntimeLimits(run_timeout_s=40000) - ) - } - ) - async with ModalRuntime()(declared): - pass - - assert calls["sandbox_kwargs"]["timeout"] == 40000 - - -async def test_modal_task_runtime_config_overlays_provider_defaults( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - provider = ModalRuntime( - runtime_config=RuntimeConfig( - resources=RuntimeResources(cpu=2, memory_mb=4096), - limits=RuntimeLimits(startup_timeout_s=30, run_timeout_s=120), - ), - ) - task = Task(env="any-env", id="t", runtime_config=RuntimeConfig(image="img:task")) - - async with provider(task) as runtime: - assert runtime.config == RuntimeConfig( - image="img:task", - resources=RuntimeResources(cpu=2, memory_mb=4096), - limits=RuntimeLimits(startup_timeout_s=30, run_timeout_s=120), - ) - - assert calls["registry_image"] == "img:task" - assert calls["ready_timeout"] == 30 - assert calls["sandbox_kwargs"] == { - "app": "app", - "image": _ModalImageRef("registry", "img:task"), - "workdir": None, - "unencrypted_ports": [8765], - "readiness_probe": ("tcp", 8765), - "timeout": 120, - "cpu": 2, - "memory": 4096, - } - - -async def test_modal_runtime_config_image_overrides_image_name( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - config = RuntimeConfig(image="img:tag", resources=RuntimeResources(gpu=RuntimeGPU())) - async with ModalRuntime("ignored-name", runtime_config=config)(_row()) as runtime: - assert runtime.config == config - - assert calls["registry_image"] == "img:tag" - - -async def test_modal_runtime_can_override_workdir( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - config = RuntimeConfig(image="img:tag") - provider = ModalRuntime(runtime_config=config, workdir="/app") - - async with provider(_row()) as runtime: - assert runtime.config == config - - sandbox_kwargs = calls["sandbox_kwargs"] - assert isinstance(sandbox_kwargs, dict) - assert sandbox_kwargs["workdir"] == "/app" - - -async def test_modal_runtime_passes_env_vars_to_sandbox( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - config = RuntimeConfig(image="img:tag") - provider = ModalRuntime(runtime_config=config, env_vars={"TOKEN": "secret"}) - - async with provider(_row()) as runtime: - assert runtime.config == config - - sandbox_kwargs = calls["sandbox_kwargs"] - assert isinstance(sandbox_kwargs, dict) - assert sandbox_kwargs["image"] == _ModalImageRef("registry", "img:tag") - assert sandbox_kwargs["env"] == {"TOKEN": "secret"} - - -async def test_modal_runtime_streams_sandbox_output_to_terminal( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - calls = _install_fake_modal(monkeypatch) - calls["sandbox_stdout"] = "server ready\nrequest complete" - calls["sandbox_stderr"] = "grader warning\n" - provider = ModalRuntime(runtime_config=RuntimeConfig(image="img:tag")) - - async with provider(_row()): - pass - - captured = capsys.readouterr() - assert captured.out == "server ready\nrequest complete" - assert captured.err == "grader warning\n" - - -async def test_modal_runtime_bounds_output_teardown( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - calls["sandbox_stdout_wait"] = asyncio.Event() - calls["sandbox_stderr_wait"] = asyncio.Event() - monkeypatch.setattr(process_module, "OUTPUT_DRAIN_TIMEOUT", 0.01) - - async def run() -> None: - async with ModalRuntime(runtime_config=RuntimeConfig(image="img:tag"))(_row()): - pass - - await asyncio.wait_for(run(), timeout=1) - assert calls["terminated"] is True - - -async def test_modal_runtime_attaches_sandbox_secrets( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - secret = _ModalSecretRef("datacuration-cpu50-grader-auth") - provider = ModalRuntime( - runtime_config=RuntimeConfig(image="img:tag"), - sandbox_secrets=[cast("Any", secret)], - ) - - async with provider(_row()): - pass - - sandbox_kwargs = calls["sandbox_kwargs"] - assert isinstance(sandbox_kwargs, dict) - assert sandbox_kwargs["secrets"] == (secret,) - assert calls["registry_secret"] is None - - -async def test_modal_runtime_passes_registry_secret_to_from_registry( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_modal(monkeypatch) - registry_secret = _ModalSecretRef("registry-creds") - provider = ModalRuntime( - runtime_config=RuntimeConfig(image="ghcr.io/org/private:tag"), - registry_secret=cast("Any", registry_secret), - ) - - async with provider(_row()): - pass - - assert calls["registry_image"] == "ghcr.io/org/private:tag" - assert calls["registry_secret"] is registry_secret - - -async def test_modal_runtime_rejects_sandbox_secrets_for_compose( - tmp_path: Path, -) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text("services:\n main:\n image: hud-env:one\n", encoding="utf-8") - provider = ModalRuntime( - runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)), - sandbox_secrets=[cast("Any", _ModalSecretRef("grader-auth"))], - ) - - with pytest.raises(ValueError, match="sandbox secrets require an image runtime"): - async with provider(_row()): - pass - - -async def test_daytona_runtime_config_flows_into_daytona_sdk( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_daytona(monkeypatch).calls - config = RuntimeConfig( - image="img:tag", - resources=RuntimeResources( - cpu=2, - memory_mb=4096, - gpu=RuntimeGPU(type=["H100", "A100"], count=2), - ), - limits=RuntimeLimits(startup_timeout_s=45), - ) - provider = DaytonaRuntime(runtime_config=config) - - async with provider(_row()) as runtime: - assert runtime.url == "tcp://127.0.0.1:54321" - assert runtime.params == {"provider": "daytona", "instance_id": "sandbox-1"} - assert runtime.config == config - - create_call = calls["create"] - assert isinstance(create_call, tuple) - create_params, create_timeout = create_call - assert create_params == _CreateSandboxFromImageParams( - image=_DaytonaImage("img:tag"), - ephemeral=True, - auto_stop_interval=0, - resources=_DaytonaResources( - cpu=2, - memory=4, - gpu=2, - gpu_type=[_DaytonaGpuType("H100"), _DaytonaGpuType("A100")], - ), - ) - assert create_timeout == 45 - assert calls["session"] == "hud-serve" - assert calls["execute"] == ( - "hud-serve", - _SessionExecuteRequest( - command=( - 'cd /app && PATH="$PWD/.venv/bin:$PATH" hud serve env.py --host 0.0.0.0 --port 8765' - ), - run_async=True, - ), - ) - assert calls["ssh_expires"] == 24 * 60 - assert calls["ssh_connect"] == ( - ("ssh.app.daytona.io",), - {"username": "ssh-token", "known_hosts": None}, - ) - assert calls["forward"] == ("127.0.0.1", 0, "127.0.0.1", 8765) - assert calls["delete"] == "sandbox-1" - - -async def test_daytona_runtime_streams_sandbox_output_to_terminal( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - calls = _install_fake_daytona(monkeypatch).calls - calls["daytona_stdout"] = "server ready\n" - calls["daytona_stderr"] = "grader warning\n" - - async with DaytonaRuntime("snapshot")(_row()): - pass - - captured = capsys.readouterr() - assert captured.out == "server ready\n" - assert captured.err == "grader warning\n" - assert calls["stream_logs"] == ("hud-serve", "cmd-1") - - -async def test_daytona_runtime_bounds_output_teardown( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_daytona(monkeypatch).calls - calls["daytona_log_wait"] = asyncio.Event() - monkeypatch.setattr(process_module, "OUTPUT_DRAIN_TIMEOUT", 0.01) - - async def run() -> None: - async with DaytonaRuntime("snapshot")(_row()): - pass - - await asyncio.wait_for(run(), timeout=1) - assert calls["delete"] == "sandbox-1" - - -async def test_daytona_runtime_rejects_compose( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - client = _install_fake_daytona(monkeypatch) - compose = tmp_path / "compose.yaml" - compose.write_text("services: {}\n", encoding="utf-8") - - with pytest.raises(ValueError, match=r"does not support runtime_config\.compose"): - async with DaytonaRuntime( - runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)) - )(_row()): - pass - - assert client.created == [] - - -def test_compose_config_normalizes_supported_short_syntax(tmp_path: Path) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text( - """ -services: - main: - image: example:latest - environment: - - FIRST=one - - EMPTY= - entrypoint: /bin/start --flag - command: serve --port 8000 - expose: [8000] - ports: - - 8000 - - 127.0.0.1:8080:80/tcp -""", - encoding="utf-8", - ) - - service = ComposeConfig.from_file(compose).services["main"] - - assert service.environment == {"FIRST": "one", "EMPTY": ""} - assert service.entrypoint == ["/bin/start", "--flag"] - assert service.command == ["serve", "--port", "8000"] - assert service.expose == ["8000"] - assert [port.model_dump(exclude_none=True) for port in service.ports] == [ - {"target": 8000, "protocol": "tcp"}, - { - "target": 80, - "protocol": "tcp", - "published": 8080, - "host_ip": "127.0.0.1", - }, - ] - - mapped = ComposeService.model_validate({"environment": {"COUNT": 2, "ENABLED": True}}) - assert mapped.environment == {"COUNT": "2", "ENABLED": "true"} - - -def test_compose_service_tcp_ports_excludes_udp_expose() -> None: - service = ComposeService.model_validate({"expose": [80, "443/tcp", "53/udp"]}) - - assert service.tcp_ports == {80, 443} - - -@pytest.mark.parametrize( - "service", - [ - "extends: {file: base.yaml, service: base}", - ], -) -def test_compose_config_rejects_external_resolution(tmp_path: Path, service: str) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text(f"services:\n main:\n {service}\n", encoding="utf-8") - - with pytest.raises(ValueError, match="does not support Compose"): - ComposeConfig.from_file(compose) - - -def test_compose_config_interpolates_only_artifact_supplied_values( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("HOST_ONLY", "secret") - (tmp_path / ".env").write_text( - "REGISTRY=example\nIMAGE=${REGISTRY}:1\nEMPTY=\nTOKEN=cost$5\n", - encoding="utf-8", - ) - compose = tmp_path / "compose.yaml" - compose.write_text( - """ -services: - main: - image: ${IMAGE} - command: "${EMPTY:-serve} $$HOME ${MISSING-default} $? $" - environment: - LITERAL: '$HOST_ONLY' - TOKEN: ${TOKEN} -""", - encoding="utf-8", - ) - - config = ComposeConfig.from_file(compose) - service = config.services["main"] - - assert service.image == "example:1" - assert service.command == ["serve", "$$HOME", "default", "$$?", "$$"] - assert service.environment == {"LITERAL": "$$HOST_ONLY", "TOKEN": "cost$$5"} - - normalized = tmp_path / "normalized.json" - normalized.write_text(json.dumps(config.model_dump(mode="json")), encoding="utf-8") - assert ComposeConfig.from_file(normalized) == config - - -@pytest.mark.parametrize( - ("image", "message"), - [ - ("$HOST_ONLY", r"HOST_ONLY.*not set by the project \.env"), - ("${IMAGE:?set IMAGE in .env}", r"set IMAGE in \.env"), - ], -) -def test_compose_config_rejects_unbound_variables( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - image: str, - message: str, -) -> None: - monkeypatch.setenv("HOST_ONLY", "secret") - compose = tmp_path / "compose.yaml" - compose.write_text( - f"services:\n main:\n image: {image}\n", - encoding="utf-8", - ) - - with pytest.raises(ValueError, match=message): - ComposeConfig.from_file(compose) - - -def test_compose_config_relocates_project_paths() -> None: - compose = ComposeConfig.model_validate( - { - "services": { - "main": { - "build": { - "context": ".", - "additional_contexts": { - "fixture": "./fixture", - "base": "service:base", - }, - }, - "env_file": [".env", {"path": "./optional.env", "required": False}], - "label_file": "./labels", - "volumes": [ - "./data:/data:ro", - "cache:/cache", - {"type": "bind", "source": "./config", "target": "/config"}, - ], - }, - "base": {"image": "alpine"}, - }, - "configs": {"settings": {"file": "./settings.json"}}, - "secrets": {"token": {"file": "./token"}}, - } - ) - - relocated = compose.with_project_directory("./environment").model_dump( - mode="json", exclude_none=True - ) - main = relocated["services"]["main"] - - assert main["build"] == { - "context": "./environment", - "additional_contexts": { - "fixture": "./environment/fixture", - "base": "service:base", - }, - } - assert main["env_file"] == [ - "./environment/.env", - {"path": "./environment/optional.env", "required": False}, - ] - assert main["label_file"] == "./environment/labels" - assert main["volumes"] == [ - "./environment/data:/data:ro", - "cache:/cache", - {"type": "bind", "source": "./environment/config", "target": "/config"}, - ] - assert relocated["configs"] == {"settings": {"file": "./environment/settings.json"}} - assert relocated["secrets"] == {"token": {"file": "./environment/token"}} - - -@pytest.mark.parametrize("path", ["../data", "./nested/../../data"]) -def test_compose_config_rejects_paths_outside_project(path: str) -> None: - compose = ComposeConfig.model_validate({"services": {"main": {"volumes": [f"{path}:/data"]}}}) - - with pytest.raises(ValueError, match="escapes its project directory"): - compose.with_project_directory("./environment") - - -def test_compose_healthcheck_does_not_invent_duration_values() -> None: - service = ComposeService( - healthcheck=ComposeHealthcheck(test=["CMD", "healthcheck"]), - ) - - assert service.model_dump(exclude_none=True)["healthcheck"] == {"test": ["CMD", "healthcheck"]} - - -async def test_daytona_task_runtime_config_overlays_provider_defaults( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_daytona(monkeypatch).calls - provider = DaytonaRuntime( - runtime_config=RuntimeConfig( - resources=RuntimeResources(cpu=2, memory_mb=4096), - limits=RuntimeLimits(startup_timeout_s=45), - ), - ) - task = Task(env="any-env", id="t", runtime_config=RuntimeConfig(image="img:task")) - - async with provider(task) as runtime: - assert runtime.config == RuntimeConfig( - image="img:task", - resources=RuntimeResources(cpu=2, memory_mb=4096), - limits=RuntimeLimits(startup_timeout_s=45), - ) - - create_call = calls["create"] - assert isinstance(create_call, tuple) - create_params, create_timeout = create_call - assert create_params == _CreateSandboxFromImageParams( - image=_DaytonaImage("img:task"), - ephemeral=True, - auto_stop_interval=0, - resources=_DaytonaResources(cpu=2, memory=4), - ) - assert create_timeout == 45 - - -async def test_daytona_snapshot_sandboxes_disable_auto_stop( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls = _install_fake_daytona(monkeypatch).calls - - async with DaytonaRuntime("snapshot")(_row()): - pass - - create_call = calls["create"] - assert isinstance(create_call, tuple) - create_params, create_timeout = create_call - assert create_params == _CreateSandboxFromSnapshotParams( - snapshot="snapshot", - ephemeral=True, - auto_stop_interval=0, - ) - assert create_timeout == 120 - - -@dataclass(frozen=True) -class _BuildContextEntry: - source_path: str - archive_path: str - - -class _BuildImage: - def __init__(self, context: Path) -> None: - self._context_list = [_BuildContextEntry(source_path=str(context), archive_path=".")] - - def dockerfile(self) -> str: - return "FROM python:3.11-slim\nCOPY . .\n" - - -def _build_image(context: Path) -> _BuildImage: - """A stand-in for ``daytona.Image.from_dockerfile``: the Dockerfile text plus - the context entries the SDK would archive and upload.""" - return _BuildImage(context) - - -async def _boot_snapshot(context: Path, daytona: _FakeDaytonaClient) -> str: - """Acquire once through a fresh provider; return the snapshot it booted.""" - async with DaytonaRuntime("env", image=cast("DaytonaImage", _build_image(context)))(_row()): - pass - return daytona.created[-1].snapshot - - -async def test_daytona_rebuilds_the_snapshot_in_place_when_the_env_changes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - # The name is the durable handle; content drift rebuilds it, never renames it — - # otherwise every rollout after an edit silently measures the old code. - daytona = _install_fake_daytona(monkeypatch) - (tmp_path / "env.py").write_text("REWARD = 1.0\n") - - first = await _boot_snapshot(tmp_path, daytona) - unchanged = await _boot_snapshot(tmp_path, daytona) - (tmp_path / "env.py").write_text("REWARD = 0.0\n") - edited = await _boot_snapshot(tmp_path, daytona) - - assert first == unchanged == edited == "env" - # Built once per distinct content; the unchanged re-acquisition reused it, - # and the stale build was replaced, not left behind. - assert daytona.snapshot.builds == ["env", "env"] - assert list(daytona.snapshot.snapshots) == ["env"] - - -async def test_daytona_registry_ref_image_builds_and_reuses_by_name( - monkeypatch: pytest.MonkeyPatch, -) -> None: - # image= is documented as "Dockerfile/registry ref"; a plain ref string must - # build once, reuse while unchanged, and rebuild when repointed. - daytona = _install_fake_daytona(monkeypatch) - - for ref in ("registry/env:1", "registry/env:1", "registry/env:2"): - async with DaytonaRuntime("env", image=ref)(_row()): - pass - - assert daytona.snapshot.builds == ["env", "env"] - assert daytona.snapshot.snapshots["env"].image_name == "registry/env:2" - - -async def test_daytona_sends_cpu_as_a_whole_number(monkeypatch: pytest.MonkeyPatch) -> None: - # Daytona's API rejects 2.0, and no equality assertion can catch it: 2.0 == 2. - daytona = _install_fake_daytona(monkeypatch) - config = RuntimeConfig(image="img:tag", resources=RuntimeResources(cpu=2)) - - async with DaytonaRuntime(runtime_config=config)(_row()): - pass - - assert isinstance(daytona.created[-1].resources.cpu, int) - - -async def test_daytona_rejects_a_fractional_cpu_request(monkeypatch: pytest.MonkeyPatch) -> None: - _install_fake_daytona(monkeypatch) - config = RuntimeConfig(image="img:tag", resources=RuntimeResources(cpu=1.5)) - - with pytest.raises(ValueError, match="whole number of CPUs"): - async with DaytonaRuntime(runtime_config=config)(_row()): - pass - - -async def test_daytona_rounds_minimum_storage_up_to_gibibytes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - daytona = _install_fake_daytona(monkeypatch) - config = RuntimeConfig(image="img:tag", resources=RuntimeResources(storage_mb=1025)) - - async with DaytonaRuntime(runtime_config=config)(_row()): - pass - - assert daytona.created[-1].resources.disk == 2 - - -async def test_modal_accepts_best_effort_storage( - monkeypatch: pytest.MonkeyPatch, -) -> None: - config = RuntimeConfig(image="img:tag", resources=RuntimeResources(storage_mb=1024)) - calls = _install_fake_modal(monkeypatch) - - async with ModalRuntime(runtime_config=config)(_row()): - pass - - assert calls["sandbox_kwargs"] == { - "app": "app", - "image": _ModalImageRef("registry", "img:tag"), - "workdir": None, - "unencrypted_ports": [8765], - "readiness_probe": ("tcp", 8765), - "timeout": 3600, - } - - -async def test_docker_admits_minimum_free_disk( - tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _install_fake_docker( - tmp_path, - port_behavior="echo 127.0.0.1:43210", - monkeypatch=monkeypatch, - ) - config = RuntimeConfig(image="img:tag", resources=RuntimeResources(storage_mb=1024)) - - async with DockerRuntime(runtime_config=config)(_row()): - pass - - calls = await _docker_calls(docker_log) - assert "exec cid-42 df -Pk /" in calls - - -async def test_docker_honors_startup_timeout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: list[tuple[tuple[str, ...], dict[str, Any]]] = [] - - async def fake_docker(*args: str, **kwargs: Any) -> tuple[str, str]: - calls.append((args, kwargs)) - if args[0] == "run": - return "cid-42\n", "" - if args[0] == "port": - return "127.0.0.1:43210\n", "" - return "", "" - - monkeypatch.setattr(runtime_module, "_docker", fake_docker) - config = RuntimeConfig(image="img:tag", limits=RuntimeLimits(startup_timeout_s=300)) - - async with DockerRuntime(runtime_config=config)(_row()) as runtime: - assert runtime.params == {"ready_timeout": 300} - - run = next(call for call in calls if call[0][0] == "run") - assert run[1]["deadline"] == 300 - - -@pytest.mark.parametrize("compose", [False, True]) -async def test_docker_run_timeout_owns_shared_substrate_lifetime( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - *, - compose: bool, -) -> None: - torn_down = asyncio.Event() - teardown_calls: list[tuple[str, ...]] = [] - provisions = 0 - - async def fake_docker(*args: str, **kwargs: Any) -> tuple[str, str]: - nonlocal provisions - if args[0] == "run": - provisions += 1 - return "cid-42\n", "" - if "up" in args: - provisions += 1 - if args[0] == "port" or args[-3:] == ("port", "main", "8765"): - return "127.0.0.1:43210\n", "" - if args[-3:] == ("ps", "--quiet", "main"): - return "cid-42\n", "" - if args[:2] == ("rm", "--force") or "down" in args: - teardown_calls.append(args) - torn_down.set() - return "", "" - - monkeypatch.setattr(runtime_module, "_docker", fake_docker) - - limits = RuntimeLimits(run_timeout_s=1) - if compose: - compose_file = tmp_path / "compose.yaml" - compose_file.write_text("services:\n main:\n image: img:tag\n") - source = RuntimeConfig(compose=ComposeProject(document=compose_file), limits=limits) - else: - source = RuntimeConfig(image="img:tag", limits=limits) - - async with Shared(DockerRuntime(runtime_config=source), width=2) as runtime: - async with runtime(_row()): - pass - async with runtime(_row()): - await asyncio.wait_for(torn_down.wait(), 2) - async with runtime(_row()): - pass - - assert teardown_calls - assert provisions == 2 - - -async def test_daytona_names_a_sandbox_it_could_not_delete( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture -) -> None: - # Teardown must report the leak without shadowing the run's own error. - daytona = _install_fake_daytona(monkeypatch) - daytona.delete_fails = True - - with caplog.at_level(logging.WARNING, logger="hud.eval.runtime"): - async with DaytonaRuntime("snapshot")(_row()): - pass - - assert "sandbox-1" in caplog.text - - -async def test_daytona_sizes_each_row_from_its_own_runtime_config( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - # Resources are baked into a snapshot at build time, so rows that request - # different sizes must boot distinct snapshots, not the first row's. - daytona = _install_fake_daytona(monkeypatch) - (tmp_path / "env.py").write_text("REWARD = 1.0\n") - provider = DaytonaRuntime("env", image=cast("DaytonaImage", _build_image(tmp_path))) - - for cpu in (2, 4): - task = Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig(resources=RuntimeResources(cpu=cpu)), - ) - async with provider(task): - pass - - assert daytona.snapshot.builds == ["env-2cpu", "env-4cpu"] - assert [params.snapshot for params in daytona.created] == ["env-2cpu", "env-4cpu"] - - -async def test_daytona_attaches_the_env_output_to_a_failed_handshake( - monkeypatch: pytest.MonkeyPatch, -) -> None: - # The caller gets "closed connection during 'hello'" and nothing else. - _install_fake_daytona(monkeypatch) - - with pytest.raises(EOFError) as excinfo: - async with DaytonaRuntime("snapshot")(_row()): - raise EOFError("env closed connection during 'hello'") - - assert any("no module named bugs" in note for note in excinfo.value.__notes__) - - -async def test_daytona_runtime_config_rejects_unsupported_fields( - monkeypatch: pytest.MonkeyPatch, -) -> None: - _install_fake_daytona(monkeypatch) - with pytest.raises(ValueError, match="resources"): - async with DaytonaRuntime( - "snapshot", - runtime_config=RuntimeConfig(resources=RuntimeResources(cpu=1)), - )(_row()): - pass - - with pytest.raises(ValueError, match="run_timeout_s"): - async with DaytonaRuntime( - "snapshot", - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=60)), - )(_row()): - pass - - with pytest.raises(ValueError, match="run_timeout_s"): - async with DaytonaRuntime( - runtime_config=RuntimeConfig( - image="img:tag", - limits=RuntimeLimits(run_timeout_s=60), - ), - )(_row()): - pass - - -async def test_container_that_dies_before_serving_fails_with_its_logs( - tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - # ``docker port`` on an exited container prints nothing. - _install_fake_docker(tmp_path, port_behavior=":", monkeypatch=monkeypatch) - - provider = DockerRuntime("img:tag") - with pytest.raises(RuntimeError, match="exited before serving") as err: - async with provider(_row()): - pass - - assert "ImportError: boom" in str(err.value) - calls = await _docker_calls(docker_log) - assert "logs --tail 40 cid-42" in calls - assert calls[-2] == "rm --force cid-42" - assert calls[-1].startswith("volume rm --force hud-runtime-sessions-") - - -def test_docker_profile_allows_workspace_namespace_syscalls() -> None: - profile = json.loads(runtime_module._DOCKER_SECCOMP_PROFILE.read_text()) - denied = {name for rule in profile["syscalls"] for name in rule["names"]} - ptrace = next(rule for rule in profile["syscalls"] if "ptrace" in rule["names"]) - - assert profile["defaultAction"] == "SCMP_ACT_ALLOW" - assert { - "mount", - "pivot_root", - "setns", - "umount", - "umount2", - "unshare", - }.isdisjoint(denied) - assert { - "bpf", - "keyctl", - "perf_event_open", - "ptrace", - "userfaultfd", - } <= denied - assert ptrace["excludes"] == {"caps": ["CAP_SYS_PTRACE"]} - - -async def test_docker_runtime_always_prepares_for_workspace_isolation( - tmp_path: Path, docker_log: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _install_fake_docker(tmp_path, port_behavior="echo 127.0.0.1:43210", monkeypatch=monkeypatch) - - async with DockerRuntime("img:tag")(_row()): - pass - - calls = await _docker_calls(docker_log) - run = next(call for call in calls if call.startswith("run ")) - assert f"seccomp={runtime_module._DOCKER_SECCOMP_PROFILE}" in run - assert "seccomp=unconfined" not in run - assert "systempaths=unconfined" in run - assert "apparmor=unconfined" in run - - -def test_compose_network_owner_follows_service_chains_and_stages_its_port( - tmp_path: Path, -) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text( - "services:\n" - " main:\n image: hud:latest\n network_mode: service:relay\n" - " relay:\n image: relay:latest\n network_mode: service:gateway\n" - " gateway:\n image: gateway:latest\n", - encoding="utf-8", - ) - config = ComposeConfig.from_file(compose) - - assert config.network_owner("main") == "gateway" - with ComposeProject(document=compose).stage( - "127.0.0.1::8765", - port_service=config.network_owner("main"), - seccomp="profile.json", - ) as files: - assert " gateway:" in files.ports.read_text("utf-8") - - -async def test_compose_startup_failure_uses_network_owner_logs( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text( - "services:\n" - " main:\n image: hud:latest\n network_mode: service:gateway\n" - " gateway:\n image: gateway:latest\n", - encoding="utf-8", - ) - calls: list[tuple[str, ...]] = [] - - async def fake_docker(*args: str, **_kwargs: Any) -> tuple[str, str]: - calls.append(args) - if args[-4:] == ("logs", "--tail", "40", "gateway"): - return "gateway failed\n", "" - return "", "" - - monkeypatch.setattr(runtime_module, "_docker", fake_docker) - task = Task( - env="any-env", - id="t", - runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)), - ) - - with pytest.raises(RuntimeError, match="Compose gateway service exited") as error: - async with DockerRuntime()(task): - pass - - assert "gateway failed" in str(error.value) - assert any(call[-4:] == ("logs", "--tail", "40", "gateway") for call in calls) - - -def test_compose_stage_archives_the_normalized_document(tmp_path: Path) -> None: - project = tmp_path / "project" - recipe = project / "recipe" / "compose.yaml" - recipe.parent.mkdir(parents=True) - (recipe.parent / ".env").write_text("IMAGE=example:1\n", encoding="utf-8") - recipe.write_text( - "services:\n main:\n image: ${IMAGE}\n environment:\n HOME: '$HOME'\n", - encoding="utf-8", - ) - - with ComposeProject(document=recipe, root=project).stage( - "127.0.0.1::8765", - seccomp="profile.json", - archive=True, - ) as files: - assert files.archive is not None - assert files.project_directory == recipe.parent.resolve() - assert json.loads(files.compose.read_text("utf-8"))["services"]["main"] == { - "image": "example:1", - "environment": {"HOME": "$$HOME"}, - "expose": [], - "ports": [], - "volumes": [], - } - with tarfile.open(files.archive, "r:gz") as archive: - archived = archive.extractfile("recipe/compose.yaml") - assert archived is not None - assert json.load(archived)["services"]["main"]["image"] == "example:1" - - assert "${IMAGE}" in recipe.read_text("utf-8") - - -@pytest.mark.parametrize( - ("services", "message"), - [ - ( - { - "main": {"network_mode": "service:relay"}, - "relay": {"network_mode": "service:main"}, - }, - "cycle", - ), - ({"main": {"network_mode": "service:missing"}}, "unknown service"), - ], -) -def test_compose_network_owner_rejects_invalid_graphs( - services: dict[str, dict[str, str]], - message: str, -) -> None: - config = ComposeConfig.model_validate({"services": services}) - - with pytest.raises(ValueError, match=message): - config.network_owner("main") - - -async def test_docker_rejects_insufficient_free_disk( - tmp_path: Path, - docker_log: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - _install_fake_docker( - tmp_path, - port_behavior="echo 127.0.0.1:43210", - monkeypatch=monkeypatch, - ) - config = RuntimeConfig(image="img:tag", resources=RuntimeResources(storage_mb=8192)) - - with pytest.raises(RuntimeError, match=r"requires 8192 MB.*has 4096 MB"): - async with DockerRuntime(runtime_config=config)(_row()): - pass - - assert (await _docker_calls(docker_log))[-2] == "rm --force cid-42" diff --git a/hud/eval/tests/test_file_tracking_observer.py b/hud/eval/tests/test_file_tracking_observer.py deleted file mode 100644 index b348cb5d4..000000000 --- a/hud/eval/tests/test_file_tracking_observer.py +++ /dev/null @@ -1,323 +0,0 @@ -"""file_tracking_observer: setup must gate polling.""" - -from __future__ import annotations - -import asyncio -import json -from typing import TYPE_CHECKING, Any, cast - -from hud.capabilities import Capability -from hud.environment.file_tracker import FileTracker -from hud.eval import file_tracking as observer -from hud.telemetry.context import set_trace_context - -if TYPE_CHECKING: - import pytest - - from hud.clients.client import HudClient - -_CAP = Capability( - name="filetracking", - protocol="filetracking/1", - url="tcp://127.0.0.1:1", - params={"setup_diff": True}, -) -_LEGACY_CAP = Capability(name="filetracking", protocol="filetracking/1", url=_CAP.url) - - -class _FakeFt: - def __init__( - self, - *, - setup_raises: bool = False, - flush_result: dict[str, Any] | None = None, - ) -> None: - self.setup_raises = setup_raises - self.flush_result = flush_result - self.advance_calls = 0 - self.setup_calls = 0 - self.snapshot_calls = 0 - self.diff_calls = 0 - self.flush_calls = 0 - self.close_calls = 0 - - async def close(self) -> None: - self.close_calls += 1 - - async def call(self, method: str) -> dict[str, Any]: - if method == "advance": - self.advance_calls += 1 - return {"advanced": True} - if method == "setup": - self.setup_calls += 1 - if self.setup_raises: - raise RuntimeError("setup boom") - return {"files_changed": 1, "patches": [{"path": "setup.txt"}]} - if method == "snapshot": - self.snapshot_calls += 1 - return {"files": [], "files_scanned": 0} - if method == "diff": - self.diff_calls += 1 - return {"files_changed": 1, "patches": [{"path": "a.txt"}]} - if method == "flush": - self.flush_calls += 1 - if self.flush_result is not None: - return self.flush_result - return { - "diff": {"files_changed": 1, "patches": [{"path": "final.txt"}]}, - "capture": {"files_captured": 1, "files": [{"path": "report.xlsx"}]}, - } - raise ValueError(method) - - -class _BoundClient: - def __init__(self, cap: Capability = _CAP) -> None: - self.cap = cap - - def binding(self, name: str) -> Capability: - return self.cap - - -def _record_emitters( - monkeypatch: pytest.MonkeyPatch, -) -> tuple[list[Any], list[Any], list[Any], list[Any]]: - diffs: list[Any] = [] - setups: list[Any] = [] - snapshots: list[Any] = [] - captures: list[Any] = [] - - def _emit( - span_name: str, - payload: Any, - *, - started_at: str, - ended_at: str | None = None, - ) -> bool: - if span_name == "filetracking.diff": - diffs.append(payload) - elif span_name == "filetracking.setup": - setups.append(payload) - elif span_name == "filetracking.snapshot": - snapshots.append(payload) - elif span_name == "filetracking.capture": - captures.append(payload) - return True - - monkeypatch.setattr(observer, "_emit_file_tracking", _emit) - return diffs, setups, snapshots, captures - - -def _connects_to( - monkeypatch: pytest.MonkeyPatch, - ft: _FakeFt, - cap: Capability = _CAP, -) -> None: - class _FakeFileTrackingClient: - @classmethod - async def connect(cls, requested_cap: Capability) -> _FakeFt: - assert requested_cap == cap - return ft - - monkeypatch.setattr(observer, "FileTrackingClient", _FakeFileTrackingClient) - - -async def test_setup_failure_skips_polling(monkeypatch: pytest.MonkeyPatch) -> None: - from hud.settings import settings - - monkeypatch.setattr(settings, "telemetry_enabled", True) - monkeypatch.setattr(settings, "file_tracking_interval", 0.01) - diffs, setups, snapshots, captures = _record_emitters(monkeypatch) - ft = _FakeFt(setup_raises=True) - _connects_to(monkeypatch, ft) - - async with observer.file_tracking_observer(cast("HudClient", _BoundClient())): - await asyncio.sleep(0.05) - - assert ft.setup_calls == 1 - assert ft.snapshot_calls == 0 - assert ft.diff_calls == 0 - assert ft.flush_calls == 0 - assert ft.close_calls == 1 - assert diffs == [] - assert snapshots == [] - assert setups == [] - assert captures == [] - - -async def test_successful_setup_anchors_and_polls(monkeypatch: pytest.MonkeyPatch) -> None: - from hud.settings import settings - - monkeypatch.setattr(settings, "telemetry_enabled", True) - monkeypatch.setattr(settings, "file_tracking_interval", 0.01) - diffs, setups, snapshots, captures = _record_emitters(monkeypatch) - ft = _FakeFt() - _connects_to(monkeypatch, ft) - - async with observer.file_tracking_observer(cast("HudClient", _BoundClient())): - await asyncio.sleep(0.05) - - assert ft.setup_calls == 1 - assert setups == [{"files_changed": 1, "patches": [{"path": "setup.txt"}]}] - assert len(snapshots) == 1 # manifest anchor emitted once - assert ft.diff_calls >= 1 - assert ft.flush_calls == 1 - assert ft.close_calls == 1 - assert diffs # at least one diff streamed - assert {"files_changed": 1, "patches": [{"path": "final.txt"}]} in diffs - assert captures == [{"files_captured": 1, "files": [{"path": "report.xlsx"}]}] - - -async def test_legacy_server_advances_without_setup_event( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.settings import settings - - monkeypatch.setattr(settings, "telemetry_enabled", True) - monkeypatch.setattr(settings, "file_tracking_interval", 60.0) - diffs, setups, snapshots, captures = _record_emitters(monkeypatch) - ft = _FakeFt() - _connects_to(monkeypatch, ft, _LEGACY_CAP) - - async with observer.file_tracking_observer(cast("HudClient", _BoundClient(_LEGACY_CAP))): - pass - - assert ft.advance_calls == 1 - assert ft.setup_calls == 0 - assert ft.snapshot_calls == 1 - assert setups == [] - assert len(snapshots) == 1 - assert diffs == [{"files_changed": 1, "patches": [{"path": "final.txt"}]}] - assert captures == [{"files_captured": 1, "files": [{"path": "report.xlsx"}]}] - - -async def test_flush_emits_skipped_capture_metadata(monkeypatch: pytest.MonkeyPatch) -> None: - from hud.settings import settings - - capture = { - "files_changed": 1, - "files_eligible": 1, - "files_captured": 0, - "files_skipped": 1, - "truncated": True, - "files": [], - } - monkeypatch.setattr(settings, "telemetry_enabled", True) - monkeypatch.setattr(settings, "file_tracking_interval", 60.0) - diffs, setups, snapshots, captures = _record_emitters(monkeypatch) - ft = _FakeFt(flush_result={"diff": {"files_changed": 0, "patches": []}, "capture": capture}) - _connects_to(monkeypatch, ft) - - async with observer.file_tracking_observer(cast("HudClient", _BoundClient())): - pass - - assert ft.flush_calls == 1 - assert len(snapshots) == 1 - assert len(setups) == 1 - assert diffs == [] - assert captures == [capture] - - -async def test_connect_failure_does_not_break_the_rollout(monkeypatch: pytest.MonkeyPatch) -> None: - from hud.settings import settings - - monkeypatch.setattr(settings, "telemetry_enabled", True) - diffs, setups, snapshots, captures = _record_emitters(monkeypatch) - - class _FailingFileTrackingClient: - @classmethod - async def connect(cls, cap: Capability) -> object: - assert cap == _CAP - raise ConnectionError("tunnel refused") - - monkeypatch.setattr(observer, "FileTrackingClient", _FailingFileTrackingClient) - - # A failed connection must degrade to a no-op, not raise into the agent loop. - ran = False - async with observer.file_tracking_observer(cast("HudClient", _BoundClient())): - ran = True - - assert ran - assert diffs == [] - assert snapshots == [] - assert setups == [] - assert captures == [] - - -async def test_file_tracking_client_reads_large_snapshot_frame() -> None: - large_path = "a" * (80 * 1024) - - async def handle( - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, - ) -> None: - await reader.readline() - writer.write( - json.dumps( - { - "jsonrpc": "2.0", - "id": 1, - "result": { - "files_scanned": 1, - "files": [{"path": large_path, "sha256": "0" * 64, "size": 1}], - }, - }, - separators=(",", ":"), - ).encode("utf-8") - + b"\n" - ) - await writer.drain() - writer.close() - await writer.wait_closed() - - server = await asyncio.start_server(handle, "127.0.0.1", 0) - host, port = server.sockets[0].getsockname()[:2] - cap = Capability(name="filetracking", protocol="filetracking/1", url=f"tcp://{host}:{port}") - try: - ft = await observer.FileTrackingClient.connect(cap) - snapshot = await ft.call("snapshot") - assert snapshot["files"][0]["path"] == large_path - finally: - server.close() - await server.wait_closed() - - -def test_file_tracking_frame_limit_covers_flush_caps() -> None: - encoded_capture_cap = 4 * ((FileTracker._MAX_CAPTURE_TOTAL_BYTES + 2) // 3) - escaped_diff_cap = 2 * FileTracker._MAX_DIFF_BYTES - assert escaped_diff_cap + encoded_capture_cap <= observer._FRAME_LIMIT_BYTES - - -def test_emit_file_tracking_noops_without_a_trace_context( - monkeypatch: pytest.MonkeyPatch, -) -> None: - captured: list[dict[str, Any]] = [] - monkeypatch.setattr(observer, "queue_span", captured.append) - - emitted = observer._emit_file_tracking( - "filetracking.diff", - {"files_changed": 1}, - started_at="2026-06-18T22:00:00Z", - ) - - assert emitted is False - assert captured == [] - - -def test_emit_file_tracking_builds_schema_tagged_span(monkeypatch: pytest.MonkeyPatch) -> None: - captured: list[dict[str, Any]] = [] - monkeypatch.setattr(observer, "queue_span", captured.append) - - with set_trace_context("run-abc-123"): - emitted = observer._emit_file_tracking( - "filetracking.capture", - {"files_captured": 1, "files": [{"path": "report.xlsx"}]}, - started_at="2026-06-18T22:00:00Z", - ) - - assert emitted is True - span = captured[0] - assert span["name"] == "filetracking.capture" - assert span["attributes"]["hud.schema"] == "hud.filetracking.v1" - assert span["attributes"]["hud.task_run_id"] == "run-abc-123" - assert span["attributes"]["hud.payload"]["files"][0]["path"] == "report.xlsx" - assert len(span["trace_id"]) == 32 diff --git a/hud/eval/tests/test_hosted.py b/hud/eval/tests/test_hosted.py deleted file mode 100644 index f5b3abe36..000000000 --- a/hud/eval/tests/test_hosted.py +++ /dev/null @@ -1,1052 +0,0 @@ -"""HUD-hosted placement: agent spec, submission/polling, and scheduler dispatch. - -The hosted path never opens a local connection — :class:`HostedRuntime` submits the -rollout to the platform, polls the trace until terminal, and folds the result -into a ``Run``. The scheduler (:meth:`Taskset.run`) chooses between ``HostedRuntime`` -and a local provider. These tests fake the platform client at the -``PlatformClient`` seam, so they cover everything local: spec serialization, -payload shape, id canonicalization, terminal detection, timeout cancel, the -Run the caller gets back, and the dispatch. -""" - -from __future__ import annotations - -import asyncio -import json -import uuid -from typing import TYPE_CHECKING, Any, ClassVar, cast - -import pytest - -from hud.agents.base import Agent -from hud.agents.claude import ClaudeCLIAgent, ClaudeCLIConfig -from hud.agents.openai_compatible import OpenAIChatAgent -from hud.agents.types import OpenAIChatConfig -from hud.eval.job import Job -from hud.eval.run import Run -from hud.eval.runtime import ( - ComposeProject, - HostedRuntime, - HUDRuntime, - ModalRuntime, - Runtime, - RuntimeConfig, - RuntimeGPU, - RuntimeLimits, - RuntimeResources, -) -from hud.eval.runtime.core import resolve_runtime_config -from hud.eval.runtime.hud import _splice_websocket -from hud.eval.task import Task -from hud.eval.taskset import Taskset - -if TYPE_CHECKING: - from pathlib import Path - -from hud.settings import settings -from hud.telemetry.context import set_trace_context - - -class _FakePlatform: - """Scripted PlatformClient: records posts, serves trace states in order.""" - - api_key = "test-key" - - def __init__(self, states: list[dict[str, Any]]) -> None: - self.states = states - self.posts: list[tuple[str, dict[str, Any]]] = [] - self.polled = 0 - - async def apost(self, path: str, *, json: Any | None = None) -> Any: - self.posts.append((path, json or {})) - return {"status": "queued"} - - async def aget(self, path: str, *, params: dict[str, Any] | None = None) -> Any: - assert path.startswith("/trace/") and path.count("/") == 2 - state = self.states[min(self.polled, len(self.states) - 1)] - self.polled += 1 - return state - - -class _FakeResponse: - def __init__(self, body: dict[str, Any]) -> None: - self.body = body - - def raise_for_status(self) -> None: - return None - - def json(self) -> dict[str, Any]: - return self.body - - -def _agent() -> OpenAIChatAgent: - return OpenAIChatAgent( - OpenAIChatConfig(model="test-model", api_key="k", base_url="http://localhost") - ) - - -@pytest.mark.parametrize("runtime_type", [HostedRuntime, HUDRuntime]) -def test_runtime_constructor_timeout_is_a_deprecated_alias(runtime_type: type[Any]) -> None: - with pytest.warns(DeprecationWarning, match="rollout_timeout"): - runtime = runtime_type(run_timeout=90.0) - - assert runtime.run_timeout == 90.0 - - -def test_dump_serializes_full_config() -> None: - agent = _agent() - agent.config.system_prompt = "be brief" - agent.config.max_steps = 7 - agent.config.timeout_seconds = 3600 - agent.config.tool_timeout_seconds = 1800 - - config = agent.dump() - - # The full config travels, so every knob is preserved... - assert config["model"] == "test-model" - assert config["max_steps"] == 7 - assert config["system_prompt"] == "be brief" - assert config["timeout_seconds"] == 3600 - assert config["tool_timeout_seconds"] == 1800 - # ...minus what can't or shouldn't cross the wire. - assert "model_client" not in config - assert "api_key" not in config - assert "base_url" not in config - assert "hosted_tools" not in config - - -def test_dump_preserves_training_config( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The constructor builds the runtime client without putting it in config.""" - from hud.agents import create_agent - from hud.utils.gateway import GatewayModelInfo, GatewayProviderInfo - - class _GatewayStub: - pass - - client = _GatewayStub() - model = GatewayModelInfo( - id="arith-rl", - model_name="arith-rl", - sdk_agent_type="openai_compatible", - provider=GatewayProviderInfo(name="openai"), - ) - monkeypatch.setattr("hud.utils.gateway.list_gateway_models", lambda *_: [model]) - monkeypatch.setattr("hud.agents.settings.api_key", "test-key") - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", lambda _provider: client) - - agent = create_agent( - "arith-rl", - system_prompt="/no_think", - completion_kwargs={ - "extra_body": { - "return_token_ids": True, - "chat_template_kwargs": {"enable_thinking": False}, - } - }, - ) - assert isinstance(agent, OpenAIChatAgent) - assert agent.config.model_client is None - assert agent.oai is client - - config = agent.dump() - assert config["model"] == "arith-rl" - assert config["system_prompt"] == "/no_think" - assert config["completion_kwargs"] == { - "extra_body": { - "return_token_ids": True, - "chat_template_kwargs": {"enable_thinking": False}, - } - } - assert "model_client" not in config - - -def test_dump_rejects_custom_model_client() -> None: - agent = _agent() - agent.config = OpenAIChatConfig(model="m", model_client=object()) - with pytest.raises(ValueError, match=r"custom model_client.*HUDRuntime"): - agent.dump() - - -@pytest.mark.asyncio -async def test_run_rejects_unregistered_agent() -> None: - """An agent that can't serialize its identity yields a failed Run, not a crash.""" - - class _LocalOnlyAgent(Agent): - async def __call__(self, run: Run) -> None: - run.trace.content = "done" - - run = await HostedRuntime(poll_interval=0.0).run( - Task(env="e", id="x"), - _LocalOnlyAgent(), - job_id="j", - ) - assert run.trace.is_error - assert "registered agent types" in (run.trace.error or "") - assert "_LocalOnlyAgent" in (run.trace.error or "") - - -@pytest.mark.asyncio -async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform( - [ - {"status": "pending"}, - {"status": "running"}, - {"status": "completed", "reward": 0.5}, - ] - ) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - hosted = HostedRuntime(poll_interval=0.0) - trace_id = uuid.uuid4().hex - job_id = uuid.uuid4().hex - parent_trace_id = uuid.uuid4().hex - task = Task( - env="sums", - id="add", - slug="sums-add", - args={"a": 1, "b": 2}, - agent_config={"timeout_seconds": 45.0}, - runtime_config=RuntimeConfig( - image="registry.example/sums:latest", - resources=RuntimeResources(cpu=2, gpu=RuntimeGPU(type="L4", count=1)), - limits=RuntimeLimits(startup_timeout_s=120, run_timeout_s=900), - ), - verifier=Task( - env="judge", - id="verify", - args={"expected": 3}, - runtime_config=RuntimeConfig(resources=RuntimeResources(memory_mb=4096)), - ), - ) - - with set_trace_context(parent_trace_id): - run = await hosted.run(task, _agent(), job_id=job_id, group_id="g1", trace_id=trace_id) - - assert run.reward == 0.5 - assert run.trace.status == "completed" - assert run.trace.trace_id == trace_id - assert run.job_id == job_id - assert run.group_id == "g1" - assert run.slug == task.slug - assert Job(id="job", name="test", runs=[run]).results == {"sums-add": [run]} - assert platform.polled == 3 - (path, payload) = platform.posts[0] - assert path == "/rollouts/submit" - # Hex ids travel as canonical UUID strings. - assert payload["trace_id"] == str(uuid.UUID(trace_id)) - assert payload["job_id"] == str(uuid.UUID(job_id)) - assert payload["parent_trace_id"] == str(uuid.UUID(parent_trace_id)) - assert payload["env"] == "sums" - assert payload["task"] == "add" - assert payload["slug"] == "sums-add" - assert payload["args"] == {"a": 1, "b": 2} - assert payload["runtime_config"] == { - "image": "registry.example/sums:latest", - "resources": {"cpu": 2.0, "gpu": {"type": "L4", "count": 1}}, - "limits": {"startup_timeout_s": 120, "run_timeout_s": 900}, - } - assert payload["verifier"] == { - "env": "judge", - "id": "verify", - "args": {"expected": 3}, - "slug": "verify-5579a3e5", - "runtime_config": {"resources": {"memory_mb": 4096}}, - } - assert payload["group_id"] == "g1" - assert payload["agent"]["type"] == "openai_compatible" - assert payload["agent"]["config"]["model"] == "test-model" - assert payload["agent"]["config"]["timeout_seconds"] == 45.0 - - -@pytest.mark.asyncio -async def test_run_submits_registered_cli_agent(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - agent = ClaudeCLIAgent( - ClaudeCLIConfig( - model="claude-sonnet-4-6", - max_steps=23, - gateway=True, - ) - ) - - run = await HostedRuntime(poll_interval=0.0).run( - Task(env="coding", id="solve"), - agent, - job_id=uuid.uuid4().hex, - trace_id=uuid.uuid4().hex, - ) - - assert run.reward == 1.0 - submitted = platform.posts[0][1]["agent"] - assert submitted["type"] == "claude_cli" - assert submitted["config"]["model"] == "claude-sonnet-4-6" - assert submitted["config"]["max_steps"] == 23 - assert submitted["config"]["gateway"] is True - - -@pytest.mark.asyncio -async def test_run_preserves_runtime_config_null_override( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 0.5}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - await HostedRuntime(poll_interval=0.0).run( - Task(env="sums", id="add", runtime_config=RuntimeConfig(resources=None)), - _agent(), - job_id=uuid.uuid4().hex, - trace_id=uuid.uuid4().hex, - ) - - assert platform.posts[0][1]["runtime_config"] == {"resources": None} - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("parent_trace_id", "trace_id"), - [ - ("external-run-id", "00000000000000000000000000000001"), - ( - "00000000-0000-0000-0000-000000000001", - "00000000000000000000000000000001", - ), - ( - "{00000000-0000-0000-0000-000000000001}", - "00000000000000000000000000000001", - ), - ( - "urn:uuid:00000000-0000-0000-0000-000000000001", - "00000000000000000000000000000001", - ), - ], -) -async def test_run_omits_unusable_parent_trace_id( - monkeypatch: pytest.MonkeyPatch, - parent_trace_id: str, - trace_id: str, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 0.5}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - with set_trace_context(parent_trace_id): - run = await HostedRuntime(poll_interval=0.0).run( - Task(env="sums", id="add"), - _agent(), - job_id=uuid.uuid4().hex, - trace_id=trace_id, - ) - - assert run.trace.status == "completed" - assert "parent_trace_id" not in platform.posts[0][1] - - -@pytest.mark.asyncio -async def test_run_submits_compose_document( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - compose = tmp_path / "compose.json" - compose.write_text( - json.dumps( - { - "services": { - "main": {"image": "ghcr.io/hud-evals/harbor-main:latest"}, - "database": {"image": "postgres:16"}, - } - } - ), - encoding="utf-8", - ) - platform = _FakePlatform([{"status": "completed", "reward": 0.5}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - await HostedRuntime(poll_interval=0.0).run( - Task( - env="harbor", - id="solve", - runtime_config=RuntimeConfig( - compose=ComposeProject(document=compose, service_access=True) - ), - ), - _agent(), - job_id=uuid.uuid4().hex, - trace_id=uuid.uuid4().hex, - ) - - runtime_config = platform.posts[0][1]["runtime_config"] - assert runtime_config["compose"]["document"]["services"]["database"]["image"] == "postgres:16" - assert runtime_config["compose"]["service_access"] is True - assert str(compose) not in json.dumps(runtime_config) - - -@pytest.mark.asyncio -async def test_run_timeout_requests_platform_cancel(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "running"}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - hosted = HostedRuntime(poll_interval=0.0) - task = Task(env="sums", id="add", args={}) - - run = await hosted.run( - task, - _agent(), - job_id=uuid.uuid4().hex, - rollout_timeout=0.001, - ) - await asyncio.sleep(0) - - cancel_posts = [(p, b) for p, b in platform.posts if p == "/rollouts/cancel"] - assert len(cancel_posts) == 1 - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - - -@pytest.mark.asyncio -async def test_omitted_rollout_timeout_allows_long_environment_timeout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=18_000)), - ) - - run = await HostedRuntime(poll_interval=0.0).run( - task, - _agent(), - job_id=uuid.uuid4().hex, - ) - - assert run.trace.status == "completed" - assert platform.posts[0][0] == "/rollouts/submit" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("environment_timeout", [3_600, 18_000]) -async def test_explicit_rollout_timeout_rejects_environment_timeout_at_or_beyond_it( - monkeypatch: pytest.MonkeyPatch, - environment_timeout: int, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=environment_timeout)), - ) - - with pytest.raises(ValueError, match=r"actor runtime_config\.limits\.run_timeout_s"): - await HostedRuntime().run( - task, - _agent(), - job_id=uuid.uuid4().hex, - rollout_timeout=3_600, - ) - - assert platform.posts == [] - - -@pytest.mark.asyncio -async def test_taskset_rollout_timeout_reaches_hosted_runtime( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=18_000)), - ) - - job = await Taskset("sums", [task]).run( - _agent(), - runtime=HostedRuntime(poll_interval=0.0), - rollout_timeout=18_600, - ) - - assert job.runs[0].trace.status == "completed" - assert platform.posts[0][0] == "/rollouts/submit" - - -@pytest.mark.asyncio -async def test_agent_timeout_must_fit_explicit_environment_timeout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - agent_config={"timeout_seconds": 5_000}, - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=3_600)), - ) - - with pytest.raises(ValueError, match=r"agent timeout \(5000s\).+run_timeout_s \(3600s\)"): - await HostedRuntime().run(task, _agent(), job_id=uuid.uuid4().hex) - - assert platform.posts == [] - - -@pytest.mark.asyncio -async def test_agent_timeout_must_fit_explicit_rollout_timeout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - agent_config={"timeout_seconds": 5_000}, - ) - - with pytest.raises(ValueError, match=r"agent timeout \(5000s\).+rollout_timeout \(3600s\)"): - await HostedRuntime().run( - task, - _agent(), - job_id=uuid.uuid4().hex, - rollout_timeout=3_600, - ) - - assert platform.posts == [] - - -@pytest.mark.asyncio -async def test_agent_timeout_must_fit_provider_runtime_timeout() -> None: - task = Task( - env="sums", - id="add", - agent_config={"timeout_seconds": 900}, - ) - runtime = ModalRuntime(runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=600))) - - with pytest.raises(ValueError, match=r"agent timeout \(900s\).+run_timeout_s \(600s\)"): - await Taskset("sums", [task]).run( - _agent(), - runtime=runtime, - rollout_timeout=1_200, - ) - - -def test_task_runtime_timeout_overrides_provider_runtime_timeout() -> None: - runtime = ModalRuntime(runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=600))) - task = Task( - env="sums", - id="add", - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=1_000)), - ) - - config = resolve_runtime_config(runtime, task) - - assert config is not None - assert config.limits == RuntimeLimits(run_timeout_s=1_000) - - -@pytest.mark.asyncio -async def test_provider_startup_timeout_must_fit_rollout_timeout() -> None: - task = Task( - env="sums", - id="add", - agent_config={"timeout_seconds": 30}, - ) - runtime = ModalRuntime( - runtime_config=RuntimeConfig(limits=RuntimeLimits(startup_timeout_s=600)) - ) - - with pytest.raises( - ValueError, - match=r"actor runtime_config\.limits\.startup_timeout_s \(600s\)", - ): - await Taskset("sums", [task]).run( - _agent(), - runtime=runtime, - rollout_timeout=600, - ) - - -@pytest.mark.asyncio -async def test_rollout_timeout_validates_separable_verifier_limits() -> None: - task = Task( - env="actor", - id="solve", - verifier=Task( - env="judge", - id="verify", - runtime_config=RuntimeConfig(limits=RuntimeLimits(startup_timeout_s=90)), - ), - ) - - with pytest.raises( - ValueError, - match=r"verifier runtime_config\.limits\.startup_timeout_s \(90s\)", - ): - await Taskset("separable", [task]).run( - _agent(), - runtime=Runtime("tcp://127.0.0.1:1"), - rollout_timeout=90, - ) - - -@pytest.mark.asyncio -async def test_submit_timeout_requests_platform_cancel(monkeypatch: pytest.MonkeyPatch) -> None: - never = asyncio.Event() - - class _StuckSubmitPlatform(_FakePlatform): - async def apost(self, path: str, *, json: Any | None = None) -> Any: - self.posts.append((path, json or {})) - if path == "/rollouts/submit": - await never.wait() - return {"status": "queued"} - - platform = _StuckSubmitPlatform([]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - run = await HostedRuntime().run( - Task(env="sums", id="add"), - _agent(), - job_id=uuid.uuid4().hex, - rollout_timeout=0.001, - ) - await asyncio.sleep(0) - - assert run.trace.stop_reason == "timeout" - assert any(path == "/rollouts/cancel" for path, _ in platform.posts) - - -@pytest.mark.asyncio -async def test_run_folds_completed_receipt(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0, "error": None}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - task = Task(env="sums", id="add", args={"a": 2, "b": 3}) - run = await HostedRuntime(poll_interval=0.0).run(task, _agent(), job_id=uuid.uuid4().hex) - - assert run.reward == 1.0 - assert run.trace.status == "completed" - assert not run.trace.is_error - assert run.runtime == f"hud://trace/{run.trace.trace_id}" - # The platform owns the trace lifecycle: no local client ever existed. - with pytest.raises(RuntimeError, match="no live client"): - _ = run.client - - -@pytest.mark.asyncio -@pytest.mark.parametrize("is_error", [False, True]) -async def test_run_preserves_structured_grade( - monkeypatch: pytest.MonkeyPatch, is_error: bool -) -> None: - result = { - "score": 0.5, - "info": {"passed": 3}, - "content": "verifier failed" if is_error else "partial credit", - "isError": is_error, - "subscores": [{"name": "accuracy", "weight": 1.0, "value": 0.5}], - } - platform = _FakePlatform( - [ - { - "status": "error" if is_error else "completed", - "reward": 0.5, - "evaluation_result": result, - } - ] - ) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - run = await HostedRuntime(poll_interval=0.0).run( - Task(env="sums", id="add"), _agent(), job_id=uuid.uuid4().hex - ) - assert run.reward == 0.5 - assert run.grade.raw == result - assert run.grade.info == {"passed": 3} - assert run.grade.is_error is is_error - assert run.grade.content == result["content"] - assert platform.polled == 1 - - -@pytest.mark.asyncio -async def test_run_reports_malformed_grade_as_a_failed_run(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "completed", "evaluation_result": {"score": "bad"}}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task(env="sums", id="add") - run = await HostedRuntime(poll_interval=0.0).run(task, _agent(), job_id=uuid.uuid4().hex) - assert run.trace.is_error - assert "numeric 'score'" in (run.trace.error or "") - assert run.slug == task.slug - - -@pytest.mark.asyncio -async def test_run_folds_error_receipt(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "error", "reward": None, "error": "env exploded"}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - task = Task(env="sums", id="add", args={}) - run = await HostedRuntime(poll_interval=0.0).run(task, _agent(), job_id=uuid.uuid4().hex) - - assert run.reward == 0.0 - assert run.trace.is_error - assert "env exploded" in (run.trace.error or "") - - -@pytest.mark.asyncio -async def test_run_keeps_a_grade_from_an_errored_hosted_trace( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "error", "reward": 0.75, "error": "agent timed out"}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - task = Task(env="sums", id="add", args={}) - run = await HostedRuntime(poll_interval=0.0).run( - task, - _agent(), - job_id=uuid.uuid4().hex, - ) - job = Job(id="job", name="test", runs=[run]) - - assert run.trace.is_error - assert not run.grade.is_error - assert run.evaluation == {"score": 0.75} - assert job.reward == 0.75 - assert job.errors == [] - - -@pytest.mark.asyncio -async def test_run_folds_ungraded_cancellation_as_an_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "cancelled", "reward": None, "error": None}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - task = Task(env="sums", id="add", args={}) - run = await HostedRuntime(poll_interval=0.0).run( - task, - _agent(), - job_id=uuid.uuid4().hex, - ) - job = Job(id="job", name="test", runs=[run]) - - assert run.trace.status == "cancelled" - assert run.grade.is_error - assert job.reward == 0.0 - assert job.errors == [run] - - -@pytest.mark.asyncio -async def test_scheduler_drives_provider_locally(monkeypatch: pytest.MonkeyPatch) -> None: - """A Provider placement goes through the local rollout atom, not HostedRuntime.""" - import hud.eval.taskset as taskset_mod - - seen: dict[str, Any] = {} - - async def fake_rollout(task: Task, agent: Any, **kwargs: Any) -> Run: - seen.update(kwargs) - run = Run(None, task.id, {}) - run.trace.status = "completed" - return run - - monkeypatch.setattr(taskset_mod, "rollout", fake_rollout) - - job = await Taskset("t", [Task(env="e", id="x")]).run( - _agent(), runtime=Runtime("tcp://127.0.0.1:1") - ) - - assert len(job.runs) == 1 - assert isinstance(seen["runtime"], Runtime) - assert "job_id" in seen and "group_id" in seen - - -@pytest.mark.asyncio -async def test_scheduler_delegates_hosted(monkeypatch: pytest.MonkeyPatch) -> None: - """A HostedRuntime placement is delegated to via HostedRuntime.run, not the local atom.""" - seen: dict[str, Any] = {} - - class _RecordingHostedRuntime(HostedRuntime): - async def run(self, task: Task, agent: Agent, **kwargs: Any) -> Run: - seen.update(kwargs) - run = Run(None, task.id, {}) - run.trace.status = "completed" - return run - - job = await Taskset("t", [Task(env="e", id="x")]).run( - _agent(), runtime=_RecordingHostedRuntime(), rollout_timeout=90.0 - ) - - assert len(job.runs) == 1 - assert "job_id" in seen and "group_id" in seen - assert seen["rollout_timeout"] == 90.0 - - -@pytest.mark.asyncio -async def test_hud_runtime_drives_local_rollout(monkeypatch: pytest.MonkeyPatch) -> None: - seen: dict[str, Any] = {} - - async def fake_rollout(task: Task, agent: Any, **kwargs: Any) -> Run: - seen.update(kwargs) - run = Run(None, task.id, {}) - run.trace.status = "completed" - return run - - monkeypatch.setattr("hud.eval.runtime.hud.rollout", fake_rollout) - - runtime = HUDRuntime() - job_id = uuid.uuid4().hex - trace_id = uuid.uuid4().hex - run = await runtime.run( - Task(env="e", id="x"), - _agent(), - job_id=job_id, - group_id="g1", - trace_id=trace_id, - rollout_timeout=90.0, - ) - - assert run.trace.status == "completed" - assert seen["runtime"] is runtime - assert seen["job_id"] == job_id - assert seen["group_id"] == "g1" - assert seen["trace_id"] == trace_id - assert seen["rollout_timeout"] == 90.0 - - with pytest.raises(ValueError, match="placement requirements"): - async with runtime( - Task( - env="e", - id="x", - runtime_config=RuntimeConfig(resources=RuntimeResources(gpu=RuntimeGPU())), - ) - ): - pass - - -@pytest.mark.asyncio -async def test_runtime_session_create_payload_omits_trace_id( - monkeypatch: pytest.MonkeyPatch, -) -> None: - posts: list[dict[str, Any]] = [] - session_id = str(uuid.uuid4()) - - class _RecordingAsyncClient: - def __init__(self, *args: Any, **kwargs: Any) -> None: - pass - - async def __aenter__(self) -> _RecordingAsyncClient: - return self - - async def __aexit__(self, *args: Any) -> None: - return None - - async def post( - self, - path: str, - *, - headers: dict[str, str], - json: dict[str, Any], - ) -> _FakeResponse: - posts.append({"path": path, "headers": headers, "json": json}) - return _FakeResponse({"id": session_id}) - - monkeypatch.setattr("hud.eval.runtime.hud.httpx.AsyncClient", _RecordingAsyncClient) - - created = await HUDRuntime()._create_runtime_session( - "https://mcp.hud.ai", - "sk-hud-test", - Task(env="e", id="x"), - ) - - assert created == session_id - assert posts == [ - { - "path": "https://mcp.hud.ai/runtime/sessions", - "headers": {"Authorization": "Bearer sk-hud-test"}, - "json": {"environment": "e"}, - } - ] - - -@pytest.mark.asyncio -async def test_runtime_session_create_payload_includes_current_trace_id( - monkeypatch: pytest.MonkeyPatch, -) -> None: - posts: list[dict[str, Any]] = [] - session_id = str(uuid.uuid4()) - trace_id = uuid.uuid4().hex - - class _RecordingAsyncClient: - def __init__(self, *args: Any, **kwargs: Any) -> None: - pass - - async def __aenter__(self) -> _RecordingAsyncClient: - return self - - async def __aexit__(self, *args: Any) -> None: - return None - - async def post( - self, - path: str, - *, - headers: dict[str, str], - json: dict[str, Any], - ) -> _FakeResponse: - posts.append({"path": path, "headers": headers, "json": json}) - return _FakeResponse({"id": session_id}) - - monkeypatch.setattr("hud.eval.runtime.hud.httpx.AsyncClient", _RecordingAsyncClient) - - with set_trace_context(trace_id): - created = await HUDRuntime()._create_runtime_session( - "https://mcp.hud.ai", - "sk-hud-test", - Task(env="e", id="x"), - ) - - assert created == session_id - assert posts == [ - { - "path": "https://mcp.hud.ai/runtime/sessions", - "headers": {"Authorization": "Bearer sk-hud-test"}, - "json": {"environment": "e", "trace_id": str(uuid.UUID(trace_id))}, - } - ] - - -@pytest.mark.asyncio -async def test_runtime_session_sets_runtime_connection_params( - monkeypatch: pytest.MonkeyPatch, -) -> None: - session_id = str(uuid.uuid4()) - deleted: list[tuple[str, str, str]] = [] - - class _Socket: - def getsockname(self) -> tuple[str, int]: - return ("127.0.0.1", 4321) - - class _Server: - sockets: ClassVar[list[_Socket]] = [_Socket()] - - def __init__(self) -> None: - self.closed = False - self.waited = False - - def close(self) -> None: - self.closed = True - - async def wait_closed(self) -> None: - self.waited = True - - server = _Server() - - async def fake_start_server(*args: Any, **kwargs: Any) -> _Server: - return server - - async def fake_create_runtime_session( - self: HUDRuntime, - runtime_url: str, - api_key: str, - task: Task, - ) -> str: - assert runtime_url == "https://mcp.hud.ai" - assert api_key == "sk-hud-test" - assert task.env == "e" - return session_id - - async def fake_delete_runtime_session( - self: HUDRuntime, - runtime_url: str, - api_key: str, - session: str, - ) -> None: - deleted.append((runtime_url, api_key, session)) - - monkeypatch.setattr(settings, "api_key", "sk-hud-test") - monkeypatch.setattr("hud.eval.runtime.hud.asyncio.start_server", fake_start_server) - monkeypatch.setattr(HUDRuntime, "_create_runtime_session", fake_create_runtime_session) - monkeypatch.setattr(HUDRuntime, "_delete_runtime_session", fake_delete_runtime_session) - - with pytest.warns(DeprecationWarning, match="rollout_timeout"): - cloud = HUDRuntime(runtime_url="https://mcp.hud.ai/", run_timeout=30.0) - async with cloud._runtime_session(Task(env="e", id="x")) as runtime: - assert runtime.url == "tcp://127.0.0.1:4321" - assert runtime.params == { - "session_id": session_id, - "gateway_url": "https://mcp.hud.ai", - "ready_timeout": 300.0, - } - - assert deleted == [("https://mcp.hud.ai", "sk-hud-test", session_id)] - assert server.closed - assert server.waited - - -@pytest.mark.asyncio -async def test_splice_websocket_propagates_relay_errors() -> None: - class _Reader: - def __init__(self) -> None: - self.reads = [b"payload", b""] - - async def read(self, _limit: int) -> bytes: - return self.reads.pop(0) - - class _Writer: - def write(self, _data: bytes) -> None: - pass - - async def drain(self) -> None: - pass - - class _WebSocket: - async def send(self, _data: bytes) -> None: - raise RuntimeError("relay failed") - - def __aiter__(self) -> _WebSocket: - return self - - async def __anext__(self) -> bytes: - await asyncio.sleep(60.0) - raise StopAsyncIteration - - with pytest.raises(RuntimeError, match="relay failed"): - await _splice_websocket( - cast("asyncio.StreamReader", _Reader()), - cast("asyncio.StreamWriter", _Writer()), - _WebSocket(), - ) diff --git a/hud/eval/tests/test_job.py b/hud/eval/tests/test_job.py deleted file mode 100644 index eb0d2e8b0..000000000 --- a/hud/eval/tests/test_job.py +++ /dev/null @@ -1,135 +0,0 @@ -"""``hud.eval.job`` reporting — the trace-exit payload sent to the platform. - -No network: the platform client is replaced with a recorder. -""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING, Any - -import pytest - -from hud.eval import job as job_mod -from hud.eval.run import Run - -if TYPE_CHECKING: - from collections.abc import Iterator - - -class _Recorder: - """Stand-in platform client that captures the last reported body.""" - - def __init__(self) -> None: - self.calls: list[tuple[str, dict[str, Any]]] = [] - - async def apost(self, path: str, *, json: dict[str, Any]) -> dict[str, Any]: - self.calls.append((path, json)) - return {} - - -@pytest.fixture -def recorder(monkeypatch: pytest.MonkeyPatch) -> Iterator[_Recorder]: - from hud.settings import settings - - monkeypatch.setattr(settings, "telemetry_enabled", True) - monkeypatch.setattr(settings, "api_key", "sk-hud-test") - rec = _Recorder() - monkeypatch.setattr(job_mod.PlatformClient, "from_settings", classmethod(lambda cls: rec)) - yield rec - - -def _run_with(trace_id: str, *, extra: dict[str, Any]) -> Run: - run = Run(None, "task", {}) - run.trace.trace_id = trace_id - run.trace.status = "completed" - run.trace.extra = extra - return run - - -async def test_trace_enter_reports_task_and_group_identity(recorder: _Recorder) -> None: - await job_mod.trace_enter( - "abc", - job_id="job-1", - group_id="group-1", - task_slug="fix-bug-3", - model="test-model", - parent_trace_id="parent-1", - ) - - assert recorder.calls == [ - ( - "/trace/abc/enter", - { - "job_id": "job-1", - "group_id": "group-1", - "task_slug": "fix-bug-3", - "model": "test-model", - "parent_trace_id": "parent-1", - }, - ) - ] - - -async def test_job_enter_logs_canonical_web_uuid( - recorder: _Recorder, - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - from hud.settings import settings - - compact_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - canonical_id = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" - monkeypatch.setattr(settings, "hud_web_url", "https://hud.test") - caplog.set_level(logging.INFO, logger="hud.eval.job") - - await job_mod.job_enter(compact_id, name="test", group=1) - - assert recorder.calls[0][0] == f"/trace/job/{compact_id}/enter" - assert f"job: https://hud.test/jobs/{canonical_id}" in caplog.text - - -async def test_trace_exit_propagates_stop_reason(recorder: _Recorder) -> None: - run = _run_with("abc", extra={}) - run.trace.stop_reason = "max_steps" - await job_mod.trace_exit(run) - - assert len(recorder.calls) == 1 - path, body = recorder.calls[0] - assert path == "/trace/abc/exit" - assert body["stop_reason"] == "max_steps" - assert "metadata" not in body - - -async def test_trace_exit_omits_metadata_when_extra_empty(recorder: _Recorder) -> None: - await job_mod.trace_exit(_run_with("abc", extra={})) - - assert len(recorder.calls) == 1 - _, body = recorder.calls[0] - assert "metadata" not in body - - -def test_errored_runs_do_not_deflate_the_job_reward() -> None: - """Infrastructure failure is never a score: a run that errored (a launch - failure, or a hosted trace that ended in error) carries no verdict and - must not drag the job mean down as a silent zero.""" - from hud.eval.job import Job - from hud.eval.run import Grade - - graded = _run_with("t1", extra={}) - graded.grade = Grade(reward=1.0) - failed = Run.failed("provisioning never finished") - - job = Job(id="j1", name="test", runs=[graded, failed]) - - assert job.reward == 1.0 - assert job.errors == [failed] - - -def test_job_with_only_errors_reports_zero_reward() -> None: - from hud.eval.job import Job - - job = Job(id="j2", name="test", runs=[Run.failed("boom")]) - - assert job.reward == 0.0 - assert job.errors and job.errors[0].trace.is_error diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py deleted file mode 100644 index 5fa4fc682..000000000 --- a/hud/eval/tests/test_local_runtime.py +++ /dev/null @@ -1,535 +0,0 @@ -"""LocalRuntime serves environments from live instances and source recipes.""" - -from __future__ import annotations - -import asyncio -import sys -from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime -from pathlib import Path -from typing import Any, cast - -import pytest - -import hud.eval.runtime.local as local_runtime_module -from hud.agents.base import Agent -from hud.environment import Environment, load_environment -from hud.eval import LocalRuntime, RuntimeConfig, SubprocessRuntime, Task, Taskset - -_SUMS_ENV = """\ -from hud import Environment - -env = Environment("{name}") - - -@env.template(id="add") -async def add(a: int, b: int): - answer = yield f"add:{{a}}:{{b}}" - yield 1.0 if answer == str(a + b) else 0.0 -""" - - -def _sums_env(name: str = "sums") -> Environment: - env = Environment(name) - - @env.template(id="add") - async def add(a: int, b: int) -> AsyncGenerator[Any, Any]: - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - return env - - -class _FnAgent(Agent): - """Stateless agent: answers each run by applying ``fn`` to ``run.prompt``.""" - - def __init__(self, fn: Any) -> None: - super().__init__() - self._fn = fn - - async def __call__(self, run: Any) -> None: - run.trace.content = self._fn(run.prompt) - - -def _solve_add(prompt: str) -> str: - _, a, b = prompt.split(":") - return str(int(a) + int(b)) - - -# ─── LocalRuntime sources ────────────────────────────────────────────── - - -async def test_source_path_serves_a_fresh_env_per_rollout(tmp_path) -> None: - # LOADS is module state: a fresh throwaway import per acquisition means - # every rollout sees exactly one load. - (tmp_path / "env.py").write_text( - "from hud import Environment\n\n" - "LOADS = []\n" - 'env = Environment("sums")\n\n\n' - '@env.template(id="add")\nasync def add(a: int, b: int):\n' - " LOADS.append(1)\n" - ' answer = yield f"add:{a}:{b}:{len(LOADS)}"\n' - " yield 1.0 if answer == str(a + b) else 0.0\n", - encoding="utf-8", - ) - - def _solve(prompt: str) -> str: - _, a, b, loads = prompt.split(":") - assert loads == "1" - return str(int(a) + int(b)) - - job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run( - _FnAgent(_solve), - runtime=LocalRuntime(tmp_path / "env.py"), - group=2, - ) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - - -async def test_subprocess_runtime_streams_environment_output( - tmp_path, - capsys: pytest.CaptureFixture[str], -) -> None: - source = tmp_path / "env.py" - source.write_text( - "import sys\n\n" - "from hud import Environment\n\n" - 'env = Environment("sums")\n\n' - "@env.initialize\n" - "async def start():\n" - ' print("x" * 100_000, flush=True)\n' - ' print("environment booted", flush=True)\n' - ' print("y" * 100_000, file=sys.stderr, flush=True)\n' - ' print("environment warning", file=sys.stderr, flush=True)\n', - encoding="utf-8", - ) - - async with SubprocessRuntime(source)(Task(env="sums", id="add")): - pass - - captured = capsys.readouterr() - assert "x" * 100_000 in captured.out - assert "environment booted" in captured.out - assert "y" * 100_000 in captured.err - assert "environment warning" in captured.err - - -def test_subprocess_runtime_serves_a_live_env_from_its_template_file(tmp_path, request) -> None: - module_name = f"sums_env_{request.node.name}" - env_py = tmp_path / f"{module_name}.py" - env_py.write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") - tasks_py = tmp_path / "tasks.py" - tasks_py.write_text(f"from {module_name} import add\n\ntasks = [add(a=2, b=3)]\n") - request.addfinalizer(lambda: sys.modules.pop(module_name, None)) - - task = next(iter(Taskset.from_module(tasks_py))) - assert task._env is not None - provider = SubprocessRuntime(task._env) - - assert provider.source == env_py.resolve() - assert provider.env == "sums" - - -def test_subprocess_runtime_rejects_env_without_templates() -> None: - with pytest.raises(ValueError, match="exactly one source file"): - SubprocessRuntime(Environment("demo")) - - -def test_subprocess_runtime_rejects_env_pin_for_live_env() -> None: - with pytest.raises(TypeError, match="env= applies only to source paths"): - SubprocessRuntime(_sums_env(), env="sums") - - -async def test_subprocess_runtime_fails_when_stdout_closes_before_serving( - tmp_path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - source = tmp_path / "env.py" - source.write_text("", encoding="utf-8") - output = asyncio.StreamReader() - output.feed_eof() - error = asyncio.StreamReader() - error.feed_data(b"stdout closed\n") - error.feed_eof() - - class Process: - stdout = output - stderr = error - returncode = None - terminated = False - - async def wait(self) -> int: - await asyncio.Event().wait() - return 0 - - async def terminate(self) -> None: - self.terminated = True - - process = Process() - - async def create_process(*args: Any, **kwargs: Any) -> Any: - return process - - monkeypatch.setattr(local_runtime_module, "create_process_group_exec", create_process) - - async def run() -> None: - async with SubprocessRuntime(source, ready_timeout=30)(Task(env="closed", id="noop")): - pass - - with pytest.raises(RuntimeError, match=r"(?s)closed stdout.*stdout closed"): - await asyncio.wait_for(run(), timeout=1) - assert process.terminated is True - - -async def test_constructor_builds_fresh_per_rollout_from_the_row() -> None: - built: list[str] = [] - started = 0 - all_started = asyncio.Event() - - def env_for(task: Task) -> Environment: - built.append(task.env) - env = _sums_env(task.env) - - @env.initialize - async def start() -> None: - nonlocal started - started += 1 - if started == 3: - all_started.set() - await all_started.wait() - - return env - - job = await Task(env="sums", id="add", args={"a": 1, "b": 2}).run( - _FnAgent(_solve_add), - runtime=LocalRuntime(env_for, ready_timeout=2), - group=3, - max_concurrent=3, - ) - - assert all(run.reward == 1.0 for run in job.runs) - assert built == ["sums", "sums", "sums"] - - -async def test_live_environment_is_served_serially() -> None: - env = Environment("sums") - active = 0 - - @env.template(id="add") - async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - @env.initialize - async def _start() -> None: - nonlocal active - active += 1 - assert active == 1 - await asyncio.sleep(0) - - @env.shutdown - async def _stop() -> None: - nonlocal active - active -= 1 - - job = await add(a=2, b=3).run( - _FnAgent(_solve_add), - group=2, - max_concurrent=2, - ) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - assert active == 0 - - -@pytest.mark.parametrize( - "source", ["directory", "split_file", "package", "factory", "separate_providers"] -) -async def test_cached_environment_lifecycle_is_serialized(tmp_path, request, source) -> None: - module_name = f"cached_lifecycle_{source}" - project = tmp_path / module_name - project.mkdir() - if source == "package": - (project / "__init__.py").write_text("from .env import env\n") - events = tmp_path / "lifecycle.txt" - (project / f"{module_name}.py").write_text( - _SUMS_ENV.format(name="sums") - + f""" -import asyncio -from pathlib import Path -events = Path({str(events)!r}) - -@env.initialize -async def start(): - with events.open("a") as log: - log.write("start\\n") - await asyncio.sleep(0) - -@env.shutdown -async def stop(): - with events.open("a") as log: - log.write("stop\\n") -""" - ) - prefix = "." if source == "package" else "" - entrypoint = project / "env.py" - entrypoint.write_text(f"from {prefix}{module_name} import env\n") - - def cleanup() -> None: - for name, module in tuple(sys.modules.items()): - file = getattr(module, "__file__", None) - if file and Path(file).is_relative_to(project): - del sys.modules[name] - - request.addfinalizer(cleanup) - if source in {"factory", "separate_providers"}: - env = load_environment(entrypoint) - runtime = LocalRuntime(lambda _task: env) - if source == "separate_providers": - runtime = lambda task: LocalRuntime(env)(task) - else: - runtime = LocalRuntime(project if source == "directory" else entrypoint) - job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run( - _FnAgent(_solve_add), runtime=runtime, group=2, max_concurrent=2 - ) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - assert events.read_text().splitlines() == ["start", "stop", "start", "stop"] - - -async def test_cancelling_a_waiting_acquisition_preserves_the_running_environment() -> None: - env = _sums_env() - active = False - - @env.initialize - async def start() -> None: - nonlocal active - active = True - - @env.shutdown - async def stop() -> None: - nonlocal active - active = False - - runtime = LocalRuntime(lambda _task: env) - task = Task(env="sums", id="add") - - async def acquire() -> None: - async with runtime(task): - pytest.fail("the environment already has an active acquisition") - - async with runtime(task): - waiter = asyncio.create_task(acquire()) - await asyncio.sleep(0) - waiter.cancel() - with pytest.raises(asyncio.CancelledError): - await waiter - assert active - - assert not active - - -def test_local_environment_can_run_in_successive_event_loops() -> None: - env = _sums_env() - task = Task(env="sums", id="add", args={"a": 2, "b": 3}) - - for _ in range(2): - job = asyncio.run( - task.run(_FnAgent(_solve_add), runtime=LocalRuntime(env), group=2, max_concurrent=2) - ) - assert [run.reward for run in job.runs] == [1.0, 1.0] - - -async def test_serialized_task_does_not_retain_local_placement() -> None: - env = Environment("sums") - - @env.template(id="add") - async def add(a: int, b: int): - yield f"add:{a}:{b}" - yield 1.0 - - task = add(a=2, b=3) - portable = Task.model_validate(task.model_dump()) - - with pytest.raises(ValueError, match="no placement: pass runtime="): - await portable.run(_FnAgent(_solve_add)) - - -async def test_source_missing_env_name_fails_loudly(tmp_path) -> None: - (tmp_path / "env.py").write_text( - 'from hud import Environment\n\nenv = Environment("sums")\n', - encoding="utf-8", - ) - provider = LocalRuntime(tmp_path / "env.py") - - with pytest.raises(ValueError, match="no Environment named 'other'"): - async with provider(Task(env="other", id="add")): - pass - - -async def test_module_taskset_uses_factory_environments(tmp_path) -> None: - source = tmp_path / "tasks.py" - source.write_text( - _SUMS_ENV.format(name="sums") + "\ntasks = [add(a=2, b=3), add(a=4, b=5)]\n", - encoding="utf-8", - ) - - job = await Taskset.from_module(source).run(_FnAgent(_solve_add)) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - - -async def test_tasks_module_uses_factory_environment(tmp_path, request) -> None: - module_name = f"sums_env_{request.node.name}" - (tmp_path / f"{module_name}.py").write_text( - _SUMS_ENV.format(name="sums"), - encoding="utf-8", - ) - tasks = tmp_path / "tasks.py" - tasks.write_text( - f"from {module_name} import add\n\ntasks = [add(a=2, b=3)]\n", - encoding="utf-8", - ) - request.addfinalizer(lambda: sys.modules.pop(module_name, None)) - - job = await Taskset.from_module(tasks).run(_FnAgent(_solve_add)) - - assert job.reward == 1.0 - - -async def test_ad_hoc_taskset_requires_explicit_placement() -> None: - with pytest.raises(ValueError, match="no placement: pass runtime="): - await Taskset("sums", [Task(env="sums", id="add")]).run(_FnAgent(_solve_add)) - - -async def test_container_rows_start_their_image_by_default(monkeypatch) -> None: - """A row that names an image is placeable without ``runtime=``: it gets DockerRuntime.""" - import hud.eval.taskset as taskset_module - - live = LocalRuntime(_sums_env()) - monkeypatch.setattr( - taskset_module, - "DockerRuntime", - lambda: lambda task: live(task.model_copy(update={"runtime_config": None})), - ) - portable = Task(env="sums", id="add", args={"a": 2, "b": 3}) - container = Task( - env="sums", id="add", args={"a": 4, "b": 5}, runtime_config=RuntimeConfig(image="sums") - ) - - job = await Taskset("sums", [container]).run(_FnAgent(_solve_add)) - assert [run.reward for run in job.runs] == [1.0] - - with pytest.raises(ValueError, match="no placement"): - await Taskset("sums", [container, portable]).run(_FnAgent(_solve_add)) - - -async def test_container_row_with_shared_verifier_is_placeable(monkeypatch) -> None: - """A verifier on the same env with no runtime of its own rides the actor's container.""" - import hud.eval.taskset as taskset_module - - env = _sums_env() - - @env.template(id="verify") - async def verify() -> AsyncGenerator[Any, Any]: - actor_grade = yield "" - yield 1.0 if actor_grade["score"] == 1.0 else 0.0 - - live = LocalRuntime(env) - starts: list[str] = [] - - def docker(task: Task) -> Any: - starts.append(task.id) - return live(task.model_copy(update={"runtime_config": None})) - - monkeypatch.setattr(taskset_module, "DockerRuntime", lambda: docker) - container = Task( - env="sums", - id="add", - args={"a": 4, "b": 5}, - runtime_config=RuntimeConfig(image="sums"), - verifier=Task(env="sums", id="verify"), - ) - - job = await Taskset("sums", [container]).run(_FnAgent(_solve_add)) - assert [run.reward for run in job.runs] == [1.0] - assert starts == ["add"] # one container for actor and verifier - - -async def test_empty_taskset_needs_no_placement() -> None: - job = await Taskset("empty", []).run(_FnAgent(_solve_add)) - - assert job.runs == [] - - -def test_rejects_a_non_pointer_argument() -> None: - with pytest.raises(TypeError, match="expected a source path"): - LocalRuntime(cast("Any", 42)) - - -async def test_failed_startup_still_runs_shutdown_hooks() -> None: - env = _sums_env() - lifecycle: list[str] = [] - - @env.initialize - async def _up() -> None: - lifecycle.append("up") - - @env.shutdown - async def _down() -> None: - lifecycle.append("down") - - @env.initialize - async def _boom() -> None: - raise RuntimeError("daemon failed to start") - - with pytest.raises(RuntimeError, match="daemon failed to start"): - async with LocalRuntime(env)(Task(env="sums", id="add")): - pass - - assert lifecycle == ["up", "down"] - - -async def test_template_can_lazily_import_a_sibling_module(tmp_path) -> None: - (tmp_path / "lazy_helper.py").write_text("ANSWER_SUFFIX = ':ok'\n", encoding="utf-8") - (tmp_path / "env.py").write_text( - "from hud import Environment\n\n" - 'env = Environment("sums")\n\n\n' - '@env.template(id="add")\nasync def add(a: int, b: int):\n' - " import lazy_helper\n" - ' answer = yield f"add:{a}:{b}{lazy_helper.ANSWER_SUFFIX}"\n' - " yield 1.0 if answer == str(a + b) else 0.0\n", - encoding="utf-8", - ) - - def _solve(prompt: str) -> str: - _, a, b, ok = prompt.split(":") - assert ok == "ok" - return str(int(a) + int(b)) - - job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run( - _FnAgent(_solve), - runtime=LocalRuntime(tmp_path / "env.py"), - group=2, - max_concurrent=2, - ) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - - -# ─── seam defenses ───────────────────────────────────────────────────── - - -@pytest.mark.filterwarnings("ignore::RuntimeWarning") # the broken source leaks its coroutine -async def test_unguarded_run_call_in_source_names_the_mistake(tmp_path) -> None: - (tmp_path / "env.py").write_text( - "import asyncio\n\nfrom hud import Environment\n\n" - 'env = Environment("sums")\n\n' - "asyncio.run(asyncio.sleep(0))\n", - encoding="utf-8", - ) - provider = LocalRuntime(tmp_path / "env.py") - - with pytest.raises(RuntimeError, match='if __name__ == "__main__"'): - async with provider(Task(env="sums", id="add")): - pass diff --git a/hud/eval/tests/test_rollout.py b/hud/eval/tests/test_rollout.py deleted file mode 100644 index e786a3dde..000000000 --- a/hud/eval/tests/test_rollout.py +++ /dev/null @@ -1,1538 +0,0 @@ -"""The rollout engine: ``rollout(task, agent)`` and its schedulers. - -These drive the engine end-to-end through the real placement path: a pure-data -``Task`` row plus ``runtime=SubprocessRuntime(env_file)`` — a child process serves the env, the -engine connects over the wire, the agent answers, grading comes back. The -engine contract is a graded :class:`Run` with a trace id (always under a job — -there are no standalone traces), and failure isolation that never raises: a -pre-launch failure yields a synthesized ``Run.failed``; a mid-run failure -keeps the real run and its evidence. ``Task.run`` / ``Taskset.run`` schedule -the atom and return a :class:`Job`. -""" - -from __future__ import annotations - -import asyncio -import contextlib -import json -import os -import signal -import subprocess -import sys -import textwrap -from contextlib import asynccontextmanager -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any - -import mcp.types as mcp_types -import pytest -from pydantic import BaseModel - -import hud.eval.run as run_module -from hud.agents.base import Agent -from hud.agents.openai_compatible import OpenAIChatAgent -from hud.agents.types import OpenAIChatConfig -from hud.capabilities import Connection -from hud.clients.client import HudClient -from hud.environment import Answer, Environment -from hud.eval import ( - Job, - LocalRuntime, - Runtime, - RuntimeConfig, - RuntimeLimits, - SubprocessRuntime, - Task, - Taskset, -) -from hud.eval.run import Run, rollout -from hud.telemetry.context import get_current_trace_id, get_trace_headers, set_trace_context - - -async def test_taskset_rejects_zero_group_before_execution(): - with pytest.raises(ValueError, match="group must be >= 1"): - await Taskset("empty", []).run(_FnAgent(lambda _: "answer"), group=0) - - -if TYPE_CHECKING: - from collections.abc import AsyncIterator - from pathlib import Path - - from hud.eval.task import Task as TaskRow - -_SUMS_ENV = """\ -from hud import Environment - -env = Environment("sums") - - -@env.template() -async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 -""" - - -@pytest.fixture(scope="module") -def env_file(tmp_path_factory: pytest.TempPathFactory) -> Path: - path = tmp_path_factory.mktemp("sums") / "env.py" - path.write_text(textwrap.dedent(_SUMS_ENV), encoding="utf-8") - return path - - -class _FnAgent(Agent): - """Stateless agent: answers each run by applying ``fn`` to ``run.prompt``.""" - - def __init__(self, fn: Any) -> None: - super().__init__() - self._fn = fn - - async def __call__(self, run: Any) -> None: - run.trace.content = self._fn(run.prompt) - - -class _SequencedCompletions: - def __init__(self, responses: list[Any]) -> None: - self._responses = responses - self.requests: list[dict[str, Any]] = [] - - async def create(self, **kwargs: Any) -> Any: - self.requests.append(kwargs) - return self._responses.pop(0) - - -class _FakeOpenAI: - def __init__(self, responses: list[Any]) -> None: - self.chat = SimpleNamespace(completions=_SequencedCompletions(responses)) - - -def _chat_response(content: str, tool_calls: list[Any] | None = None) -> Any: - message = SimpleNamespace( - content=content, - tool_calls=tool_calls or [], - refusal=None, - model_dump=lambda exclude_none=True: {"role": "assistant", "content": content}, - ) - choice = SimpleNamespace(message=message, finish_reason="stop", logprobs=None) - return SimpleNamespace( - choices=[choice], - model="fake-openai-compatible", - usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, prompt_tokens_details=None), - ) - - -def _tool_call(name: str, arguments: str) -> Any: - return SimpleNamespace( - type="function", - id=f"call_{name}", - function=SimpleNamespace(name=name, arguments=arguments), - ) - - -def _add_task(a: int, b: int) -> Task: - """A pure data row; the env it names is defined by the spawned file.""" - return Task(env="sums", id="add", args={"a": a, "b": b}) - - -def _solve_add(prompt: str) -> str: - _, a, b = prompt.split(":") - return str(int(a) + int(b)) - - -def _pid_status(pid: int) -> str | None: - result = subprocess.run( - ["ps", "-o", "stat=", "-p", str(pid)], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ) - return result.stdout.strip() or None - - -def _pid_is_running(pid: int) -> bool: - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - status = _pid_status(pid) - if status is None: - return False - return not status.startswith("Z") - - -async def _wait_for_pid_inactive(pid: int, max_wait: float = 2.0) -> bool: - loop = asyncio.get_running_loop() - deadline = loop.time() + max_wait - while loop.time() < deadline: - if not _pid_is_running(pid): - return True - await asyncio.sleep(0.05) - return not _pid_is_running(pid) - - -async def test_rollout_returns_graded_run_with_trace_id(env_file: Path) -> None: - run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file)) - - assert run.reward == 1.0 - assert run.trace.content == "5" - assert run.trace_id is not None - # No standalone traces: a bare rollout registers a single-run job itself. - assert run.job_id is not None - # The factual placement record: the runtime this run executed against. - assert run.runtime is not None - assert run.runtime.startswith("tcp://127.0.0.1:") - - -def test_run_owns_named_controller_connections() -> None: - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - run = Run(None, "task", {}, connections=(connection,)) - - assert run.connections == {"inference": connection} - assert "scoped-runtime-token" not in repr(connection) - - -async def test_verifier_task_replaces_the_actor_grade_in_the_same_runtime() -> None: - env = Environment("reviewed") - completed: list[str] = [] - - @env.template() - async def solve(): - answer = yield "answer secret" - completed.append(f"actor:{answer}") - yield {"score": 0.25, "answer": answer} - - @env.template() - async def verify(expected: str): - result = yield "" - completed.append(f"verifier:{result['answer']}") - yield 1.0 if result["answer"] == expected else 0.0 - - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify", args={"expected": "secret"}), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) - - assert run.reward == 1.0 - assert completed == ["actor:secret", "verifier:secret"] - evaluations = [ - step.task_call.name - for step in run.trace.steps - if step.task_call is not None and step.task_call.phase == "evaluate" - ] - assert evaluations == ["solve", "verify"] - - -async def test_actor_result_is_forwarded_to_the_verifier() -> None: - env = Environment("reviewed") - received: list[str] = [] - - class ActorResult(BaseModel): - score: float - answer: str - - @env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.0, "answer": answer} - - @env.template(returns=ActorResult) - async def verify(): - answer = yield "" - assert isinstance(answer, Answer) - assert isinstance(answer.content, ActorResult) - received.append(answer.content.answer) - yield 1.0 - - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify"), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) - - assert run.reward == 1.0 - assert received == ["secret"] - - -async def test_malformed_subscores_fail_inside_the_rollout_boundary( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.clients import HudClient - - env = Environment("malformed-grade") - - @env.template() - async def solve(): - yield "answer" - yield 1.0 - - async def malformed_grade( - _client: HudClient, - _payload: dict[str, Any], - ) -> dict[str, Any]: - return {"score": 1.0, "subscores": [{"name": "missing-value"}]} - - reported: list[dict[str, Any]] = [] - - async def report(run: Run) -> None: - reported.append(run.evaluation) - - monkeypatch.setattr(HudClient, "grade", malformed_grade) - monkeypatch.setattr("hud.eval.run.trace_exit", report) - - run = await rollout( - Task(env="malformed-grade", id="solve"), - _FnAgent(lambda _prompt: "done"), - runtime=LocalRuntime(env), - ) - - assert run.trace.is_error - assert "value" in (run.trace.error or "") - assert reported == [{}] - - -async def test_independent_verifier_receives_runtime_session_files(tmp_path: Path) -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - transfers: list[tuple[str, str]] = [] - - @actor_env.template() - async def solve(): - yield "answer secret" - yield {"score": 0.0} - - @verifier_env.template() - async def verify(): - yield "" - yield 1.0 - - class ActorRuntime(Runtime): - @asynccontextmanager - async def snapshot_session(self, session_id: str) -> AsyncIterator[Path | None]: - destination = tmp_path / "session.tar.gz" - await asyncio.to_thread(destination.write_text, session_id, encoding="utf-8") - transfers.append(("actor", session_id)) - yield destination - - class VerifierRuntime(Runtime): - async def restore_session(self, session_id: str, source: Path) -> None: - content = await asyncio.to_thread(source.read_text, encoding="utf-8") - assert content.startswith("sess-") - transfers.append(("verifier", session_id)) - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - env = actor_env if row.env == "actor" else verifier_env - async with LocalRuntime(env)(row) as runtime: - yield ( - ActorRuntime(runtime.url) if row.env == "actor" else VerifierRuntime(runtime.url) - ) - - task = Task( - env="actor", - id="solve", - verifier=Task(env="judge", id="verify"), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.reward == 1.0 - assert [runtime for runtime, _ in transfers] == ["actor", "verifier"] - assert transfers[0][1].startswith("sess-") - assert transfers[1][1].startswith("sess-") - assert transfers[0][1] != transfers[1][1] - - -async def test_same_env_verifier_runs_beside_an_actor_that_cannot_transport_its_session() -> None: - env = Environment("solo") - - @env.template() - async def solve(): - yield "answer secret" - yield {"score": 0.0} - - @env.template() - async def verify(): - yield "" - yield 1.0 - - placements: list[str] = [] - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.id) - async with LocalRuntime(env)(row) as runtime: - yield runtime - - verifier_config = RuntimeConfig(limits=RuntimeLimits(run_timeout_s=60)) - task = Task( - env="solo", - id="solve", - verifier=Task(env="solo", id="verify", runtime_config=verifier_config), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.reward == 1.0 - assert placements == ["solve"] - - -async def test_same_env_verifier_gets_its_own_runtime_when_the_session_transports( - tmp_path: Path, -) -> None: - env = Environment("solo") - - @env.template() - async def solve(): - yield "answer secret" - yield {"score": 0.0} - - @env.template() - async def verify(): - yield "" - yield 1.0 - - placements: list[str] = [] - - class TransportingRuntime(Runtime): - transports_sessions = True - - @asynccontextmanager - async def snapshot_session(self, session_id: str) -> AsyncIterator[Path | None]: - destination = tmp_path / "session.tar.gz" - await asyncio.to_thread(destination.write_text, session_id, encoding="utf-8") - yield destination - - async def restore_session(self, session_id: str, source: Path) -> None: - pass - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.id) - async with LocalRuntime(env)(row.model_copy(update={"runtime_config": None})) as runtime: - yield TransportingRuntime(runtime.url) - - verifier_config = RuntimeConfig(limits=RuntimeLimits(run_timeout_s=60)) - task = Task( - env="solo", - id="solve", - verifier=Task(env="solo", id="verify", runtime_config=verifier_config), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.reward == 1.0 - assert placements == ["solve", "verify"] - - -async def test_runtime_session_transfer_rejects_invalid_ids(tmp_path: Path) -> None: - runtime = Runtime("tcp://127.0.0.1:8765") - - with pytest.raises(ValueError, match="single path component"): - async with runtime.snapshot_session("../actor"): - pass - with pytest.raises(ValueError, match="single path component"): - await runtime.restore_session("", tmp_path / "session.tar.gz") - - -async def test_verifier_with_its_own_environment_is_placed_after_the_actor() -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - placements: list[str] = [] - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(f"start:{row.env}") - try: - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)( - row - ) as runtime: - yield runtime - finally: - placements.append(f"stop:{row.env}") - - task = Task( - env="actor", - id="solve", - verifier=Task(env="judge", id="verify"), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.reward == 1.0 - assert placements == ["start:actor", "stop:actor", "start:judge", "stop:judge"] - - -async def test_verifier_remains_authoritative_after_an_agent_error() -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - placements: list[str] = [] - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.env) - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)(row) as runtime: - yield runtime - - task = Task( - env="actor", - id="solve", - verifier=Task(env="judge", id="verify"), - ) - run = await rollout(task, _AnswerThenBoomAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.trace.is_error - assert "agent exploded after answering" in (run.trace.error or "") - assert run.reward == 1.0 - assert placements == ["actor", "judge"] - - -@pytest.mark.parametrize("agent_fails", [False, True]) -async def test_verifier_remains_authoritative_when_actor_grading_fails( - agent_fails: bool, -) -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - placements: list[str] = [] - - @actor_env.template() - async def solve(): - yield "answer secret" - raise RuntimeError("actor grade exploded") - - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.env) - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)(row) as runtime: - yield runtime - - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) - agent = ( - _AnswerThenBoomAgent(lambda _prompt: "secret") - if agent_fails - else _FnAgent(lambda _prompt: "secret") - ) - run = await rollout(task, agent, runtime=provider) - - assert run.trace.is_error - assert "actor grade exploded" in (run.trace.error or "") - assert run.reward == 1.0 - assert placements == ["actor", "judge"] - - -async def test_verifier_remains_authoritative_when_actor_grade_is_scoreless( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.clients import HudClient - - actor_env = Environment("actor") - verifier_env = Environment("judge") - placements: list[str] = [] - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.env) - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)(row) as runtime: - yield runtime - - original_grade = HudClient.grade - grade_calls = 0 - - async def return_scoreless_actor_grade( - self: HudClient, payload: dict[str, Any] - ) -> dict[str, Any]: - nonlocal grade_calls - grade_calls += 1 - if grade_calls == 1: - return {"reward": 1.0} - return await original_grade(self, payload) - - monkeypatch.setattr(HudClient, "grade", return_scoreless_actor_grade) - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) - - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - job = Job(id="actor-scoreless", name="actor-scoreless", runs=[run]) - - assert run.trace.is_error - assert "numeric 'score'" in (run.trace.error or "") - assert run.reward == 1.0 - assert run.grade.raw["score"] == 1.0 - assert placements == ["actor", "judge"] - assert job.errors == [] - - -async def test_verifier_provisioning_failure_leaves_the_run_ungraded() -> None: - actor_env = Environment("actor") - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - if row.env == "judge": - raise RuntimeError("verifier unavailable") - async with LocalRuntime(actor_env)(row) as runtime: - yield runtime - - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.trace.is_error - assert "verifier unavailable" in (run.trace.error or "") - assert run.grade.raw == {} - assert run.reward == 0.0 - - -async def test_verifier_channel_failure_leaves_the_run_ungraded( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.clients import HudClient - - env = Environment("reviewed") - - @env.template() - async def solve(): - yield "answer secret" - yield 0.25 - - @env.template() - async def verify(): - yield "" - yield 1.0 - - original_grade = HudClient.grade - grade_calls = 0 - - async def fail_verifier_grade(self: HudClient, payload: dict[str, Any]) -> dict[str, Any]: - nonlocal grade_calls - grade_calls += 1 - if grade_calls == 2: - raise ConnectionError("channel died after a partial grade frame") - return await original_grade(self, payload) - - monkeypatch.setattr(HudClient, "grade", fail_verifier_grade) - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify"), - ) - - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) - job = Job(id="verify-failed", name="verify-failed", runs=[run]) - - assert run.trace.is_error - assert "channel died after a partial grade frame" in (run.trace.error or "") - assert run.grade.raw == {} - assert job.errors == [run] - - -async def test_scoreless_verifier_frame_is_an_ungraded_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.clients import HudClient - - env = Environment("reviewed") - - @env.template() - async def solve(): - yield "answer secret" - yield 0.25 - - @env.template() - async def verify(): - yield "" - yield 1.0 - - original_grade = HudClient.grade - grade_calls = 0 - - async def return_scoreless_frame(self: HudClient, payload: dict[str, Any]) -> dict[str, Any]: - nonlocal grade_calls - grade_calls += 1 - if grade_calls == 2: - return {"reward": 1.0} - return await original_grade(self, payload) - - monkeypatch.setattr(HudClient, "grade", return_scoreless_frame) - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify"), - ) - - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) - job = Job(id="verify-scoreless", name="verify-scoreless", runs=[run]) - - assert run.trace.is_error - assert "numeric 'score'" in (run.trace.error or "") - assert run.grade.raw == {} - assert job.errors == [run] - - -async def test_zero_score_verifier_grade_counts_an_errored_rollout() -> None: - env = Environment("reviewed") - - @env.template() - async def solve(): - yield "answer secret" - yield 0.25 - - @env.template() - async def verify(): - yield "" - yield 0.0 - - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify"), - ) - run = await rollout( - task, - _AnswerThenBoomAgent(lambda _prompt: "secret"), - runtime=LocalRuntime(env), - ) - job = Job(id="verify-zero", name="verify-zero", runs=[run]) - - assert run.trace.is_error - assert run.grade.raw["score"] == 0.0 - assert job.reward == 0.0 - assert job.errors == [] - - -def _bindings_env(published: Any) -> Environment: - """An env whose task publishes per-episode binding data alongside the prompt.""" - env = Environment("slots") - - @env.template() - async def claim(): - yield {"prompt": "go", "bindings": published} - yield 1.0 - - return env - - -async def test_episode_bindings_from_the_start_frame_reach_the_agent() -> None: - seen: dict[str, dict[str, Any]] = {} - - class _Claiming(Agent): - async def __call__(self, run: Any) -> None: - seen.update(run.bindings) - - env = _bindings_env({"robot": {"token": "slot-2"}}) - run = await rollout(Task(env="slots", id="claim"), _Claiming(), runtime=LocalRuntime(env)) - - # Episode-scoped connection data reaches the agent by capability name, - # without reading it back off the recorded setup step. - assert seen == {"robot": {"token": "slot-2"}} - assert run.bindings == {"robot": {"token": "slot-2"}} - - -async def test_a_malformed_bindings_frame_fails_the_rollout_loudly() -> None: - env = _bindings_env({"robot": "slot-2"}) # capability data must be an object - run = await rollout( - Task(env="slots", id="claim"), _FnAgent(lambda _p: ""), runtime=LocalRuntime(env) - ) - - assert run.trace.status == "error" - assert "bindings" in str(run.trace.steps[-1].error) - - -async def test_openai_compatible_write_reaches_workspace_grader(tmp_path: Path) -> None: - workspace = tmp_path / "workspace" - report = workspace / "REPORT.md" - env = Environment("opencode_report") - env.workspace(workspace, guest_path=str(workspace)) - - @env.initialize - async def seed() -> None: - workspace.mkdir(parents=True, exist_ok=True) - report.unlink(missing_ok=True) - - @env.template() - async def write_report(): - yield "Write PASS to REPORT.md." - yield 1.0 if report.exists() and report.read_text().strip() == "PASS" else 0.0 - - model_client = _FakeOpenAI( - [ - _chat_response( - "", - [_tool_call("write", json.dumps({"filePath": str(report), "content": "PASS"}))], - ), - _chat_response("done"), - ] - ) - agent = OpenAIChatAgent( - OpenAIChatConfig(model="qwen3.6-plus", model_client=model_client, max_steps=4) - ) - - run = await rollout( - Task(env="opencode_report", id="write_report"), - agent, - runtime=LocalRuntime(env), - ) - - assert run.reward == 1.0 - assert report.read_text() == "PASS" - tools = model_client.chat.completions.requests[0]["extra_body"]["tools"] - assert [tool["function"]["name"] for tool in tools] == [ - "bash", - "read", - "glob", - "grep", - "edit", - "write", - ] - - -async def test_tool_agent_timeout_stops_running_workspace_command_before_grading( - tmp_path: Path, -) -> None: - workspace = tmp_path / "workspace" - started = workspace / "started" - late = workspace / "late" - env = Environment("timeout_cleanup") - env.workspace(workspace, guest_path=str(workspace)) - - @env.initialize - async def seed() -> None: - workspace.mkdir(parents=True, exist_ok=True) - started.unlink(missing_ok=True) - late.unlink(missing_ok=True) - - # Timing contract, sized to survive loaded CI: the command must start - # within the 2s agent budget, still be running when that budget expires - # (it sleeps 3s), and the grading delay must outlast the sleep so a - # command that survived the timeout provably leaves `late` behind. - @env.template() - async def wait_for_cleanup(): - yield "Start the requested command." - await asyncio.sleep(4.0) - yield 1.0 if started.exists() and not late.exists() else 0.0 - - model_client = _FakeOpenAI( - [ - _chat_response( - "", - [_tool_call("bash", '{"command":"touch started; sleep 3; touch late"}')], - ), - ] - ) - agent = OpenAIChatAgent( - OpenAIChatConfig( - model="qwen3.6-plus", - model_client=model_client, - max_steps=2, - timeout_seconds=2.0, - ) - ) - - run = await rollout( - Task(env="timeout_cleanup", id="wait_for_cleanup"), - agent, - runtime=LocalRuntime(env), - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert started.exists() - assert not late.exists() - assert run.reward == 1.0 - - -@pytest.mark.skipif(sys.platform == "win32", reason="POSIX process-group regression") -async def test_local_runtime_startup_failure_kills_spawned_children(tmp_path: Path) -> None: - env_file = tmp_path / "env.py" - env_file.write_text( - textwrap.dedent( - """ - import asyncio - from pathlib import Path - - from hud import Environment - - env = Environment("leaky") - - - @env.initialize - async def start_child(): - proc = await asyncio.create_subprocess_exec( - "sleep", - "120", - stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.DEVNULL, - ) - Path("child.pid").write_text(str(proc.pid), encoding="utf-8") - raise RuntimeError("startup boom") - """ - ), - encoding="utf-8", - ) - pid_file = tmp_path / "child.pid" - pid: int | None = None - - try: - with pytest.raises(RuntimeError, match="startup boom"): - async with SubprocessRuntime(env_file, ready_timeout=30.0)( - Task(env="leaky", id="noop") - ): - pass - pid = int(pid_file.read_text()) - assert await _wait_for_pid_inactive(pid) - finally: - if pid is not None and _pid_is_running(pid): - with contextlib.suppress(ProcessLookupError): - os.kill(pid, signal.SIGKILL) - - -async def test_mid_run_failure_keeps_the_real_run_and_its_evidence(env_file: Path) -> None: - def boom(prompt: str) -> str: - raise RuntimeError("agent exploded") - - run = await rollout(_add_task(2, 3), _FnAgent(boom), runtime=SubprocessRuntime(env_file)) - - assert run.trace.is_error - assert "agent exploded" in (run.trace.error or "") - assert run.trace_id is not None # failed runs still key a trajectory - # The session was live, so the receipt keeps the evidence: the prompt the - # agent saw and the runtime the rollout executed against. - assert run.prompt == "add:2:3" - assert run.runtime is not None - assert run.reward == 0.0 # graded best-effort, but the agent never answered → 0.0 - - -class _AnswerThenBoomAgent(Agent): - """Records a correct answer, then raises — a mid-run failure after the env - already has a gradable answer in hand.""" - - def __init__(self, fn: Any) -> None: - super().__init__() - self._fn = fn - - async def __call__(self, run: Any) -> None: - run.trace.content = self._fn(run.prompt) - raise RuntimeError("agent exploded after answering") - - -async def test_mid_run_failure_still_grades_best_effort(env_file: Path) -> None: - # The agent answers correctly, then fails. The env is still alive, so the - # run is graded best-effort: the reward is captured even though it errored. - run = await rollout( - _add_task(2, 3), _AnswerThenBoomAgent(_solve_add), runtime=SubprocessRuntime(env_file) - ) - - assert run.trace.is_error - assert "agent exploded after answering" in (run.trace.error or "") - assert run.reward == 1.0 # graded despite the failure - assert run.trace.status == "error" # the failure is preserved, not masked - - -class _SlowAgent(Agent): - """Answers, then hangs — to exercise the agent-loop timeout.""" - - def __init__(self, fn: Any) -> None: - super().__init__() - self._fn = fn - self.cancelled = asyncio.Event() - - async def __call__(self, run: Any) -> None: - run.trace.content = self._fn(run.prompt) - try: - await asyncio.sleep(30) - finally: - self.cancelled.set() - - -async def test_agent_loop_timeout_is_an_explicit_failure() -> None: - env = Environment("sums") - task_cancelled = asyncio.Event() - - @env.template() - async def add(a: int, b: int): - try: - yield f"add:{a}:{b}" - yield 1.0 - finally: - task_cancelled.set() - - agent = _SlowAgent(_solve_add) - run = await rollout( - _add_task(2, 3), - agent, - runtime=LocalRuntime(env), - rollout_timeout=0.2, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert run.grade.raw == {} - await asyncio.wait_for(agent.cancelled.wait(), 1.0) - await asyncio.wait_for(task_cancelled.wait(), 1.0) - assert run.trace_id is not None - - -async def test_task_agent_timeout_still_grades_completed_work() -> None: - env = Environment("sums") - - @env.template() - async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - agent = _SlowAgent(_solve_add) - task = _add_task(2, 3).model_copy(update={"agent_config": {"timeout_seconds": 0.05}}) - - run = await rollout(task, agent, runtime=LocalRuntime(env)) - - assert run.reward == 1.0 - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert any("agent timed out" in (step.error or "") for step in run.trace.steps) - assert agent.cancelled.is_set() - job = Job(id="timeout-job", name="timeout", runs=[run]) - assert job.reward == 1.0 - assert job.errors == [] - - -@pytest.mark.parametrize("agent_timeout", [None, 10.0]) -async def test_agent_timeout_error_is_not_the_phase_deadline(agent_timeout: float | None) -> None: - env = Environment("sums") - - @env.template() - async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - class TimeoutAgent(Agent): - async def __call__(self, run: Any) -> None: - run.trace.content = _solve_add(run.prompt) - raise TimeoutError("provider timed out") - - task = _add_task(2, 3).model_copy( - update={"agent_config": {"timeout_seconds": agent_timeout} if agent_timeout else None} - ) - run = await rollout(task, TimeoutAgent(), runtime=LocalRuntime(env)) - - assert run.reward == 1.0 - assert run.trace.status == "error" - assert run.trace.stop_reason != "timeout" - assert "provider timed out" in (run.trace.error or "") - assert not any("agent timed out after" in (step.error or "") for step in run.trace.steps) - - -async def test_timeout_includes_grading() -> None: - env = Environment("sums") - grading_cancelled = asyncio.Event() - never = asyncio.Event() - - @env.template() - async def add(a: int, b: int): - yield f"add:{a}:{b}" - try: - await never.wait() - finally: - grading_cancelled.set() - yield 1.0 - - run = await rollout( - _add_task(2, 3), - _FnAgent(_solve_add), - runtime=LocalRuntime(env), - rollout_timeout=0.2, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert run.grade.raw == {} - await asyncio.wait_for(grading_cancelled.wait(), 1.0) - - -async def test_timeout_aborts_when_cancel_rpc_hangs(monkeypatch: pytest.MonkeyPatch) -> None: - env = Environment("sums") - agent_started = asyncio.Event() - cancel_started = asyncio.Event() - - @env.template() - async def add(a: int, b: int): - yield f"add:{a}:{b}" - await asyncio.Event().wait() - - class WaitingAgent(Agent): - async def __call__(self, run: Run) -> None: - agent_started.set() - await asyncio.Event().wait() - - aborted: list[bool] = [] - - async def hang_cancel(self: HudClient) -> None: - cancel_started.set() - await asyncio.Event().wait() - - real_abort = HudClient.abort - real_wait = asyncio.wait - - def track_abort(self: HudClient) -> None: - aborted.append(True) - real_abort(self) - - async def wait_after_agent_starts( - tasks: set[asyncio.Task[None]], **kwargs: Any - ) -> tuple[set[asyncio.Task[None]], set[asyncio.Task[None]]]: - # Exercise cancel/abort after connection setup, independent of startup speed. - await asyncio.wait_for(agent_started.wait(), timeout=5.0) - return await real_wait(tasks, **kwargs) - - monkeypatch.setattr(HudClient, "cancel", hang_cancel) - monkeypatch.setattr(HudClient, "abort", track_abort) - monkeypatch.setattr(run_module.asyncio, "wait", wait_after_agent_starts) - - run = await asyncio.wait_for( - rollout( - _add_task(2, 3), - WaitingAgent(), - runtime=LocalRuntime(env), - rollout_timeout=0.2, - ), - timeout=10.0, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert cancel_started.is_set() - assert aborted - - -async def test_timeout_does_not_wait_for_provider_cleanup() -> None: - env = Environment("sums") - cleanup_started = asyncio.Event() - cleanup_finished = asyncio.Event() - release_cleanup = asyncio.Event() - - @env.template() - async def add(a: int, b: int): - yield f"add:{a}:{b}" - yield 1.0 - - @asynccontextmanager - async def provider(_task: TaskRow) -> AsyncIterator[Runtime]: - try: - async with LocalRuntime(env)(_task) as runtime: - yield runtime - finally: - cleanup_started.set() - await release_cleanup.wait() - cleanup_finished.set() - - run = await rollout( - _add_task(2, 3), - _FnAgent(_solve_add), - runtime=provider, - rollout_timeout=0.2, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert run.reward == 1.0 - assert cleanup_started.is_set() - assert not cleanup_finished.is_set() - - release_cleanup.set() - await asyncio.wait_for(cleanup_finished.wait(), 1.0) - - -async def test_timeout_during_actor_cleanup_does_not_start_the_verifier() -> None: - actor_env = Environment("actor") - cleanup_started = asyncio.Event() - cleanup_finished = asyncio.Event() - release_cleanup = asyncio.Event() - placements: list[str] = [] - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.env) - if row.env == "judge": - raise AssertionError("verifier started after the rollout returned") - try: - async with LocalRuntime(actor_env)(row) as runtime: - yield runtime - finally: - cleanup_started.set() - await release_cleanup.wait() - cleanup_finished.set() - - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) - run = await rollout( - task, - _FnAgent(lambda _prompt: "secret"), - runtime=provider, - rollout_timeout=0.2, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert placements == ["actor"] - release_cleanup.set() - await asyncio.wait_for(cleanup_finished.wait(), 1.0) - await asyncio.sleep(0) - assert placements == ["actor"] - - -async def test_timeout_does_not_cancel_verifier_provider_cleanup() -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - cleanup_started = asyncio.Event() - cleanup_finished = asyncio.Event() - release_cleanup = asyncio.Event() - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - try: - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)( - row - ) as runtime: - yield runtime - finally: - if row.env == "judge": - cleanup_started.set() - await release_cleanup.wait() - cleanup_finished.set() - - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) - run = await rollout( - task, - _FnAgent(lambda _prompt: "secret"), - runtime=provider, - rollout_timeout=0.2, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert run.reward == 1.0 - assert cleanup_started.is_set() - assert not cleanup_finished.is_set() - - release_cleanup.set() - await asyncio.wait_for(cleanup_finished.wait(), 1.0) - - -async def test_pre_launch_failure_yields_a_synthesized_failed_run() -> None: - @asynccontextmanager - async def broken_provider(task: TaskRow) -> AsyncIterator[Runtime]: - raise RuntimeError("no substrate for you") - yield # pragma: no cover - - run = await rollout(_add_task(1, 1), _FnAgent(_solve_add), runtime=broken_provider) - - assert run.trace.is_error - assert "no substrate for you" in (run.trace.error or "") - assert run.trace_id is not None - assert run.prompt is None # nothing ever started - assert run.runtime is None - - -async def test_pre_launch_failure_keeps_the_notes_the_provider_attached() -> None: - # A provider attaches what only it can see — the env's output inside a remote - # sandbox — as a note, and str(exc) drops those on the way to the receipt. - @asynccontextmanager - async def broken_provider(task: TaskRow) -> AsyncIterator[Runtime]: - exc = EOFError("env closed connection during 'hello'") - exc.add_note("env output in sandbox ae964e25:\nModuleNotFoundError: No module named 'bugs'") - raise exc - yield # pragma: no cover - - run = await rollout(_add_task(1, 1), _FnAgent(_solve_add), runtime=broken_provider) - - assert "No module named 'bugs'" in (run.trace.error or "") - - -async def test_provider_is_called_with_the_task_row_being_placed(env_file: Path) -> None: - placed: list[str] = [] - - def placer(task: TaskRow) -> Any: - # The scheduler half of placement: the row is the request, so a - # provider can size/route each substrate per task. - placed.append(f"{task.env}/{task.id}:{task.args['a']}") - return SubprocessRuntime(env_file)(task) - - run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=placer) - - assert run.reward == 1.0 - assert placed == ["sums/add:2"] - - -async def test_task_run_schedules_a_single_task_job(env_file: Path) -> None: - job = await _add_task(2, 3).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(env_file)) - - (run,) = job.runs - assert job.reward == 1.0 - assert run.trace.content == "5" - assert run.job_id == job.id # the run's trace reports under the job - - -async def test_task_run_has_taskset_scheduling_semantics(env_file: Path) -> None: - job = await _add_task(1, 2).run( - _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file), group=2, max_concurrent=1 - ) - - assert job.group == 2 - assert [run.reward for run in job.runs] == [1.0, 1.0] - # The group repeats one task, so they share a GRPO group id. - assert len({run.group_id for run in job.runs}) == 1 - - -async def test_open_job_spans_multiple_scheduler_calls(env_file: Path) -> None: - session = await Job.start("session", group=2) - provider = SubprocessRuntime(env_file) - - job1 = await _add_task(1, 1).run(_FnAgent(_solve_add), runtime=provider, job=session) - job2 = await _add_task(2, 2).run(_FnAgent(_solve_add), runtime=provider, job=session) - - # Both calls accumulate into the one open job (group defaults to the job's). - assert job1 is session - assert job2 is session - assert len(session.runs) == 4 - assert {run.job_id for run in session.runs} == {session.id} - assert session.reward == 1.0 - - -_TWO_ENVS = """\ -from hud import Environment - -alpha = Environment("alpha") -beta = Environment("beta") - - -@alpha.template() -async def add_a(a: int, b: int): - answer = yield f"alpha:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - -@beta.template() -async def add_b(a: int, b: int): - answer = yield f"beta:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 -""" - - -async def test_one_spawn_serves_each_rows_env_in_a_mixed_taskset( - tmp_path_factory: pytest.TempPathFactory, -) -> None: - path = tmp_path_factory.mktemp("zoo") / "envs.py" - path.write_text(_TWO_ENVS, encoding="utf-8") - rows = [ - Task(env="alpha", id="add_a", args={"a": 1, "b": 2}), - Task(env="beta", id="add_b", args={"a": 3, "b": 4}), - ] - - # One provider, two envs: each acquisition serves the row it was called - # with (the task ids only exist on their own env, so a misplacement - # would fail the rollout). - job = await Taskset("zoo", rows).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(path)) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - assert [run.prompt for run in job.runs] == ["alpha:1:2", "beta:3:4"] - - -async def test_rollout_threads_job_and_group_ids(env_file: Path) -> None: - run = await rollout( - _add_task(1, 1), - _FnAgent(_solve_add), - runtime=SubprocessRuntime(env_file), - job_id="j1", - group_id="g1", - ) - - assert run.reward == 1.0 - assert run.job_id == "j1" - assert run.group_id == "g1" - - -@pytest.mark.parametrize( - ("ambient_trace_id", "trace_id", "expected_parent_trace_id"), - [ - ("parent-trace", "child-trace", "parent-trace"), - ( - "00000000-0000-0000-0000-000000000001", - "00000000000000000000000000000001", - None, - ), - ], -) -async def test_nested_rollout_binds_child_and_parent_trace_context( - monkeypatch: pytest.MonkeyPatch, - ambient_trace_id: str, - trace_id: str, - expected_parent_trace_id: str | None, -) -> None: - entered: dict[str, Any] = {} - active: dict[str, str | None] = {} - env = Environment("nested") - - @env.template(id="add") - async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - async def capture_enter(trace_id: str, **kwargs: Any) -> None: - entered["trace_id"] = trace_id - entered.update(kwargs) - - class ContextAgent(Agent): - async def __call__(self, run: Run) -> None: - active["trace_id"] = get_current_trace_id() - active["parent_trace_id"] = get_trace_headers().get("X-HUD-Parent-Trace-Id") - run.trace.content = "2" - - monkeypatch.setattr(run_module, "trace_enter", capture_enter) - - with set_trace_context(ambient_trace_id): - run = await rollout( - Task(env="nested", id="add", args={"a": 1, "b": 1}), - ContextAgent(), - runtime=LocalRuntime(env), - job_id="job-1", - trace_id=trace_id, - ) - - assert run.trace_id == trace_id - assert active == { - "trace_id": trace_id, - "parent_trace_id": expected_parent_trace_id, - } - assert entered["trace_id"] == trace_id - assert entered["parent_trace_id"] == expected_parent_trace_id - - -# ─── Run prompt views (what agents consume) ─────────────────────────── - - -def _run_with_prompt(prompt: Any) -> Run: - run = Run(None, "t", {}) - run.prompt = prompt - return run - - -def test_prompt_messages_wraps_plain_text_as_one_user_turn() -> None: - (msg,) = _run_with_prompt("hello").prompt_messages - assert msg.role == "user" - assert isinstance(msg.content, mcp_types.TextContent) - assert msg.content.text == "hello" - - -def test_prompt_messages_no_prompt_is_one_empty_user_turn() -> None: - (msg,) = _run_with_prompt(None).prompt_messages - assert isinstance(msg.content, mcp_types.TextContent) - assert msg.content.text == "" - - -def test_prompt_messages_normalizes_chat_dicts_and_passes_through() -> None: - existing = mcp_types.PromptMessage( - role="assistant", content=mcp_types.TextContent(type="text", text="prior") - ) - msgs = _run_with_prompt( - [ - {"role": "user", "content": {"type": "text", "text": "hi"}}, - {"role": "system", "content": "be nice"}, # outside MCP vocab → user - existing, - ] - ).prompt_messages - assert [m.role for m in msgs] == ["user", "user", "assistant"] - assert msgs[2] is existing - - -def test_prompt_text_flattens_text_turns_and_drops_non_text() -> None: - image = mcp_types.PromptMessage( - role="user", - content=mcp_types.ImageContent(type="image", data="aGk=", mimeType="image/png"), - ) - run = _run_with_prompt([{"role": "user", "content": "first"}, image, "second"]) - assert run.prompt_text == "first\n\nsecond" - - -@pytest.mark.parametrize("character", ["x", "é"]) -async def test_oversized_task_args_become_trace_error_before_setup(character: str) -> None: - encoded_character_bytes = len(json.dumps(character)) - 2 - data = character * (16 * 1024 * 1024 // encoded_character_bytes + 1) - env = Environment("large-task") - started = False - - @env.template() - async def task(criteria: str): - nonlocal started - started = True - yield "ready" - yield 1.0 - - run = await rollout( - Task(env=env.name, id="task", args={"criteria": data}), - _FnAgent(lambda _: pytest.fail("agent must not launch")), - runtime=LocalRuntime(env), - ) - assert not started - assert run.trace.is_error - assert "[starting task]" in (run.trace.error or "") - assert "'tasks.start' request" in (run.trace.error or "") - assert "limit is 16777216 bytes" in (run.trace.error or "") - assert "file ID" in (run.trace.error or "") - assert "EOFError" not in (run.trace.error or "") diff --git a/hud/eval/tests/test_shared.py b/hud/eval/tests/test_shared.py deleted file mode 100644 index 71d744347..000000000 --- a/hud/eval/tests/test_shared.py +++ /dev/null @@ -1,244 +0,0 @@ -"""``Shared``: one substrate leased to many concurrent rollouts. - -The contract: the substrate boots lazily on the first lease and lives for the -enclosing ``async with`` scope (one boot, deterministic teardown); ``width`` -bounds concurrent occupancy by making the next lease wait, not error; and -``Taskset.run`` scopes a context-manager placement to the call, so a bare -``runtime=Shared(...)`` needs no ceremony. -""" - -from __future__ import annotations - -import asyncio -import contextlib -from contextlib import asynccontextmanager -from typing import TYPE_CHECKING, Any - -import pytest - -from hud.agents.base import Agent -from hud.environment import Environment -from hud.eval import LocalRuntime, RuntimeConfig, Shared, Task, Taskset -from hud.eval.runtime import Runtime - -if TYPE_CHECKING: - from collections.abc import AsyncIterator - - from hud.eval.task import Task as TaskRow - - -class _CountingProvider: - """Wraps one local env, counting provisions and teardowns.""" - - def __init__(self, env: Environment) -> None: - self.runtime = LocalRuntime(env) - self.provisions = 0 - self.teardowns = 0 - - @asynccontextmanager - async def __call__(self, task: TaskRow) -> AsyncIterator[Runtime]: - self.provisions += 1 - try: - async with self.runtime(task) as runtime: - yield runtime - finally: - self.teardowns += 1 - - -def _echo_env() -> Environment: - env = Environment("pool") - - @env.template() - async def echo(tag: str = "x"): - answer = yield f"go {tag}" - yield 1.0 if answer == "ok" else 0.0 - - return env - - -class _OkAgent(Agent): - async def __call__(self, run: Any) -> None: - run.trace.content = "ok" - - -async def test_one_boot_serves_a_whole_grouped_run_and_closes_after() -> None: - inner = _CountingProvider(_echo_env()) - shared = Shared(inner, width=2) - - job = await Taskset("pool", [Task(env="pool", id="echo")]).run( - _OkAgent(), runtime=shared, group=3 - ) - - assert [run.reward for run in job.runs] == [1.0, 1.0, 1.0] - # All repeats shared one substrate, and one group_id (ordinary semantics). - assert inner.provisions == 1 - assert inner.teardowns == 1 # scoped to the run() call - assert len({run.group_id for run in job.runs}) == 1 - - -async def test_an_open_scope_keeps_the_substrate_warm_across_calls() -> None: - inner = _CountingProvider(_echo_env()) - taskset = Taskset("pool", [Task(env="pool", id="echo")]) - - async with Shared(inner, width=2) as shared: - await taskset.run(_OkAgent(), runtime=shared) - await taskset.run(_OkAgent(), runtime=shared) - assert inner.teardowns == 0 # still warm between calls - assert inner.provisions == 1 - assert inner.teardowns == 1 - - -async def test_distinct_task_placements_get_distinct_substrates() -> None: - provisions: list[str] = [] - teardowns: list[str] = [] - - @asynccontextmanager - async def provider(task: Task) -> AsyncIterator[Runtime]: - provisions.append(task.env) - try: - yield Runtime(f"tcp://{task.env}") - finally: - teardowns.append(task.env) - - async with Shared(provider, width=1) as shared: - async with shared(Task(env="actor", id="solve")) as actor: - assert actor.url == "tcp://actor" - async with shared(Task(env="verifier", id="verify")) as verifier: - assert verifier.url == "tcp://verifier" - - assert provisions == ["actor", "verifier"] - assert teardowns == ["verifier", "actor"] - - -async def test_width_one_releases_the_actor_lease_before_verifying() -> None: - env = Environment("reviewed") - - @env.template() - async def solve(): - answer = yield "answer" - yield {"score": 0.0, "answer": answer} - - @env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "ok" else 0.0 - - provisions = 0 - - @asynccontextmanager - async def provider(task: Task) -> AsyncIterator[Runtime]: - nonlocal provisions - provisions += 1 - async with LocalRuntime(env)(task.model_copy(update={"runtime_config": None})) as runtime: - yield runtime - - config = RuntimeConfig() - task = Task( - env="reviewed", - id="solve", - runtime_config=config, - verifier=Task( - env="reviewed", - id="verify", - runtime_config=config, - ), - ) - - job = await task.run( - _OkAgent(), - runtime=Shared(provider, width=1), - rollout_timeout=1, - ) - - assert job.reward == 1.0 - assert provisions == 1 - - -async def test_width_bounds_occupancy_by_waiting_not_erroring() -> None: - live = 0 - peak = 0 - - class _SlowAgent(Agent): - async def __call__(self, run: Any) -> None: - nonlocal live, peak - live += 1 - peak = max(peak, live) - await asyncio.sleep(0.05) - live -= 1 - run.trace.content = "ok" - - job = await Taskset("pool", [Task(env="pool", id="echo")]).run( - _SlowAgent(), runtime=Shared(_CountingProvider(_echo_env()), width=2), group=5 - ) - - assert [run.reward for run in job.runs] == [1.0] * 5 - assert peak <= 2 # lease 3 waited for a slot instead of raising - - -async def test_a_failed_boot_fails_one_lease_and_the_next_retries() -> None: - attempts = 0 - env = _echo_env() - - @asynccontextmanager - async def flaky(task: TaskRow) -> AsyncIterator[Runtime]: - nonlocal attempts - attempts += 1 - if attempts == 1: - raise RuntimeError("boot boom") - async with LocalRuntime(env)(task) as runtime: - yield runtime - - job = await Taskset("pool", [Task(env="pool", id="echo")]).run( - _OkAgent(), runtime=Shared(flaky, width=1), group=2 - ) - - # One rollout absorbed the boot failure; the retry served the other. - assert attempts == 2 - assert sorted(run.reward for run in job.runs) == [0.0, 1.0] - assert any(run.trace.is_error for run in job.runs) - - -async def test_leasing_outside_any_scope_errors_loudly() -> None: - shared = Shared(_CountingProvider(_echo_env()), width=1) - with pytest.raises(RuntimeError, match="scope"): - async with shared(Task(env="pool", id="echo")): - pass - - -async def test_scope_exit_without_any_lease_is_clean() -> None: - inner = _CountingProvider(_echo_env()) - async with Shared(inner, width=1): - pass - assert (inner.provisions, inner.teardowns) == (0, 0) - - -async def test_teardown_error_surfaces_at_scope_exit() -> None: - @asynccontextmanager - async def leaky(task: TaskRow) -> AsyncIterator[Runtime]: - try: - yield Runtime("tcp://127.0.0.1:1") - finally: - raise RuntimeError("teardown boom") - - with pytest.raises(RuntimeError, match="teardown boom"): - async with Shared(leaky, width=1) as shared: - async with shared(Task(env="pool", id="echo")): - pass - - -async def test_width_must_be_positive() -> None: - with pytest.raises(ValueError, match="width"): - Shared(_CountingProvider(_echo_env()), width=0) - - -async def test_non_pooling_placements_are_untouched_by_the_scope_hook() -> None: - # A plain provider is not a context manager; run() must not try to enter it. - inner = _CountingProvider(_echo_env()) - assert not isinstance(inner, contextlib.AbstractAsyncContextManager) - - job = await Taskset("pool", [Task(env="pool", id="echo")]).run( - _OkAgent(), runtime=inner, group=2 - ) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - assert inner.provisions == 2 # one substrate per rollout, as before diff --git a/hud/eval/tests/test_sync.py b/hud/eval/tests/test_sync.py deleted file mode 100644 index 267a80b5f..000000000 --- a/hud/eval/tests/test_sync.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Platform persistence: diff plans, record mapping, and the upload payload.""" - -from __future__ import annotations - -import json -from typing import TYPE_CHECKING, Any - -from hud.eval import Task, Taskset -from hud.eval.runtime import ComposeProject, RuntimeConfig -from hud.eval.sync import ( - diff, - fetch_taskset_tasks, - resolve_taskset_id, - task_upload_payload, - upload_taskset, -) -from hud.utils.platform import PlatformClient - -if TYPE_CHECKING: - from pathlib import Path - - import pytest - - -def _row(slug: str, n: object) -> Task: - return Task(env="e", id="solve", args={"n": n}, slug=slug) - - -def test_diff_classifies_create_update_unchanged_and_remote_only() -> None: - local_a = _row("a", 1) - local_b = _row("b", 2) - local_c = _row("c", 3) - remote_a = Task.model_validate(local_a.model_dump()) - remote_b = _row("b", 99) - remote_old = _row("old", 0) - - plan = diff( - Taskset("demo", [local_a, local_b, local_c]), - Taskset("demo", [remote_a, remote_b, remote_old]), - ) - - assert [t.slug for t in plan.to_create] == ["c"] - assert [t.slug for t in plan.to_update] == ["b"] - assert [t.slug for t in plan.unchanged] == ["a"] - assert [t.slug for t in plan.remote_only] == ["old"] - assert plan.to_apply == [local_c, local_b] - assert "Create: 1" in plan.summary() - - -def test_fetched_tasks_map_canonical_export_fields( - monkeypatch: pytest.MonkeyPatch, -) -> None: - # The CP export emits the canonical {env, scenario} pair (any legacy v5 env - # qualifier is stripped server-side), so the SDK maps the fields straight - # onto Task without re-deriving anything from the names. - requested: dict[str, str] = {} - payload = { - "taskset_id": "ts-id", - "name": "demo", - "tasks": [ - {"scenario": "solve", "env": "myenv", "name": "a", "args": {"n": 1}}, - {"scenario": "fix_bug", "env": "other", "name": "b"}, - ], - } - - def fake_request(method: str, url: str, **kwargs: object) -> dict[str, Any]: - requested.update(method=method, url=url) - return payload - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - - name, tasks = fetch_taskset_tasks(PlatformClient("https://api.example", "token"), "ts-id") - - assert requested == {"method": "GET", "url": "https://api.example/v2/tasksets/ts-id/export"} - assert name == "demo" - assert [(t.env, t.id, t.slug) for t in tasks] == [ - ("myenv", "solve", "a"), - ("other", "fix_bug", "b"), - ] - - -def test_resolve_taskset_id_looks_up_by_name(monkeypatch: pytest.MonkeyPatch) -> None: - requested: dict[str, str] = {} - - def fake_request(method: str, url: str, **kwargs: object) -> dict[str, Any]: - requested.update(method=method, url=url) - return {"taskset_id": "ts-id", "name": "demo", "tasks": []} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - - resolved = resolve_taskset_id(PlatformClient("https://api.example", "token"), "My Demo") - - assert requested == { - "method": "GET", - "url": "https://api.example/v2/tasksets/by-name/My%20Demo", - } - assert resolved == ("ts-id", "demo") - - -def test_resolve_taskset_id_passes_uuids_through() -> None: - platform = PlatformClient("https://api.example", "token") - raw = "8f4e0d62-4a3e-4f63-9c5d-1f2a3b4c5d6e" - assert resolve_taskset_id(platform, raw) == (raw, raw) - - -def test_upload_taskset_posts_payload(monkeypatch: pytest.MonkeyPatch) -> None: - upload = Task(env="e", id="solve", args={"n": 1}, slug="solve-one") - posted: dict[str, object] = {} - - def fake_request( - method: str, url: str, json: object = None, **kwargs: object - ) -> dict[str, Any]: - posted.update(method=method, url=url, json=json, api_key=kwargs.get("api_key")) - return {"ok": True} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - - platform = PlatformClient("https://api.example", "token") - result = upload_taskset(platform, "demo", [upload]) - - assert result == {"ok": True} - assert posted["method"] == "POST" - assert posted["url"] == "https://api.example/v2/tasks/upload" - assert posted["api_key"] == "token" - assert posted["json"] == { - "taskset_name": "demo", - "tasks": [ - { - "name": "solve-one", - "env": {"name": "e"}, - "task_id": "solve", - "args": {"n": 1}, - }, - ], - } - - -def test_upload_taskset_places_a_new_taskset(monkeypatch: pytest.MonkeyPatch) -> None: - """`project_id` reaches the platform so a created taskset lands in that Project.""" - posted: dict[str, Any] = {} - - def fake_request(method: str, url: str, json: object = None, **kwargs: Any) -> dict[str, Any]: - posted.update(json=json) - return {} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - - upload_taskset( - PlatformClient("https://api.example", "token"), - "demo", - [], - project_id="project-1", - ) - - assert posted["json"]["project_id"] == "project-1" - - -def test_upload_taskset_omits_project_when_unset(monkeypatch: pytest.MonkeyPatch) -> None: - """Without a project the payload is unchanged, so the default Project applies.""" - posted: dict[str, Any] = {} - - def fake_request(method: str, url: str, json: object = None, **kwargs: Any) -> dict[str, Any]: - posted.update(json=json) - return {} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - - upload_taskset(PlatformClient("https://api.example", "token"), "demo", []) - - assert "project_id" not in posted["json"] - - -def test_task_upload_payload_sends_env_and_bare_task_id() -> None: - payload = task_upload_payload(Task(env="e", id="solve", args={"n": 1})) - - assert payload["env"] == {"name": "e"} - assert payload["task_id"] == "solve" - assert "scenario" not in payload - - -def test_task_upload_payload_includes_runtime_config() -> None: - task = Task( - env="e", - id="solve", - runtime_config=RuntimeConfig(image="img:tag"), - ) - - payload = task_upload_payload(task) - - assert payload["runtime_config"] == {"image": "img:tag"} - - -def test_task_upload_payload_embeds_compose_document(tmp_path: Path) -> None: - compose = tmp_path / "compose.json" - compose.write_text( - json.dumps( - { - "services": { - "main": {"image": "registry.example/hud-main:latest"}, - "database": {"image": "postgres:16"}, - } - } - ), - encoding="utf-8", - ) - task = Task( - env="e", - id="solve", - runtime_config=RuntimeConfig(compose=ComposeProject(document=compose, service_access=True)), - ) - - payload = task_upload_payload(task) - - assert ( - payload["runtime_config"]["compose"]["document"]["services"]["database"]["image"] - == "postgres:16" - ) - assert payload["runtime_config"]["compose"]["service_access"] is True - assert str(compose) not in json.dumps(payload) - - -def test_task_upload_payload_preserves_runtime_config_null_override() -> None: - task = Task( - env="e", - id="solve", - runtime_config=RuntimeConfig(resources=None), - ) - - payload = task_upload_payload(task) - - assert payload["runtime_config"] == {"resources": None} - - -def test_task_upload_payload_includes_verifier_task(tmp_path: Path) -> None: - compose = tmp_path / "compose.json" - compose.write_text( - json.dumps({"services": {"main": {"image": "judge:latest"}}}), - encoding="utf-8", - ) - task = Task( - env="actor", - id="solve", - verifier=Task( - env="judge", - id="verify", - runtime_config=RuntimeConfig(compose=ComposeProject(document=compose)), - ), - ) - - payload = task_upload_payload(task) - - verifier = payload["verifier"] - runtime_config = verifier.pop("runtime_config") - assert verifier == { - "env": "judge", - "id": "verify", - "args": {}, - "slug": "verify", - } - assert runtime_config["compose"]["document"]["services"]["main"]["image"] == "judge:latest" - assert str(compose) not in json.dumps(runtime_config) diff --git a/hud/eval/tests/test_task.py b/hud/eval/tests/test_task.py deleted file mode 100644 index b12aaf157..000000000 --- a/hud/eval/tests/test_task.py +++ /dev/null @@ -1,511 +0,0 @@ -"""``Task`` construction, the portable row shape, and taskset collection. - -The model is the row: plain pydantic (``model_validate``/``model_dump``) is the -whole codec for ``hud sync`` and the JSON/JSONL taskset path. ``env`` is carried -as its name, the join key to whatever placement can bring that environment up. -Placement is never part of the row. Factory-created tasks can run locally; -portable rows require a provider or source placement. -""" - -from __future__ import annotations - -import json -from typing import TYPE_CHECKING - -import pytest - -from hud.agents.base import Agent -from hud.environment import Environment -from hud.eval import ( - HUDRuntime, - Run, - RuntimeConfig, - RuntimeGPU, - RuntimeResources, - RuntimeTPU, - Task, - Taskset, -) -from hud.eval.runtime.compose import ComposeConfig, ComposeProject, ComposeProjectRef - -if TYPE_CHECKING: - from pathlib import Path - - -def test_env_task_call_returns_public_task() -> None: - env = Environment("e") - - @env.template() - async def solve(n: int): - yield f"solve:{n}" - yield 1.0 - - runnable = solve(n=3) - assert isinstance(runnable, Task) - assert runnable.id == "solve" - assert runnable.args == {"n": 3} - assert runnable.env == "e" # the row carries the env's name, not the object - - -def test_slug_defaults_to_task_id_without_args() -> None: - v = Task(env="e", id="solve") - assert v.slug == "solve" - - -def test_slug_default_is_deterministic_with_args() -> None: - a = Task(env="e", id="solve", args={"b": 2, "a": 1}) - b = Task(env="e", id="solve", args={"a": 1, "b": 2}) # key order differs - assert a.slug == b.slug # stable: keys sorted - assert a.slug == "solve-1744f53e" - assert a.slug != Task(env="e", id="solve", args={"a": 9}).slug - - -def test_slug_rejects_none() -> None: - with pytest.raises(ValueError, match="slug"): - Task.model_validate({"env": "e", "id": "solve", "slug": None}) - - -def test_slug_rejects_empty_string() -> None: - with pytest.raises(ValueError, match="slug"): - Task(env="e", id="solve", slug="") - - -def test_slug_rejects_empty_assignment() -> None: - task = Task(env="e", id="solve", slug="valid") - - with pytest.raises(ValueError, match="slug"): - task.slug = "" - - assert task.slug == "valid" - - -# ─── the portable row shape ──────────────────────────────────────────── - - -def test_env_serializes_as_name_reference() -> None: - v = Task(env="team-intel", id="ask", args={"x": 1}) - data = v.model_dump(exclude_none=True) - assert data["env"] == "team-intel" - assert data["id"] == "ask" - assert data["args"] == {"x": 1} - - -def test_compact_dump_omits_unset_metadata() -> None: - data = Task(env="e", id="t").model_dump(exclude_none=True) - assert set(data) == {"env", "id", "args", "slug"} - assert data["slug"] == "t" - - data2 = Task(env="e", id="t", slug="s").model_dump(exclude_none=True) - assert data2["slug"] == "s" - - -def test_roundtrip_is_stable_through_plain_pydantic() -> None: - original = Task( - env="team-intel", - id="ask", - args={"difficulty": 3}, - slug="ask-v1", - validation=[{"name": "submit", "arguments": {"answer": "x"}}], - agent_config={"system_prompt": "be precise"}, - ).model_dump(exclude_none=True) - - rebuilt = Task.model_validate(original) - - assert rebuilt.env == "team-intel" # the name is the reference - assert rebuilt.id == "ask" - assert rebuilt.args == {"difficulty": 3} - assert rebuilt.slug == "ask-v1" - assert rebuilt.validation == original["validation"] - assert rebuilt.agent_config == {"system_prompt": "be precise"} - # ...and re-serializing yields the same portable dict. - assert rebuilt.model_dump(exclude_none=True) == original - - -def test_runtime_config_roundtrips_as_part_of_task_row() -> None: - resources = RuntimeResources( - cpu=2, - memory_mb=4096, - storage_mb=16384, - gpu=RuntimeGPU(type=["H100", "A100"]), - os="windows", - tpu=RuntimeTPU(type="v5", topology="2x2"), - ) - original = Task( - env="browser", - id="checkout", - runtime_config=RuntimeConfig( - image="hud-browser:firefox", - resources=resources, - ), - ).model_dump(exclude_none=True) - - rebuilt = Task.model_validate(original) - - assert rebuilt.runtime_config == RuntimeConfig( - image="hud-browser:firefox", - resources=resources, - ) - assert rebuilt.model_dump(exclude_none=True) == original - - -def test_verifier_roundtrips_as_an_ordinary_nested_task() -> None: - original = Task( - env="actor", - id="solve", - verifier=Task( - env="judge", - id="verify", - args={"suite": "hidden"}, - runtime_config=RuntimeConfig(image="judge:latest"), - ), - ).model_dump(exclude_none=True) - - rebuilt = Task.model_validate(original) - - assert rebuilt.verifier == Task( - env="judge", - id="verify", - args={"suite": "hidden"}, - runtime_config=RuntimeConfig(image="judge:latest"), - ) - assert rebuilt.model_dump(exclude_none=True) == original - - -def test_verifier_rejects_another_verifier() -> None: - with pytest.raises(ValueError, match="nested verifier tasks are not supported"): - Task( - env="actor", - id="solve", - verifier=Task( - env="judge", - id="verify", - verifier=Task(env="final-judge", id="verify-final"), - ), - ) - - -def test_verifier_rejects_another_verifier_on_assignment() -> None: - task = Task(env="actor", id="solve") - - with pytest.raises(ValueError, match="nested verifier tasks are not supported"): - task.verifier = Task( - env="judge", - id="verify", - verifier=Task(env="final-judge", id="verify-final"), - ) - - assert task.verifier is None - - -def test_runtime_config_rejects_unknown_fields() -> None: - with pytest.raises(ValueError, match="Extra inputs"): - RuntimeConfig.model_validate({"image": "img:tag", "provider_config": {}}) - - -def test_runtime_config_omits_null_resource_fields() -> None: - task = Task( - env="database", - id="cutover", - runtime_config=RuntimeConfig( - resources=RuntimeResources( - cpu=1, - memory_mb=None, - storage_mb=None, - gpu=None, - os=None, - tpu=None, - ), - ), - ) - - assert task.model_dump(mode="json", exclude_none=True)["runtime_config"] == { - "resources": {"cpu": 1.0} - } - - -def test_compose_runtime_config_serializes_by_task_dump_mode(tmp_path: Path) -> None: - compose = tmp_path / "compose.json" - compose.write_text( - json.dumps({"services": {"main": {"image": "hud-env:latest"}}}), - encoding="utf-8", - ) - config = RuntimeConfig(compose=ComposeProject(document=compose)) - task = Task(env="database", id="cutover", runtime_config=config) - - payload = task.model_dump(exclude_none=True) - rebuilt = Task.model_validate(payload) - - assert payload["runtime_config"] == {"compose": {"document": compose}} - assert rebuilt.runtime_config is not None - assert rebuilt.runtime_config.compose is not None - assert rebuilt.runtime_config.compose.document == compose - - payload = task.model_dump(mode="json", exclude_none=True) - rebuilt = Task.model_validate(payload) - - assert payload["runtime_config"] == { - "compose": { - "document": { - "services": { - "main": { - "image": "hud-env:latest", - "environment": {}, - "expose": [], - "ports": [], - "volumes": [], - } - }, - "networks": {}, - }, - } - } - assert rebuilt.runtime_config is not None - assert rebuilt.runtime_config.compose is not None - assert isinstance(rebuilt.runtime_config.compose.document, ComposeConfig) - assert str(compose) not in json.dumps(payload) - - -def test_compose_project_serializes_document_location_not_author_path(tmp_path: Path) -> None: - project = tmp_path / "artifact" - project.mkdir() - compose = project / "compose-project" / "compose.json" - compose.parent.mkdir() - compose.write_text( - json.dumps( - { - "services": { - "main": { - "image": "hud-harbor:local", - "build": {"context": "./main"}, - } - } - } - ), - encoding="utf-8", - ) - - payload = RuntimeConfig(compose=ComposeProject(document=compose, root=project)).model_dump( - mode="json", - exclude_unset=True, - ) - - assert payload["compose"]["root"] == {"compose_path": "compose-project/compose.json"} - assert str(tmp_path) not in json.dumps(payload) - - -def test_compose_runtime_config_round_trips_platform_records() -> None: - record = { - "compose": { - "document": { - "services": {"main": {"image": "hud-harbor:local"}}, - "networks": {}, - }, - "root": {"compose_path": "compose-project/compose.json"}, - }, - } - - config = RuntimeConfig.model_validate(record) - - assert config.compose is not None - assert isinstance(config.compose.document, ComposeConfig) - assert isinstance(config.compose.root, ComposeProjectRef) - payload = config.model_dump(mode="json", exclude_unset=True) - assert payload["compose"]["document"]["services"]["main"]["image"] == "hud-harbor:local" - assert payload["compose"]["root"] == {"compose_path": "compose-project/compose.json"} - assert RuntimeConfig.model_validate(payload) == config - - -def test_row_validation_rejects_malformed_entries() -> None: - # pydantic.ValidationError is a ValueError: callers catch one exception type. - with pytest.raises(ValueError, match="env"): - Task.model_validate({"id": "t"}) - with pytest.raises(ValueError, match="env"): - Task.model_validate({"env": {"name": "e"}, "id": "t"}) # an object is not a name - with pytest.raises(ValueError, match="id"): - Task.model_validate({"env": "e"}) - with pytest.raises(ValueError, match="args"): - Task.model_validate({"env": "e", "id": "t", "args": "nope"}) - - -# ─── placement ───────────────────────────────────────────────────────── - - -class _NoopAgent(Agent): - async def __call__(self, run: Run) -> None: - run.trace.content = "" - - -async def test_platform_taskset_defaults_to_hud_runtime(monkeypatch: pytest.MonkeyPatch) -> None: - import hud.eval.taskset as taskset_mod - - seen: dict[str, object] = {} - - async def fake_rollout(task: Task, agent: Agent, **kwargs: object) -> Run: - seen.update(kwargs) - run = Run(None, task.id, {}) - run.trace.status = "completed" - return run - - monkeypatch.setattr(taskset_mod, "rollout", fake_rollout) - - task = Task(env="hosted-env", id="solve", args={"n": 1}) - taskset = taskset_mod.Taskset("hosted", [task], taskset_id="ts_123") - job = await taskset.run(_NoopAgent()) - - (run,) = job.runs - assert run.trace.status == "completed" - assert isinstance(seen["runtime"], HUDRuntime) - assert seen["rollout_timeout"] is None - - -# ─── taskset collection ──────────────────────────────────────────────── - - -def test_taskset_is_ordered_and_keyed_by_slug() -> None: - first = Task(env="e", id="solve", args={"n": 1}, slug="first") - second = Task( - env="e", - id="solve", - args={"n": 2}, - slug="second", - verifier=Task(env="judge", id="verify"), - ) - - tasks = Taskset("demo", [first, second], taskset_id="ts_123") - - assert list(tasks) == [first, second] - assert tasks["first"] is first - filtered = tasks.filter(["second"]) - excluded = tasks.exclude(["first"]) - assert list(filtered) == [second] - assert list(excluded) == [second] - assert filtered.taskset_id == "ts_123" - assert excluded.taskset_id == "ts_123" - assert list(tasks.items()) == [("first", first), ("second", second)] - assert tasks.environment_names() == {"e", "judge"} - - -def test_taskset_from_file_loads_json_and_jsonl(tmp_path) -> None: - entries = [ - Task(env="e", id="solve", args={"n": 1}, slug="one").model_dump(exclude_none=True), - Task(env="e", id="solve", args={"n": 2}, slug="two").model_dump(exclude_none=True), - ] - - json_path = tmp_path / "tasks.json" - json_path.write_text(json.dumps(entries), encoding="utf-8") - jsonl_path = tmp_path / "tasks.jsonl" - jsonl_path.write_text("\n".join(json.dumps(entry) for entry in entries), encoding="utf-8") - - assert [t.slug for t in Taskset.from_file(json_path)] == ["one", "two"] - assert [t.slug for t in Taskset.from_file(jsonl_path)] == ["one", "two"] - - -def test_file_roundtrip_keeps_rows_and_env_names(tmp_path) -> None: - authored = [ - Task(env="authored", id="solve", args={"n": 1}, slug="one"), - Task(env="authored", id="solve", args={"n": 2}, slug="two"), - ] - out = Taskset("demo", authored).to_file(tmp_path / "tasks.json") - - loaded = Taskset.from_file(out) - - assert [t.slug for t in loaded] == ["one", "two"] - assert all(t.env == "authored" for t in loaded) - assert list(loaded) == authored # rows survive the file intact (value equality) - - -def test_taskset_file_preserves_local_compose_project(tmp_path: Path) -> None: - compose = tmp_path / "compose.yaml" - compose.write_text("services:\n main:\n image: alpine:3.21\n", encoding="utf-8") - output = Taskset( - "compose", - [ - Task( - env="compose", - id="solve", - runtime_config=RuntimeConfig( - compose=ComposeProject(document=compose, root=tmp_path), - ), - ), - ], - ).to_file(tmp_path / "tasks.json") - - row = json.loads(output.read_text(encoding="utf-8"))[0] - assert row["runtime_config"]["compose"] == { - "document": "compose.yaml", - "root": ".", - } - loaded = next(iter(Taskset.from_file(output))) - assert loaded.runtime_config is not None - assert loaded.runtime_config.compose == ComposeProject( - document=compose.resolve(), - root=tmp_path.resolve(), - ) - - -def test_taskset_to_file_writes_json_and_jsonl(tmp_path) -> None: - taskset = Taskset( - "demo", - [ - Task(env="e", id="solve", args={"n": 1}, slug="one"), - Task(env="e", id="solve", args={"n": {"x": 2}}, slug="two"), - ], - ) - - json_path = taskset.to_file(tmp_path / "tasks.json") - jsonl_path = taskset.to_file(tmp_path / "tasks.jsonl") - - assert [entry["slug"] for entry in json.loads(json_path.read_text())] == ["one", "two"] - assert [json.loads(line)["slug"] for line in jsonl_path.read_text().splitlines()] == [ - "one", - "two", - ] - with pytest.raises(ValueError, match=r"use \.json or \.jsonl"): - taskset.to_file(tmp_path / "tasks.txt") - - -def test_taskset_from_module_collects_public_tasks(tmp_path) -> None: - module = tmp_path / "local_tasks.py" - module.write_text( - """ -from hud import Task - -local = Task(env="module-env", id="solve", args={"n": 1}, slug="local") -""".strip(), - encoding="utf-8", - ) - - assert Taskset.from_module(module)["local"].args == {"n": 1} - - -def test_taskset_from_api_uses_remote_records(monkeypatch: pytest.MonkeyPatch) -> None: - def fake_request(method: str, url: str, **kwargs: object) -> dict[str, object]: - assert method == "GET" - if url.endswith("/tasksets/by-name/demo"): - return {"taskset_id": "ts_123", "name": "Demo"} - if url.endswith("/tasksets/ts_123/export"): - return { - "name": "Demo", - "tasks": [ - { - # CP export shape: the legacy env qualifier is stripped - # server-side, so env + bare scenario arrive already split. - "env": "e", - "scenario": "solve", - "args": {"n": 1}, - "name": "one", - } - ], - } - raise AssertionError(url) - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - monkeypatch.setattr("hud.settings.settings.api_key", "test-key") - - taskset = Taskset.from_api("demo") - - assert taskset.name == "Demo" - assert taskset.taskset_id == "ts_123" - assert taskset["one"].id == "solve" - assert taskset["one"].env == "e" - assert taskset["one"].args == {"n": 1} diff --git a/hud/integrations/harbor/Dockerfile b/hud/integrations/harbor/Dockerfile index 5ffff5e3a..3872f1361 100644 --- a/hud/integrations/harbor/Dockerfile +++ b/hud/integrations/harbor/Dockerfile @@ -13,10 +13,12 @@ FROM python:3.12-slim AS runtime USER root ARG HUD_REQUIREMENT=hud COPY --from=ghcr.io/astral-sh/uv:0.8.15 /uv /controller/bin/uv -COPY install.sh config.json /controller/ +COPY install.sh /controller/ COPY packages /controller/packages -COPY --from=authored-root / /rootfs RUN sh /controller/install.sh "${HUD_REQUIREMENT}" && mkdir -p /runtime +# Task-specific inputs follow the install, so every task reuses its layer. +COPY config.json /controller/ +COPY --from=authored-root / /rootfs ENV HUD_SKIP_VERSION_CHECK=1 EXPOSE 8765 diff --git a/hud/integrations/harbor/adapt.py b/hud/integrations/harbor/adapt.py index 2fde36430..c1ed443d3 100644 --- a/hud/integrations/harbor/adapt.py +++ b/hud/integrations/harbor/adapt.py @@ -25,6 +25,7 @@ ComposeProject, ComposeService, ComposeUnboundVariableError, + ComposeUnsupportedError, ) from hud.utils.naming import normalize_environment_name from hud.version import __version__ @@ -241,7 +242,7 @@ def add(code: str, message: str) -> None: return None, ( AdaptFinding(code="harbor.invalid.task_config_io", kind="invalid", message=str(error)), ) - except tomllib.TOMLDecodeError as error: + except (tomllib.TOMLDecodeError, UnicodeDecodeError) as error: return None, ( AdaptFinding( code="harbor.invalid.task_config_toml", kind="invalid", message=str(error) @@ -297,6 +298,9 @@ def add(code: str, message: str) -> None: except ComposeUnboundVariableError as error: add("harbor.unsupported.host_compose_variable", str(error)) compose = None + except ComposeUnsupportedError as error: + add("harbor.unsupported.compose_include_extends", str(error)) + compose = None except (OSError, ValueError, ValidationError) as error: add("harbor.invalid.compose", str(error)) compose = None diff --git a/hud/integrations/harbor/export.py b/hud/integrations/harbor/export.py index 4eb4bbe7c..752724b21 100644 --- a/hud/integrations/harbor/export.py +++ b/hud/integrations/harbor/export.py @@ -11,7 +11,7 @@ from hud.environment import Environment, load_environment from hud.environment.server import TaskRunner -from hud.eval import Taskset +from hud.eval import Task, Taskset from hud.utils.naming import normalize_environment_name ALLOWED_PROTOCOLS = ("ssh", "mcp") @@ -118,6 +118,8 @@ def ignore_export(dirpath: str, names: list[str]) -> set[str]: f"only {'/'.join(ALLOWED_PROTOCOLS)} are convertible.", ) + # Every task is checked before any is written, so a refused export leaves no folders. + planned: list[tuple[Task, Environment, str]] = [] for task in tasks: env = authored[task.env] if task.id not in env.tasks: @@ -139,7 +141,9 @@ def ignore_export(dirpath: str, names: list[str]) -> set[str]: f"directory {slug!r}; give them distinct slugs" ) claimed[slug] = declared + planned.append((task, env, slug)) + for task, env, slug in planned: task_dir = out / slug tests_dir = task_dir / "tests" tests_dir.mkdir(parents=True, exist_ok=True) diff --git a/hud/integrations/harbor/tests/conftest.py b/hud/integrations/harbor/tests/conftest.py deleted file mode 100644 index d50c86d90..000000000 --- a/hud/integrations/harbor/tests/conftest.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Builders for synthetic Harbor-format task directories (terminal-bench layout): - -task_name/ -├── task.toml -├── instruction.md -├── environment/Dockerfile -└── tests/test.sh -""" - -from __future__ import annotations - -import textwrap -from pathlib import Path # noqa: TC003 - used at runtime - -import pytest - -_DEFAULT_TASK_TOML = textwrap.dedent("""\ - [metadata] - category = "systems" - difficulty = "medium" - tags = ["bash", "linux"] - - [verifier] - timeout_sec = 120 -""") - -_ML_TASK_TOML = textwrap.dedent("""\ - [metadata] - category = "machine-learning" - difficulty = "hard" - tags = ["python", "ml"] - - [docker] - image = "alexgshaw/caffe-cifar-10:20251031" - - [verifier] - timeout_sec = 300 -""") - -_SIMPLE_DOCKERFILE = textwrap.dedent("""\ - FROM python:3.11-slim - RUN apt-get update && apt-get install -y curl git - WORKDIR /workspace - CMD ["bash"] -""") - -_ML_DOCKERFILE = textwrap.dedent("""\ - FROM nvidia/cuda:12.0-runtime-ubuntu22.04 - RUN apt-get update && apt-get install -y python3 python3-pip - WORKDIR /workspace - ENTRYPOINT ["/bin/bash"] -""") - - -def make_harbor_task( - parent: Path, - name: str, - instruction: str = "Solve the task.", - task_toml: str = _DEFAULT_TASK_TOML, - dockerfile: str | None = _SIMPLE_DOCKERFILE, -) -> Path: - """Create a synthetic Harbor task directory under *parent*; return it.""" - task_dir = parent / name - task_dir.mkdir(parents=True, exist_ok=True) - (task_dir / "instruction.md").write_text(instruction, encoding="utf-8") - (task_dir / "task.toml").write_text(task_toml, encoding="utf-8") - if dockerfile is not None: - env_dir = task_dir / "environment" - env_dir.mkdir(exist_ok=True) - (env_dir / "Dockerfile").write_text(dockerfile, encoding="utf-8") - tests_dir = task_dir / "tests" - tests_dir.mkdir(exist_ok=True) - (tests_dir / "test.sh").write_text( - '#!/bin/bash\necho "1.0" > /logs/verifier/reward.txt\n', encoding="utf-8" - ) - return task_dir - - -def make_multi_step_task( - parent: Path, - name: str, - steps: tuple[str, ...] = ("plan", "implement"), - dockerfile: str | None = _SIMPLE_DOCKERFILE, -) -> Path: - """Create a synthetic multi-step Harbor task: no root instruction.md; each - step under steps// carries its own instruction, declared by a - [[steps]] array in task.toml.""" - task_dir = parent / name - task_dir.mkdir(parents=True, exist_ok=True) - steps_toml = "".join(f'\n[[steps]]\nname = "{step}"\n' for step in steps) - (task_dir / "task.toml").write_text(_DEFAULT_TASK_TOML + steps_toml, encoding="utf-8") - if dockerfile is not None: - env_dir = task_dir / "environment" - env_dir.mkdir(exist_ok=True) - (env_dir / "Dockerfile").write_text(dockerfile, encoding="utf-8") - for step in steps: - step_dir = task_dir / "steps" / step - (step_dir / "tests").mkdir(parents=True, exist_ok=True) - (step_dir / "instruction.md").write_text(f"# {step}\n", encoding="utf-8") - (step_dir / "tests" / "test.sh").write_text( - '#!/bin/bash\necho "1.0" > /logs/verifier/reward.txt\n', encoding="utf-8" - ) - return task_dir - - -@pytest.fixture() -def single_task(tmp_path: Path) -> Path: - """A single standalone Harbor task directory.""" - return make_harbor_task( - tmp_path, - "cancel-async-tasks", - instruction="# Cancel Async Tasks\n\nCancel 5 asyncio tasks within 2 seconds.\n", - ) - - -@pytest.fixture() -def dataset_same_env(tmp_path: Path) -> Path: - """A dataset directory with 3 tasks sharing the same Dockerfile.""" - dataset = tmp_path / "terminal-bench-sample" - dataset.mkdir() - for name in ("cancel-async-tasks", "build-pmars", "chess-best-move"): - make_harbor_task(dataset, name, instruction=f"# {name}\n\nSolve the {name} task.\n") - return dataset - - -@pytest.fixture() -def dataset_multi_env(tmp_path: Path) -> Path: - """A dataset directory with tasks split across 2 different Dockerfiles.""" - dataset = tmp_path / "Mixed Bench" - dataset.mkdir() - for name in ("cancel-async-tasks", "build-pmars"): - make_harbor_task(dataset, name, dockerfile=_SIMPLE_DOCKERFILE) - for name in ("caffe-cifar-10", "sam-cell-seg"): - make_harbor_task(dataset, name, task_toml=_ML_TASK_TOML, dockerfile=_ML_DOCKERFILE) - return dataset diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/solution/solve.sh b/hud/integrations/harbor/tests/tasks/hello-mcp/solution/solve.sh deleted file mode 100644 index 211690d9a..000000000 --- a/hud/integrations/harbor/tests/tasks/hello-mcp/solution/solve.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -set -eu -printf '%s' 'harbor-mcp-secret-12345' > /app/secret.txt diff --git a/hud/integrations/harbor/tests/test_contract.py b/hud/integrations/harbor/tests/test_contract.py deleted file mode 100644 index 2e16c960f..000000000 --- a/hud/integrations/harbor/tests/test_contract.py +++ /dev/null @@ -1,1684 +0,0 @@ -"""Observable contracts for adapting Harbor tasks into HUD images.""" - -from __future__ import annotations - -import importlib -import json -import os -import subprocess -from pathlib import Path -from typing import Any - -import pytest - -from hud.environment import Mount, Peer -from hud.eval import RuntimeGPU, RuntimeLimits, RuntimeResources, RuntimeTPU, Taskset -from hud.eval.runtime.compose import ComposeConfig -from hud.integrations import harbor -from hud.integrations.harbor.config import ControllerConfig, TaskSpec - -from .conftest import make_harbor_task, make_multi_step_task - -adapt_module = importlib.import_module("hud.integrations.harbor.adapt") -build_module = importlib.import_module("hud.integrations.harbor.build") - - -@pytest.fixture(autouse=True) -def resolved_images(monkeypatch: pytest.MonkeyPatch) -> None: - def resolve( - source: Any, - compose_project: Any, - *, - verifier_image: str, - peer_services: set[str], - ) -> Any: - del compose_project, verifier_image - dockerfile = source.dockerfile.read_text("utf-8") if source.dockerfile.is_file() else "" - main: dict[str, Any] = { - "Env": [], - "User": "", - "WorkingDir": "/workspace", - "Entrypoint": [], - "ExposedPorts": {}, - } - if "USER 1000:2000" in dockerfile: - main["User"] = "1000:2000" - if "WORKDIR /workspace" not in dockerfile: - main["WorkingDir"] = "" - if "ENV IMAGE_ONLY=present VALUE_WITH_EQUALS=one=two" in dockerfile: - main["Env"] = ["IMAGE_ONLY=present", "VALUE_WITH_EQUALS=one=two"] - if 'ENTRYPOINT ["/usr/local/bin/start-environment"]' in dockerfile: - main["Entrypoint"] = ["/usr/local/bin/start-environment"] - if "EXPOSE 3128 3129 8765 8080" in dockerfile: - main["ExposedPorts"] = {f"{port}/tcp": {} for port in (3128, 3129, 8765, 8080)} - peers = { - service: { - "Env": [], - "User": "", - "WorkingDir": "", - "Entrypoint": [], - "ExposedPorts": {"6379/tcp": {}}, - } - for service in peer_services - } - return build_module.ResolvedImages(main=main, verifier=main, peers=peers) - - monkeypatch.setattr(adapt_module, "require_docker", lambda: None) - monkeypatch.setattr(adapt_module, "resolve_images", resolve) - - -def _adapt(path: Path, *, hud_requirement: str = "hud") -> Taskset: - result = harbor.adapt(path, hud_requirement=hud_requirement) - assert result.failures == () - return result.taskset - - -def _failure(path: Path) -> harbor.AdaptFailure: - result = harbor.adapt(path) - assert list(result.taskset) == [] - assert len(result.failures) == 1 - return result.failures[0] - - -def _tree_snapshot(root: Path) -> dict[str, tuple[str, bytes | str]]: - snapshot: dict[str, tuple[str, bytes | str]] = {} - for entry in sorted(root.rglob("*")): - relative = entry.relative_to(root).as_posix() - if entry.is_symlink(): - snapshot[relative] = ("symlink", os.readlink(entry)) - elif entry.is_file(): - snapshot[relative] = ("file", entry.read_bytes()) - else: - snapshot[relative] = ("directory", b"") - return snapshot - - -def _assert_stock_compose_complete(compose_path: Path) -> dict[str, Any]: - project = json.loads(compose_path.read_text("utf-8")) - services = project["services"] - assert isinstance(services, dict) - for name, service in services.items(): - assert isinstance(service, dict) - assert service.get("image"), name - for volume in service.get("volumes", []): - if not isinstance(volume, str): - continue - source, separator, target = volume.partition(":") - if separator and target == "/controller/tests:ro": - tests = (compose_path.parent / source).resolve() - tests.relative_to(compose_path.parent.resolve()) - assert tests.is_dir(), (name, tests) - build = service.get("build") - if build is None: - continue - build_config = {"context": build} if isinstance(build, str) else build - assert isinstance(build_config, dict) - context_value = build_config.get("context", ".") - assert isinstance(context_value, str) - context = (compose_path.parent / context_value).resolve() - context.relative_to(compose_path.parent.resolve()) - assert context.is_dir(), (name, context) - dockerfile_value = build_config.get("dockerfile", "Dockerfile") - assert isinstance(dockerfile_value, str) - assert (context / dockerfile_value).is_file(), (name, dockerfile_value) - additional_contexts = build_config.get("additional_contexts", {}) - assert isinstance(additional_contexts, dict) - for target in additional_contexts.values(): - assert isinstance(target, str) - if target.startswith("service:"): - assert target.removeprefix("service:") in services - else: - # Local additional-context paths resolve against the project - # directory (the compose file's directory), not the service - # build context. - named_context = (compose_path.parent / target).resolve() - named_context.relative_to(compose_path.parent.resolve()) - assert named_context.is_dir(), (name, named_context) - return project - - -def _environment_config(context: Path) -> dict[str, Any]: - paths = list((context / "compose-project").glob("*/config.json")) - assert len(paths) == 1 - config = json.loads(paths[0].read_text("utf-8")) - assert isinstance(config, dict) - return config - - -def test_image_resolution_builds_and_inspects_the_authored_image( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - calls: list[tuple[str, ...]] = [] - config = { - "Env": ["FROM_IMAGE=yes"], - "User": "1000:1000", - "WorkingDir": "/workspace", - "Entrypoint": ["/bin/start"], - "ExposedPorts": {"8080/tcp": {}}, - } - - overrides: list[dict[str, Any]] = [] - - def docker(*args: str, **_kwargs: Any) -> str: - calls.append(args) - if args[0] == "compose": - overrides.append(json.loads(Path(args[-3]).read_text("utf-8"))["services"]["main"]) - if args[:2] == ("image", "inspect"): - return json.dumps([{"Config": config}]) - return "" - - monkeypatch.setattr(build_module, "docker", docker) - - resolved = build_module.resolve_images( - source, - source.compose.with_project_directory("./environment"), - verifier_image=source.base_image, - peer_services=set(), - ) - - assert calls[0][0] == "compose" - assert calls[0][-2:] == ("build", "main") - assert overrides == [ - { - "image": source.base_image, - "build": {"context": ".", "dockerfile": "Dockerfile"}, - } - ] - assert calls[1] == ("image", "inspect", source.base_image) - assert resolved.main == config - assert resolved.verifier == config - assert resolved.peers == {} - - -def test_image_resolution_builds_compose_main_from_the_environment_dockerfile( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n image: authored/main:latest\n", - encoding="utf-8", - ) - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - built: list[dict[str, Any]] = [] - inspected: list[str] = [] - - def docker(*args: str, **_kwargs: Any) -> str: - if args[0] == "compose": - *_, override_path, operation, service = args - assert (operation, service) == ("build", "main") - built.append(json.loads(Path(override_path).read_text("utf-8"))["services"]["main"]) - return "" - assert args[:2] == ("image", "inspect") - inspected.append(args[2]) - return json.dumps([{"Config": {"Env": []}}]) - - monkeypatch.setattr(build_module, "docker", docker) - - build_module.resolve_images( - source, - source.compose, - verifier_image=source.base_image, - peer_services=set(), - ) - - assert built == [ - {"image": source.base_image, "build": {"context": ".", "dockerfile": "Dockerfile"}} - ] - assert inspected == [source.base_image] - - -def test_image_resolution_builds_the_interpolated_compose_document( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n build:\n context: ${CONTEXT_DIR}\n" - " image: ${MAIN_IMAGE_NAME}\n", - encoding="utf-8", - ) - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - documents: list[dict[str, Any]] = [] - - def docker(*args: str, **_kwargs: Any) -> str: - if args[0] == "compose": - document = args[args.index("--file") + 1] - documents.append(json.loads(Path(document).read_text("utf-8"))) - return "" - return json.dumps([{"Config": {"Env": []}}]) - - monkeypatch.setattr(build_module, "docker", docker) - - build_module.resolve_images( - source, - source.compose, - verifier_image=source.base_image, - peer_services=set(), - ) - - (document,) = documents - assert document["services"]["main"]["build"] == {"context": "."} - assert document["services"]["main"]["image"] == source.base_image - - -def test_separate_verifier_builds_for_the_task_image_platform( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "separate") - (task / "tests" / "Dockerfile").write_text("FROM python:3.12-slim\n", encoding="utf-8") - (task / "task.toml").write_text('[verifier]\nenvironment_mode = "separate"\n', encoding="utf-8") - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - builds: list[tuple[str, ...]] = [] - - def docker(*args: str, **_kwargs: Any) -> str: - if args[0] == "build": - builds.append(args) - if args[:3] == ("image", "inspect", "--format"): - return "linux/amd64" - if args[:2] == ("image", "inspect"): - return json.dumps([{"Config": {"Env": []}}]) - return "" - - monkeypatch.setattr(build_module, "docker", docker) - - build_module.resolve_images( - source, source.compose, verifier_image="verifier:test", peer_services=set() - ) - - (build,) = builds - assert build[build.index("--platform") + 1] == "linux/amd64" - assert build[build.index("--tag") + 1] == "verifier:test" - - -def test_separate_verifier_build_failure_names_the_platform_override( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "separate") - (task / "tests" / "Dockerfile").write_text("FROM python:3.12-slim\n", encoding="utf-8") - (task / "task.toml").write_text('[verifier]\nenvironment_mode = "separate"\n', encoding="utf-8") - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - - def docker(*args: str, **_kwargs: Any) -> str: - if args[0] == "build": - raise build_module.ImageResolutionError("docker build failed: no matching wheel") - if args[0] == "version": - return "linux/arm64" - if args[:3] == ("image", "inspect", "--format"): - return "linux/amd64" - if args[:2] == ("image", "inspect"): - return json.dumps([{"Config": {"Env": []}}]) - return "" - - monkeypatch.setattr(build_module, "docker", docker) - - with pytest.raises(build_module.ImageResolutionError, match="DOCKER_DEFAULT_PLATFORM"): - build_module.resolve_images( - source, source.compose, verifier_image="verifier:test", peer_services=set() - ) - - -def test_adapt_reports_tasks_whose_images_cannot_be_resolved( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - make_harbor_task(tmp_path, "broken", dockerfile="FROM registry.invalid/missing\n") - make_harbor_task(tmp_path, "working") - resolve = adapt_module.resolve_images - - def fail_broken(source: Any, *args: Any, **kwargs: Any) -> Any: - if source.path.name == "broken": - raise build_module.ImageResolutionError("docker build failed: pull access denied") - return resolve(source, *args, **kwargs) - - monkeypatch.setattr(adapt_module, "resolve_images", fail_broken) - - result = harbor.adapt(tmp_path) - - assert [task.slug for task in result.taskset] == ["working"] - (failure,) = result.failures - assert failure.task == "broken" - assert [(finding.code, finding.kind) for finding in failure.findings] == [ - ("harbor.invalid.image", "invalid") - ] - assert "pull access denied" in failure.findings[0].message - - -def test_adapt_runs_sidecars_that_expose_no_ports( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n worker:\n image: busybox\n", - encoding="utf-8", - ) - resolve = adapt_module.resolve_images - - def without_peer_ports(source: Any, *args: Any, **kwargs: Any) -> Any: - resolved = resolve(source, *args, **kwargs) - return build_module.ResolvedImages( - main=resolved.main, - verifier=resolved.verifier, - peers={name: {**config, "ExposedPorts": {}} for name, config in resolved.peers.items()}, - ) - - monkeypatch.setattr(adapt_module, "resolve_images", without_peer_ports) - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - assert isinstance(row.runtime_config.compose.document, Path) - project = json.loads(row.runtime_config.compose.document.read_text("utf-8")) - assert "worker" in project["services"] - (context,) = (tmp_path / ".hud-adapt").iterdir() - assert _environment_config(context)["peers"] == [] - - -def test_adapt_maps_the_agent_budget_to_the_agent_config(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - "[environment]\nbuild_timeout_sec = 900\n\n[agent]\ntimeout_sec = 28800\n", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.agent_config == {"timeout_seconds": 28800.0} - assert row.runtime_config is not None - assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=900) - - -def test_adapt_binds_the_compose_variables_harbor_defines(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - "[environment]\ncpus = 2\nmemory_mb = 4096\n", - encoding="utf-8", - ) - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - build: - context: ${CONTEXT_DIR} - image: ${MAIN_IMAGE_NAME} - deploy: - resources: - limits: - cpus: ${CPUS} - memory: ${MEMORY} -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - assert isinstance(row.runtime_config.compose.document, Path) - services = json.loads(row.runtime_config.compose.document.read_text("utf-8"))["services"] - assert services["main"]["deploy"]["resources"]["limits"] == {"cpus": "2", "memory": "4096M"} - assert services["hud-base"]["build"] == {"context": "./compose-project/environment"} - - -def test_adapt_aborts_when_the_docker_daemon_is_unreachable( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - make_harbor_task(tmp_path / "dataset", "task-a") - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - docker = bin_dir / "docker" - docker.write_text( - "#!/bin/sh\necho 'Cannot connect to the Docker daemon' >&2\nexit 1\n", - encoding="utf-8", - ) - docker.chmod(0o755) - monkeypatch.setenv("PATH", str(bin_dir)) - monkeypatch.setattr(adapt_module, "require_docker", build_module.require_docker) - - with pytest.raises(RuntimeError, match="requires a reachable Docker daemon"): - harbor.adapt(tmp_path / "dataset") - - -def test_adapt_packages_an_image_task_as_a_compose_project(tmp_path: Path) -> None: - task_dir = make_harbor_task(tmp_path, "task-a") - authored_environment = _tree_snapshot(task_dir / "environment") - - taskset = _adapt(tmp_path) - - (task,) = list(taskset) - assert task.id == "run" - assert task.slug == "task-a" - assert task.args["instruction"] == "Solve the task." - assert task.args["task"] == { - "artifacts": [], - "collect": [], - "description": "", - "id": "task-a", - "separate_verifier": False, - "verifier_timeout": 120.0, - } - assert task.runtime_config is not None - assert task.runtime_config.image is None - - (context,) = (tmp_path / ".hud-adapt").iterdir() - assert not (context / "Dockerfile").exists() - assert not any(path.name == ".hud" for path in context.rglob(".hud")) - assert {entry.name for entry in context.iterdir()} == { - "compose-project", - "compose.yaml", - "env.py", - "tasks.json", - } - # `hud deploy` resolves the context's identity from a literal Environment(...). - assert f"Environment({context.name!r})" in (context / "env.py").read_text("utf-8") - project_root = context / "compose-project" - assert _tree_snapshot(project_root / "environment") == authored_environment - payload = project_root / "main" - assert {entry.name for entry in payload.iterdir()} == { - "Dockerfile", - "config.json", - "install.sh", - "packages", - } - config = json.loads((payload / "config.json").read_text("utf-8")) - assert "controller_root" not in config - assert "rootfs" not in config - assert "runtime_root" not in config - assert config["mounts"] == [] - assert not (project_root / "build.sh").exists() - assert (project_root / "tests" / "task-a" / "test.sh").is_file() - assert not any(path.name in {"tasks", "tasks.json"} for path in payload.rglob("*")) - assert not (project_root / "compose.json").exists() - assert task.runtime_config.compose is not None - assert task.runtime_config.compose.document == context / "compose.yaml" - assert task.runtime_config.compose.root == context - project = _assert_stock_compose_complete(context / "compose.yaml") - assert set(project["services"]) == {"main", "hud-base"} - assert project["services"]["hud-base"]["build"] == {"context": "./compose-project/environment"} - assert project["services"]["hud-base"]["scale"] == 0 - main = project["services"]["main"] - assert main["image"].startswith("hud-harbor:") - assert main["build"] == { - "additional_contexts": {"hud-base": "service:hud-base"}, - "args": { - "BASE_IMAGE": "hud-base", - "HUD_REQUIREMENT": "hud", - "VERIFIER_IMAGE": project["services"]["hud-base"]["image"], - }, - "context": "./compose-project/main", - "target": "plain", - } - assert "HUD_RUNTIME_ROOT" not in main.get("environment", {}) - assert main["volumes"] == ["./compose-project/tests:/controller/tests:ro"] - assert not (project_root / "Dockerfile").exists() - - -def test_task_content_changes_do_not_rebuild_the_environment(tmp_path: Path) -> None: - task_dir = make_harbor_task(tmp_path, "task-a", instruction="First instruction") - - (before,) = list(_adapt(tmp_path)) - assert before.runtime_config is not None - assert before.runtime_config.compose is not None - assert isinstance(before.runtime_config.compose.document, Path) - before_compose = json.loads(before.runtime_config.compose.document.read_text("utf-8")) - before_image = before_compose["services"]["main"]["image"] - - (task_dir / "instruction.md").write_text("Second instruction", encoding="utf-8") - (task_dir / "tests" / "test.sh").write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") - (after,) = list(_adapt(tmp_path)) - assert after.runtime_config is not None - assert after.runtime_config.compose is not None - assert isinstance(after.runtime_config.compose.document, Path) - after_compose = json.loads(after.runtime_config.compose.document.read_text("utf-8")) - - assert after_compose["services"]["main"]["image"] == before_image - assert after.args["instruction"] == "Second instruction" - (other_release,) = list(_adapt(tmp_path, hud_requirement="hud==0.0.1")) - assert other_release.runtime_config is not None - assert other_release.runtime_config.compose is not None - assert isinstance(other_release.runtime_config.compose.document, Path) - other_compose = json.loads(other_release.runtime_config.compose.document.read_text("utf-8")) - assert other_compose["services"]["main"]["image"] != before_image - assert ( - after.runtime_config.compose.document.parent - / "compose-project" - / "tests" - / "task-a" - / "test.sh" - ).read_text("utf-8") == "#!/bin/sh\nexit 1\n" - - -def test_image_task_keeps_non_recipe_compose_names_as_context_files(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - content = '{"api_gateway": {"interval": "30s"}}\n' - (task / "environment" / "docker-compose.yml").write_text(content, encoding="utf-8") - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - environment = context / "compose-project" / "environment" - assert (environment / "docker-compose.yml").read_text("utf-8") == content - project = json.loads((context / "compose.yaml").read_text("utf-8")) - assert set(project["services"]) == {"main", "hud-base"} - - -def test_image_task_preserves_environment_ignored_paths_verbatim( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - environment = task / "environment" - (environment / ".dockerignore").write_text("hud/\nDockerfile\n", encoding="utf-8") - (environment / "ignored.txt").write_bytes(b"unchanged\x00payload") - authored = _tree_snapshot(environment) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - project = context / "compose-project" - assert _tree_snapshot(project / "environment") == authored - assert (project / "main" / "Dockerfile").is_file() - - -def test_adapt_honors_compose_main_build_settings( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a", dockerfile=None) - environment = task / "environment" - environment.mkdir() - (environment / "docker-compose.yaml").write_text( - """\ -services: - main: - build: - context: . - dockerfile: Containerfile - args: - FLAVOR: compose -""", - encoding="utf-8", - ) - (environment / "Containerfile").write_text("FROM python:3.12\n", encoding="utf-8") - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - compose_path = row.runtime_config.compose.document - assert isinstance(compose_path, Path) - project = _assert_stock_compose_complete(compose_path) - base = project["services"]["hud-base"] - assert base["build"]["dockerfile"] == "Containerfile" - assert base["build"]["args"] == {"FLAVOR": "compose"} - assert base["scale"] == 0 - - -def test_adapt_emits_compose_project_and_peers( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - environment: {FROM_COMPOSE: "yes"} - expose: [8080] - healthcheck: - test: [CMD-SHELL, curl -f http://localhost:8080/health] - interval: 2s - timeout: 3s - retries: 5 - start_period: 1s - redis: - image: redis:7-alpine - depends_on: - main: - condition: service_healthy - restart: true - command: [redis-server, --save, ""] - environment: {SIDE: car} - expose: [6379] - healthcheck: - test: [CMD, redis-cli, ping] - interval: 2s - timeout: 3s - retries: 5 - start_period: 1s -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.image is None - assert row.runtime_config.compose is not None - assert row.runtime_config.compose.service_access is None - (context,) = (tmp_path / ".hud-adapt").iterdir() - assert row.runtime_config.compose.document == context / "compose.yaml" - assert row.runtime_config.compose.root == context - assert not (context / "compose-project" / "compose.json").exists() - project = _assert_stock_compose_complete(context / "compose.yaml") - assert project["services"]["redis"]["image"] == "redis:7-alpine" - assert "build" not in project["services"]["redis"] - assert project["services"]["main"]["build"]["context"] == "./compose-project/main" - assert project["services"]["main"]["build"]["target"] == "plain" - assert project["services"]["main"]["build"]["additional_contexts"] == { - "hud-base": "service:hud-base" - } - assert project["services"]["main"]["image"].startswith("hud-harbor:") - assert project["services"]["hud-base"]["image"].startswith("hud-harbor-base:") - assert project["services"]["hud-base"]["build"]["context"] == "./compose-project/environment" - assert (context / "compose-project" / "environment" / "Dockerfile").is_file() - assert not any(path.name == ".hud" for path in context.rglob(".hud")) - redis = project["services"]["redis"] - assert redis["image"] == "redis:7-alpine" - assert redis["environment"] == {"SIDE": "car"} - assert redis["command"] == ["redis-server", "--save", ""] - assert redis["depends_on"] == {"main": {"condition": "service_started", "restart": True}} - assert redis["expose"] == ["6379"] - assert redis["healthcheck"]["test"] == ["CMD", "redis-cli", "ping"] - assert "build" not in redis - manifest = _environment_config(context) - assert manifest["environment"]["env"] == {"FROM_COMPOSE": "yes"} - assert manifest["environment"]["healthcheck"] == { - "command": "curl -f http://localhost:8080/health", - "interval_sec": 2.0, - "timeout_sec": 3.0, - "start_period_sec": 1.0, - "start_interval_sec": 5.0, - "retries": 5, - } - assert manifest["local_aliases"] == ["main"] - assert manifest["ports"] == [8080] - assert manifest["capabilities"] == [] - assert ControllerConfig.model_validate(manifest).peers == [ - Peer("redis", 6379, target=("redis", 6379)) - ] - assert ComposeConfig.from_file(context / "compose.yaml").healthchecked_services() == ["redis"] - assert project["services"]["main"]["command"] == [ - "/controller/venv/bin/hud", - "serve", - "hud.integrations.harbor.env:env", - "--host", - "0.0.0.0", - "--port", - "8765", - ] - assert "healthcheck" not in project["services"]["main"] - - -def test_compose_adapt_retains_source_builds_in_the_generated_project( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "Dockerfile").write_text( - 'FROM python:3.12\nWORKDIR /app\nENTRYPOINT ["/app/start"]\n', - encoding="utf-8", - ) - database = task / "environment" / "database" - database.mkdir() - (database / "Dockerfile").write_text("FROM postgres:16\n", encoding="utf-8") - (database / "db.env").write_text("POSTGRES_DB=test\n", encoding="utf-8") - (database / "data").mkdir() - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - env_file: ./main.env - volumes: - - ./main-data:/var/lib/main - - type: bind - source: ./readonly - target: /etc/authored - read_only: true - database: - build: - context: ./database - env_file: ./database/db.env - volumes: [./database/data:/var/lib/postgresql/data] - expose: [5432] - redis: - image: redis:7-alpine - expose: [6379] -""", - encoding="utf-8", - ) - (task / "environment" / "main.env").write_text("MAIN=true\n", encoding="utf-8") - (task / "environment" / "main-data").mkdir() - (task / "environment" / "readonly").mkdir() - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - compose_path = row.runtime_config.compose.document - assert isinstance(compose_path, Path) - project = json.loads(compose_path.read_text("utf-8")) - assert project["services"]["database"]["build"]["context"] == ( - "./compose-project/environment/database" - ) - assert project["services"]["database"]["image"].startswith("hud-harbor-sidecar:") - assert ( - project["services"]["database"]["env_file"] - == "./compose-project/environment/database/db.env" - ) - assert project["services"]["database"]["volumes"] == [ - "./compose-project/environment/database/data:/var/lib/postgresql/data" - ] - assert project["services"]["main"]["env_file"] == "./compose-project/environment/main.env" - assert ( - "./compose-project/environment/main-data:/mounts/0" - in (project["services"]["main"]["volumes"]) - ) - assert { - "type": "bind", - "source": "./compose-project/environment/readonly", - "target": "/mounts/1", - "read_only": True, - } in project["services"]["main"]["volumes"] - context = row.runtime_config.compose.root - assert isinstance(context, Path) - manifest = _environment_config(context) - assert ControllerConfig.model_validate(manifest).mounts == [ - Mount("rw", src="/mounts/0", dst="/var/lib/main"), - Mount("ro", src="/mounts/1", dst="/etc/authored"), - ] - assert project["services"]["redis"]["image"] == "redis:7-alpine" - assert project["services"]["main"]["build"]["additional_contexts"] == { - "hud-base": "service:hud-base" - } - - -def test_adapt_moves_compose_main_process_settings_into_the_workspace( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - user: 1001:1002 - entrypoint: [/compose-init] - working_dir: /compose-work -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - compose = _assert_stock_compose_complete(context / "compose.yaml") - assert manifest["image_user"] == "1001:1002" - assert manifest["entrypoint"] == ["/compose-init"] - assert manifest["workdir"] == "/compose-work" - assert "user" not in compose["services"]["main"] - assert compose["services"]["main"]["entrypoint"] == [] - - -def test_adapt_moves_compose_main_healthcheck_into_the_workspace( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - healthcheck: - test: [CMD-SHELL, curl -f http://localhost:8080/health] - interval: 2s - timeout: 3s - retries: 5 - start_period: 1s -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - compose = json.loads((context / "compose.yaml").read_text("utf-8")) - assert manifest["environment"]["healthcheck"] == { - "command": "curl -f http://localhost:8080/health", - "interval_sec": 2.0, - "timeout_sec": 3.0, - "start_period_sec": 1.0, - "start_interval_sec": 5.0, - "retries": 5, - } - assert "healthcheck" not in compose["services"]["main"] - - -def test_adapt_uses_compose_healthcheck_defaults(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n healthcheck:\n test: [CMD, 'true']\n", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["environment"]["healthcheck"] == { - "command": "true", - "interval_sec": 30.0, - "timeout_sec": 30.0, - "start_period_sec": 0.0, - "start_interval_sec": 5.0, - "retries": 3, - } - - -def test_adapt_merges_implicit_main_into_authored_compose(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n default:\n image: sidecar:latest\n expose: [1234]\n", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - compose = json.loads((context / "compose.yaml").read_text("utf-8")) - assert {"main", "default"} <= compose["services"].keys() - assert ControllerConfig.model_validate(_environment_config(context)).peers == [ - Peer("default", 1234, target=("default", 1234)) - ] - - -def test_only_network_mcp_servers_become_capabilities( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main: {}\n redis:\n image: redis:7-alpine\n expose: [6379]\n", - encoding="utf-8", - ) - (task / "task.toml").write_text( - """ -[environment] -skills_dir = "/skills" - -[[environment.mcp_servers]] -name = "redis-tools" -transport = "streamable-http" -url = "http://redis:6379/mcp" -args = [] - -[[environment.mcp_servers]] -name = "db" -transport = "stdio" -command = "db-mcp" -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["capabilities"] == [ - { - "name": "redis-tools", - "params": {"transport": "streamable-http"}, - "protocol": "mcp/2025-11-25", - "url": "http://redis:6379/mcp", - } - ] - - -@pytest.mark.parametrize("name", ["shell", "filetracking"]) -def test_mcp_server_names_cannot_shadow_workspace_capabilities( - tmp_path: Path, - name: str, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - f"""\ -[[environment.mcp_servers]] -name = "{name}" -transport = "streamable-http" -url = "http://server:8000/mcp" -""", - encoding="utf-8", - ) - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == ["harbor.invalid.reserved_mcp_name"] - assert name in failure.findings[0].message - - -def test_adapt_groups_identical_images_and_keeps_row_metadata( - dataset_same_env: Path, -) -> None: - (dataset_same_env / "build-pmars" / "task.toml").write_text( - """\ -artifacts = ["/tmp/result"] - -[metadata] -category = "systems" -difficulty = "medium" -tags = ["bash", "linux"] - -[agent] -timeout_sec = 45 - -[verifier] -timeout_sec = 30 -""", - encoding="utf-8", - ) - taskset = _adapt(dataset_same_env) - - assert len(taskset) == 3 - assert len(taskset.environment_names()) == 1 - assert all( - task.columns - == { - "category": "systems", - "difficulty": "medium", - "tags": ["bash", "linux"], - } - for task in taskset - ) - assert all(task.runtime_config is not None for task in taskset) - assert all(task.runtime_config.compose is not None for task in taskset if task.runtime_config) - assert taskset["build-pmars"].agent_config == {"timeout_seconds": 45.0} - assert taskset["build-pmars"].args["task"]["verifier_timeout"] == 30.0 - assert taskset["build-pmars"].args["task"]["artifacts"] == [ - {"service": "main", "source": "/tmp/result"} - ] - - -def test_distinct_environments_build_distinct_images( - dataset_multi_env: Path, -) -> None: - taskset = _adapt(dataset_multi_env) - - assert len(taskset.environment_names()) == 2 - assert all(task.runtime_config is not None for task in taskset) - assert all(task.runtime_config.compose is not None for task in taskset if task.runtime_config) - - -def test_adapt_maps_resources_onto_the_compose_runtime(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "gpu") - (task / "task.toml").write_text( - """ -[metadata] -difficulty = "hard" - -[environment] -cpus = 4 -memory_mb = 8192 -storage_mb = 32768 -gpus = 2 -gpu_types = ["H100"] -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.columns == {"difficulty": "hard"} - assert row.runtime_config is not None - assert row.runtime_config.image is None - assert row.runtime_config.compose is not None - assert isinstance(row.runtime_config.compose.document, Path) - assert row.runtime_config.resources is not None - assert row.runtime_config.resources.cpu == 4 - assert row.runtime_config.resources.memory_mb == 8192 - assert row.runtime_config.resources.storage_mb == 32768 - assert row.runtime_config.resources.gpu is not None - assert row.runtime_config.resources.gpu.count == 2 - assert row.runtime_config.resources.gpu.acceptable_types == ["H100"] - - -def test_env_templates_are_persisted_verbatim_not_resolved( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Host values must never be baked into the content-hashed image payload - or the persisted task rows; ``${VAR}`` templates resolve at runtime.""" - monkeypatch.setenv("HARBOR_JUDGE_KEY", "sk-live-secret") - task = make_harbor_task(tmp_path, "templated") - (task / "task.toml").write_text( - """ -[environment.env] -JUDGE_KEY = "${HARBOR_JUDGE_KEY}" -MODEL = "${HARBOR_JUDGE_MODEL:-gpt-4o}" - -[verifier] -timeout_sec = 60 - -[verifier.env] -VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" -""", - encoding="utf-8", - ) - - harbor.adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["environment"]["env"] == { - "JUDGE_KEY": "${HARBOR_JUDGE_KEY}", - "MODEL": "${HARBOR_JUDGE_MODEL:-gpt-4o}", - } - assert manifest["verifier"]["env"] == {"VERIFIER_KEY": "${HARBOR_JUDGE_KEY}"} - for persisted in sorted(path for path in context.rglob("*") if path.is_file()): - assert b"sk-live-secret" not in persisted.read_bytes(), persisted - - -def test_prebuilt_harbor_image_emits_a_conventional_wrapper_build( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "prebuilt", dockerfile=None) - (task / "task.toml").write_text( - '[environment]\ndocker_image = "registry.example/base:latest"\n', - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - project = context / "compose-project" - services = _assert_stock_compose_complete(context / "compose.yaml")["services"] - assert set(services) == {"main"} - assert services["main"]["build"]["args"]["BASE_IMAGE"] == "registry.example/base:latest" - assert "additional_contexts" not in services["main"]["build"] - assert not (project / "build.sh").exists() - - -def test_zero_gpus_is_a_valid_harbor_resource_declaration( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "cpu-only") - (task / "task.toml").write_text("[environment]\ngpus = 0\n", encoding="utf-8") - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.resources is None - - -def test_runtime_configuration_is_data_not_dockerfile_codegen( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - """ -[environment] -workdir = "/app" -network_mode = "allowlist" -allowed_hosts = ["pypi.org"] - -[environment.env] -SHARED = "yes" - -[environment.healthcheck] -command = "curl -f http://localhost:8080/health" -interval_sec = 2 -timeout_sec = 4 -start_period_sec = 6 -start_interval_sec = 1 -retries = 5 - -[agent] -user = "agent" - -[agent.env] -AGENT_ONLY = "yes" - -[verifier] -user = 0 -network_mode = "no-network" - -[verifier.env] -VERIFIER_ONLY = "yes" -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["workdir"] == "/app" - assert manifest["environment"] == { - "env": {"SHARED": "yes"}, - "network": {"enabled": True, "allowed_hosts": ["pypi.org"]}, - "healthcheck": { - "command": "curl -f http://localhost:8080/health", - "interval_sec": 2.0, - "timeout_sec": 4.0, - "start_period_sec": 6.0, - "start_interval_sec": 1.0, - "retries": 5, - }, - } - assert manifest["agent"]["user"] == "agent" - assert manifest["agent"]["env"] == {"AGENT_ONLY": "yes"} - assert manifest["agent"]["network"] == {"enabled": True, "allowed_hosts": ["pypi.org"]} - assert manifest["verifier"]["user"] == 0 - assert manifest["verifier"]["network"] == {"enabled": False, "allowed_hosts": []} - assert manifest["verifier"]["env"] == {"VERIFIER_ONLY": "yes"} - - -def test_image_entrypoint_is_preserved_as_runtime_data( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "Dockerfile").write_text( - """\ -FROM python:3.12 -USER 1000:2000 -WORKDIR /workspace -ENV IMAGE_ONLY=present VALUE_WITH_EQUALS=one=two -ENTRYPOINT ["/usr/local/bin/start-environment"] -CMD ["ignored-by-harbor"] -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["image_env"] == { - "IMAGE_ONLY": "present", - "VALUE_WITH_EQUALS": "one=two", - } - assert manifest["image_user"] == "1000:2000" - assert manifest["agent"]["user"] == "1000:2000" - assert manifest["verifier"]["user"] == "1000:2000" - assert manifest["workdir"] == "/workspace" - assert manifest["entrypoint"] == ["/usr/local/bin/start-environment"] - - -def test_dataset_adaptation_returns_successes_and_all_detectable_findings( - tmp_path: Path, -) -> None: - make_harbor_task(tmp_path, "supported") - unsupported = make_harbor_task(tmp_path, "unsupported") - (unsupported / "instruction.md").unlink() - (unsupported / "task.toml").write_text( - """\ -[environment] -os = "windows" - -[[environment.mcp_servers]] -name = "shell" -transport = "streamable-http" -""", - encoding="utf-8", - ) - (unsupported / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n image: ${HOST_ONLY_IMAGE}\n", - encoding="utf-8", - ) - - result = harbor.adapt(tmp_path) - - assert [task.slug for task in result.taskset] == ["supported"] - assert len(result.failures) == 1 - failure = result.failures[0] - assert failure.task == "unsupported" - assert {finding.code for finding in failure.findings} == { - "harbor.unsupported.host_compose_variable", - "harbor.invalid.reserved_mcp_name", - "harbor.invalid.mcp_url", - "harbor.invalid.missing_instruction", - } - assert {finding.kind for finding in failure.findings} == {"contract", "invalid"} - - -def test_unbound_compose_variables_are_deliberate_contract_refusals(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n image: ${MAIN_IMAGE}\n", - encoding="utf-8", - ) - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == [ - "harbor.unsupported.host_compose_variable" - ] - assert failure.findings[0].kind == "contract" - - -@pytest.mark.parametrize("port", [3128, 3129, 8765]) -def test_adapt_rejects_main_ports_reserved_by_hud( - tmp_path: Path, - port: int, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - f"services:\n main:\n expose: [{port}]\n", - encoding="utf-8", - ) - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == ["harbor.invalid.reserved_main_port"] - assert str(port) in failure.findings[0].message - - -def test_adapt_does_not_forward_reserved_ports_exposed_by_the_image(tmp_path: Path) -> None: - make_harbor_task( - tmp_path, - "task-a", - dockerfile="FROM python:3.11-slim\nWORKDIR /workspace\nEXPOSE 3128 3129 8765 8080\n", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - assert _environment_config(context)["ports"] == [8080] - - -def test_adapt_accepts_explicit_shared_verifier_mode(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "shared") - (task / "task.toml").write_text( - '[verifier]\nenvironment_mode = "shared"\n', - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.verifier is None - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - assert row.runtime_config.compose.service_access is None - - -def test_adapt_builds_a_separate_verifier_with_its_own_placement( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "separate") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main: {}\n redis:\n image: redis:7-alpine\n expose: [6379]\n", - encoding="utf-8", - ) - (task / "tests" / "Dockerfile").write_text( - "FROM python:3.12-alpine\nCOPY . /tests\n", - encoding="utf-8", - ) - (task / "task.toml").write_text( - """ -artifacts = ["/tmp/agent.patch"] - -[environment] -cpus = 2 -memory_mb = 2048 -build_timeout_sec = 600.5 -gpus = 1 -gpu_types = ["H100", "A100"] -os = "windows" -tpu = {type = "v5", topology = "2x2"} - -[verifier] -environment_mode = "separate" -timeout_sec = 30 - -[verifier.environment] -cpus = 4 -memory_mb = 1024 -build_timeout_sec = 1200 -gpus = 1 -gpu_types = ["T4"] -workdir = "/judge" -network_mode = "allowlist" -allowed_hosts = ["verifier.example"] - -[verifier.environment.env] -NESTED_ONLY = "yes" -SHARED = "nested" - -[verifier.env] -SHARED = "phase" - -[[verifier.collect]] -service = "redis" -command = "redis-cli save" -timeout_sec = 10 -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.id == "run" - assert row.slug == "separate" - assert row.runtime_config is not None - assert row.runtime_config.resources == RuntimeResources( - cpu=2, - memory_mb=2048, - gpu=RuntimeGPU(type=["H100", "A100"]), - os="windows", - tpu=RuntimeTPU(type="v5", topology="2x2"), - ) - assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=601) - assert row.runtime_config.compose is not None - assert row.runtime_config.compose.service_access is True - assert row.verifier is not None - assert row.verifier.runtime_config is not None - assert row.verifier.runtime_config.compose is not None - assert row.verifier.runtime_config.compose.document == row.runtime_config.compose.document - assert row.verifier.runtime_config.compose.root == row.runtime_config.compose.root - assert row.verifier.runtime_config.resources == RuntimeResources( - cpu=4, - memory_mb=1024, - gpu=RuntimeGPU(type=["T4"]), - ) - assert row.verifier.runtime_config.limits == RuntimeLimits(startup_timeout_s=1200) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["verifier_root"] == "/verifier" - assert manifest["verifier_image"]["workdir"] == "/judge" - assert manifest["verifier"] == { - "user": None, - "network": {"enabled": True, "allowed_hosts": ["verifier.example"]}, - "env": {"NESTED_ONLY": "yes", "SHARED": "phase"}, - } - assert "tasks" not in manifest - assert row.args["task"] == { - "artifacts": [{"service": "main", "source": "/tmp/agent.patch"}], - "collect": [{"command": "redis-cli save", "service": "redis", "timeout_sec": 10.0}], - "description": "", - "id": "separate", - "separate_verifier": True, - "verifier_timeout": 30.0, - } - controller = ControllerConfig.model_validate(manifest) - assert controller.verifier.network.allowed_hosts == ["verifier.example"] - spec = TaskSpec.model_validate(row.args["task"]) - assert [hook.service for hook in spec.collect] == ["redis"] - assert [artifact.source for artifact in spec.artifacts] == ["/tmp/agent.patch"] - assert not (context / "compose-project" / "main" / "tasks").exists() - compose = json.loads((context / "compose.yaml").read_text("utf-8")) - assert compose["services"]["main"].get("volumes", []) == [] - assert compose["services"]["main"]["build"]["target"] == "verifier" - assert compose["services"]["main"]["build"]["additional_contexts"] == { - "hud-base": "service:hud-base", - "hud-verifier": "service:hud-verifier", - } - assert compose["services"]["hud-verifier"]["scale"] == 0 - assert compose["services"]["hud-verifier"]["image"].startswith("hud-harbor-verifier:") - - -def test_separate_verifier_image_task_builds_the_verifier_target( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "separate-image") - (task / "tests" / "Dockerfile").write_text( - "FROM python:3.12-alpine\nCOPY . /tests\n", - encoding="utf-8", - ) - (task / "task.toml").write_text( - """ -[environment] -build_timeout_sec = 300 -cpus = 4 -memory_mb = 14336 -gpus = 1 -gpu_types = ["H100"] - -[verifier] -environment_mode = "separate" - -[verifier.environment] -build_timeout_sec = 300 -cpus = 4 -memory_mb = 14336 -gpus = 1 -gpu_types = ["H100"] -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=300) - assert row.runtime_config.compose is not None - assert row.runtime_config.compose.service_access is None - assert row.verifier is not None - assert row.verifier.runtime_config is None - assert isinstance(row.runtime_config.compose.document, Path) - project = _assert_stock_compose_complete(row.runtime_config.compose.document) - assert set(project["services"]) == {"main", "hud-base", "hud-verifier"} - build = project["services"]["main"]["build"] - assert build["target"] == "verifier" - assert build["additional_contexts"] == { - "hud-base": "service:hud-base", - "hud-verifier": "service:hud-verifier", - } - assert project["services"]["hud-base"]["scale"] == 0 - assert project["services"]["hud-verifier"]["scale"] == 0 - - -def test_separate_verifier_groups_have_distinct_environment_names( - tmp_path: Path, -) -> None: - declaration = '[verifier]\nenvironment_mode = "separate"\n' - compose = "services:\n main: {}\n redis:\n image: redis:7-alpine\n expose: [6379]\n" - verifier = "FROM python:3.12-alpine\nCOPY . /tests\n" - for name in ("task-a", "task-b"): - task = make_harbor_task(tmp_path, name) - (task / "task.toml").write_text(declaration, encoding="utf-8") - (task / "environment" / "docker-compose.yaml").write_text(compose, encoding="utf-8") - (task / "tests" / "Dockerfile").write_text(verifier, encoding="utf-8") - - rows = list(_adapt(tmp_path)) - - assert len({row.env for row in rows}) == 2 - compose_paths = { - compose - for row in rows - if row.runtime_config is not None - and row.runtime_config.compose is not None - and isinstance((compose := row.runtime_config.compose.document), Path) - } - assert len(compose_paths) == 2 - assert all(path.is_file() for path in compose_paths) - task_files = [ - json.loads((path.parent / "tasks.json").read_text("utf-8")) for path in compose_paths - ] - assert {rows[0]["slug"] for rows in task_files} == {"task-a", "task-b"} - assert {rows[0]["args"]["task"]["id"] for rows in task_files} == {"task-a", "task-b"} - - -def test_multi_step_tasks_are_refused_directly(tmp_path: Path) -> None: - make_multi_step_task(tmp_path, "multi") - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == ["harbor.unsupported.multi_step"] - - -@pytest.mark.parametrize( - "artifact", - [ - '"/"', - '"//"', - '"/workspace/../secret"', - '{ source = "/output", destination = "/tmp/out" }', - '{ source = "/output", destination = "../out" }', - '{ source = "/output", destination = "a\\\\b" }', - '{ source = "/output", destination = "manifest.json" }', - ], -) -def test_artifact_paths_stay_beneath_their_roots(tmp_path: Path, artifact: str) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - f"artifacts = [{artifact}]\n", - encoding="utf-8", - ) - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == ["harbor.invalid.task_config"] - - -def test_artifacts_keep_harbor_destination_and_exclude(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - """\ -[[artifacts]] -source = "/app/outputs" -destination = "results/outputs" -exclude = ["*.tmp", "cache"] -""", - encoding="utf-8", - ) - - taskset = _adapt(tmp_path) - - assert taskset["task-a"].args["task"]["artifacts"] == [ - { - "service": "main", - "source": "/app/outputs", - "destination": "results/outputs", - "exclude": ["*.tmp", "cache"], - } - ] - - -def test_agent_timeout_becomes_per_task_agent_policy( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text("[agent]\ntimeout_sec = 60\n", encoding="utf-8") - - taskset = _adapt(tmp_path) - - (row,) = list(taskset) - assert row.agent_config == {"timeout_seconds": 60.0} - - -def test_task_symlinks_are_copied_without_reading_host_files( - tmp_path: Path, -) -> None: - outside = tmp_path / "outside.txt" - outside.write_text("host secret", encoding="utf-8") - task = make_harbor_task(tmp_path / "dataset", "task-a") - (task / "tests" / "link").symlink_to(outside) - - _adapt(task.parent) - - (context,) = (task.parent / ".hud-adapt").iterdir() - copied = context / "compose-project" / "tests" / "task-a" / "link" - assert copied.is_symlink() - assert os.readlink(copied) == str(outside) - - -def test_adapt_hashes_links_not_their_targets( - tmp_path: Path, -) -> None: - outside = tmp_path / "outside.txt" - outside.write_text("first", encoding="utf-8") - task = make_harbor_task(tmp_path / "dataset", "task-a") - (task / "environment" / "link").symlink_to(outside) - - (before,) = list(_adapt(task.parent)) - outside.write_text("changed", encoding="utf-8") - (after,) = list(_adapt(task.parent)) - - assert before.runtime_config == after.runtime_config - - -def test_authored_runtime_assets_are_valid_source() -> None: - integration = Path(__file__).parents[1] - compile((integration / "env.py").read_text("utf-8"), "env.py", "exec") - installer = (integration / "install.sh").read_text("utf-8") - assert "python_version=3.12" in installer - assert "sys.version_info[:2] < (3, 13)" in installer - assert 'uv python install "$python_version"' in installer - assert 'python="$root/bin/python$python_version"' in installer - assert 'uv venv "$root/venv" --python "$python"' in installer - assert "dnf install -y bubblewrap" in installer - result = subprocess.run( - ["sh", "-n", integration / "install.sh"], - check=False, - capture_output=True, - ) - assert result.returncode == 0, result.stderr.decode() - - -def test_public_surface_exposes_results_and_the_two_real_operations() -> None: - assert harbor.__all__ == [ - "AdaptFailure", - "AdaptFinding", - "AdaptResult", - "adapt", - "export", - ] - - -def test_portless_sidecar_ports_are_resolved_from_its_built_image(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main: {}\n redis:\n image: redis:7-alpine\n", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - context = row.runtime_config.compose.root - assert isinstance(context, Path) - manifest = _environment_config(context) - assert ControllerConfig.model_validate(manifest).peers == [ - Peer("redis", 6379, target=("redis", 6379)) - ] - assert "peer_image_configs" not in manifest - - -def test_completed_compose_dependencies_are_not_routed_as_peers(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n" - " main:\n" - " depends_on:\n" - " seed:\n" - " condition: service_completed_successfully\n" - " seed:\n" - " build: ./seed\n", - encoding="utf-8", - ) - (task / "environment" / "seed").mkdir() - (task / "environment" / "seed" / "Dockerfile").write_text( - 'FROM busybox:1.37\nCMD ["true"]\n', - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - context = row.runtime_config.compose.root - assert isinstance(context, Path) - manifest = _environment_config(context) - assert manifest["peers"] == [] - assert "peer_image_configs" not in manifest diff --git a/hud/integrations/harbor/tests/test_harbor.py b/hud/integrations/harbor/tests/test_harbor.py deleted file mode 100644 index 229d282d8..000000000 --- a/hud/integrations/harbor/tests/test_harbor.py +++ /dev/null @@ -1,267 +0,0 @@ -"""``hud.integrations.harbor`` task adaptation and export.""" - -from __future__ import annotations - -import asyncio -import json -import textwrap -from typing import TYPE_CHECKING, Any - -import pytest - -from hud.integrations.harbor import export - -if TYPE_CHECKING: - from pathlib import Path - -# ─── export: HUD tasks -> Harbor task folders ─────────────────────────── - -_ENV_PY = """\ -from hud import Environment - -env = Environment("demo") - - -@env.template() -async def solve(n: int = 1): - yield f"solve {n}" - yield 1.0 - - -tasks = [solve(n=2)] -""" - -_DOCKERFILE = """\ -FROM python:3.11-slim -RUN pip install hud -COPY env.py ./ -CMD ["hud", "serve", "env:env"] -""" - - -def _write_env( - tmp_path: Path, *, dockerfile: bool = True, args: dict[str, Any] | None = None -) -> Path: - src = tmp_path / "env.py" - body = textwrap.dedent(_ENV_PY) - if args is not None: - body = body.replace( - "async def solve(n: int = 1):", "async def solve(n: int = 1, prompt: str = ''):" - ).replace("tasks = [solve(n=2)]", f"tasks = [solve(**{args!r})]") - src.write_text(body, encoding="utf-8") - if dockerfile: - (tmp_path / "Dockerfile").write_text(_DOCKERFILE, encoding="utf-8") - return src - - -async def test_export_writes_task_folder(tmp_path: Path) -> None: - src = _write_env(tmp_path) - out = tmp_path / "out" - - created = await export(str(src), out) - - assert len(created) == 1 - task_dir = created[0] - assert (task_dir / "task.toml").exists() - assert (task_dir / "instruction.md").exists() - assert (task_dir / "tests" / "test.sh").exists() - assert (task_dir / "environment" / "Dockerfile").exists() - assert (task_dir / "environment" / "hud_entrypoint.sh").exists() - - -async def test_export_loads_python_source_on_main_thread(tmp_path: Path) -> None: - src = _write_env(tmp_path) - source = src.read_text(encoding="utf-8") - src.write_text( - "import threading\nassert threading.current_thread() is threading.main_thread()\n" + source, - encoding="utf-8", - ) - - created = await export(str(src), tmp_path / "out") - - assert len(created) == 1 - - -async def test_requires_dockerfile(tmp_path: Path) -> None: - _write_env(tmp_path, dockerfile=False) - with pytest.raises(FileNotFoundError, match="Dockerfile"): - await export(str(tmp_path / "env.py"), tmp_path / "out") - - -async def test_instruction_has_prompt_and_answer_convention(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - instruction = (created[0] / "instruction.md").read_text(encoding="utf-8") - assert instruction.startswith("solve 2") # the materialized prompt - assert "/workspace/answer.txt" in instruction # the answer convention - - -async def test_task_toml_is_harbor_native(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - toml = (created[0] / "task.toml").read_text(encoding="utf-8") - assert 'version = "1.0"' in toml - assert "name = " in toml - assert "[verifier]" in toml and "[agent]" in toml - assert "timeout_sec" in toml - # HUD task/args preserved as metadata for the record. - assert "hud_task" in toml and "hud_args" in toml - - -async def test_scripts_drive_hud_task_lifecycle(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - boot = (created[0] / "environment" / "hud_entrypoint.sh").read_text(encoding="utf-8") - test_sh = (created[0] / "tests" / "test.sh").read_text(encoding="utf-8") - - # Boot serves the channel, parks the run via setup, then hands off. - assert "hud serve env.py:demo" in boot - assert "hud task start solve" in boot - assert "python3 -c" not in boot - assert 'exec "$@"' in boot - # Verifier grades the parked run and writes the Harbor reward. - assert "hud task grade solve" in test_sh - assert "--answer-file" in test_sh - assert "/logs/verifier/reward.txt" in test_sh - - -async def test_scripts_survive_arguments_that_need_quoting(tmp_path: Path) -> None: - # Task ids and args are data: an apostrophe must not end the shell string. - _write_env(tmp_path, args={"prompt": "what's next?"}) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - for script in ("environment/hud_entrypoint.sh", "tests/test.sh"): - text = (created[0] / script).read_text(encoding="utf-8") - proc = await asyncio.create_subprocess_exec( - "sh", "-n", "-", stdin=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - _out, err = await proc.communicate(text.encode()) - assert proc.returncode == 0, f"{script} is not valid shell: {err.decode()}" - - -async def test_environment_context_keeps_its_own_json_files(tmp_path: Path) -> None: - # Only the copied taskset source is dropped; a build context may need its - # own JSON (package.json and friends). - _write_env(tmp_path) - (tmp_path / "package.json").write_text('{"name": "app"}', encoding="utf-8") - - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - assert (created[0] / "environment" / "package.json").is_file() - - -async def test_dockerfile_appends_the_harbor_runtime(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - dockerfile = (created[0] / "environment" / "Dockerfile").read_text(encoding="utf-8") - assert "COPY --chmod=0755 hud_entrypoint.sh /hud_entrypoint.sh" in dockerfile - assert 'ENTRYPOINT ["/hud_entrypoint.sh"]' in dockerfile - assert dockerfile.rindex('CMD ["sh", "-c", "sleep infinity"]') > dockerfile.index( - 'CMD ["hud", "serve", "env:env"]' - ) - # The env build context is copied so the image can be rebuilt under Harbor. - assert (created[0] / "environment" / "env.py").exists() - - -async def test_custom_answer_file(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out", answer_file="/app/out.txt") - assert "/app/out.txt" in (created[0] / "instruction.md").read_text(encoding="utf-8") - assert "/app/out.txt" in (created[0] / "tests" / "test.sh").read_text(encoding="utf-8") - - -async def test_export_fails_loudly_when_setup_cannot_run(tmp_path: Path) -> None: - # An unstarted task must not reach the verifier as a score of 0: broken - # infrastructure and a failed agent are different outcomes. - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - boot = (created[0] / "environment" / "hud_entrypoint.sh").read_text(encoding="utf-8") - - assert "|| true" not in boot - assert "exit 1" in boot - - -async def test_export_survives_a_restrictive_dockerignore(tmp_path: Path) -> None: - # Same class as the adaptation layer's: re-including a generated file - # needs its own rule, or COPY fails and no exported image builds. - _write_env(tmp_path) - (tmp_path / ".dockerignore").write_text("*\n", encoding="utf-8") - - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - ignore = (created[0] / "environment" / ".dockerignore").read_text(encoding="utf-8") - assert "!hud_entrypoint.sh" in ignore - - -async def test_export_overrides_multiline_startup_directives(tmp_path: Path) -> None: - _write_env(tmp_path) - (tmp_path / "Dockerfile").write_text( - 'FROM python:3.12-slim\nCMD ["python", \\\n "-m", \\\n "app"]\n', encoding="utf-8" - ) - - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - dockerfile = (created[0] / "environment" / "Dockerfile").read_text(encoding="utf-8") - assert dockerfile.rindex('CMD ["sh", "-c", "sleep infinity"]') > dockerfile.index('"app"]') - - -async def test_export_preserves_a_non_root_runtime_user(tmp_path: Path) -> None: - _write_env(tmp_path) - (tmp_path / "Dockerfile").write_text( - "FROM python:3.12-slim\nRUN useradd -m app\nUSER app\n", encoding="utf-8" - ) - - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - dockerfile = (created[0] / "environment" / "Dockerfile").read_text(encoding="utf-8") - assert dockerfile.count("USER ") == 1 - assert "COPY --chmod=0755" in dockerfile - - -async def test_export_serves_the_resolved_environment(tmp_path: Path) -> None: - # The environment is bound to ``bench`` in tasks.py, while its public name - # remains ``demo``. The generated target uses the canonical loader syntax. - (tmp_path / "tasks.py").write_text( - textwrap.dedent(_ENV_PY) - .replace("env = Environment", "bench = Environment") - .replace("@env.template", "@bench.template"), - encoding="utf-8", - ) - (tmp_path / "Dockerfile").write_text(_DOCKERFILE, encoding="utf-8") - - created = await export(str(tmp_path / "tasks.py"), tmp_path / "out") - - boot = (created[0] / "environment" / "hud_entrypoint.sh").read_text(encoding="utf-8") - assert "hud serve tasks.py:demo" in boot - - -async def test_export_slugs_stay_inside_the_output_directory(tmp_path: Path) -> None: - _write_env(tmp_path) - taskset = tmp_path / "tasks.json" - taskset.write_text( - json.dumps([{"env": "demo", "id": "solve", "args": {"n": 2}, "slug": "suite/fix"}]), - encoding="utf-8", - ) - - created = await export(str(taskset), tmp_path / "out") - - assert created[0].name == "suite-fix" - taskset.write_text( - json.dumps([{"env": "demo", "id": "solve", "args": {"n": 2}, "slug": ".."}]), - encoding="utf-8", - ) - with pytest.raises(ValueError, match="usable directory name"): - await export(str(taskset), tmp_path / "invalid") - - -async def test_export_never_scores_a_broken_grader_as_zero(tmp_path: Path) -> None: - # Startup and grading are the same rule: infrastructure that cannot run is - # an error, not an agent that failed the task. - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - test_sh = (created[0] / "tests" / "test.sh").read_text(encoding="utf-8") - - assert "echo 0 > /logs/verifier/reward.txt" not in test_sh - assert "exit 1" in test_sh diff --git a/hud/integrations/harbor/tests/test_importing.py b/hud/integrations/harbor/tests/test_importing.py deleted file mode 100644 index dc5a66338..000000000 --- a/hud/integrations/harbor/tests/test_importing.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Tests for packing Harbor folders into an import bundle.""" - -from __future__ import annotations - -import os -import zipfile -from typing import TYPE_CHECKING - -import pytest - -from hud.integrations.harbor.importing import BundleError, pack_bundle - -if TYPE_CHECKING: - from pathlib import Path - - -def _task(root: Path, name: str = "hello") -> Path: - task = root / name - (task / "environment").mkdir(parents=True) - (task / "task.toml").write_text("version = '1.0'\n") - (task / "environment" / "Dockerfile").write_text("FROM ubuntu\n") - return task - - -def _members(bundle: Path) -> dict[str, bytes]: - with zipfile.ZipFile(bundle) as archive: - return {name: archive.read(name) for name in archive.namelist()} - - -def test_each_folder_is_stored_under_its_own_name(tmp_path: Path) -> None: - dataset = tmp_path / "dataset" - _task(dataset, "one") - _task(dataset, "two") - bundle = tmp_path / "bundle.zip" - - assert pack_bundle([dataset, _task(tmp_path / "loose")], bundle) == 6 - - assert sorted(_members(bundle)) == [ - "dataset/one/environment/Dockerfile", - "dataset/one/task.toml", - "dataset/two/environment/Dockerfile", - "dataset/two/task.toml", - "hello/environment/Dockerfile", - "hello/task.toml", - ] - - -def test_symlinked_files_are_stored_by_content_and_symlinked_folders_are_not_followed( - tmp_path: Path, -) -> None: - task = _task(tmp_path / "src") - shared = tmp_path / "shared" - shared.mkdir() - (shared / "data.txt").write_text("shared") - os.symlink(shared / "data.txt", task / "environment" / "data.txt") - os.symlink(shared, task / "environment" / "linked") - os.symlink(tmp_path / "missing", task / "dangling") - bundle = tmp_path / "bundle.zip" - - pack_bundle([task], bundle) - - members = _members(bundle) - assert members["hello/environment/data.txt"] == b"shared" - assert not any("linked" in name or "dangling" in name for name in members) - - -@pytest.mark.skipif(os.geteuid() == 0, reason="root reads any folder") -def test_a_folder_that_cannot_be_read_fails_packing(tmp_path: Path) -> None: - task = _task(tmp_path / "src") - locked = task / "environment" - locked.chmod(0) - try: - with pytest.raises(PermissionError): - pack_bundle([task], tmp_path / "bundle.zip") - finally: - locked.chmod(0o755) - - -def test_two_folders_with_one_name_are_refused(tmp_path: Path) -> None: - first = _task(tmp_path / "a") - second = _task(tmp_path / "b") - - with pytest.raises(BundleError, match="both named 'hello'"): - pack_bundle([first, second], tmp_path / "bundle.zip") - - -def test_folders_without_harbor_files_are_refused(tmp_path: Path) -> None: - folder = tmp_path / "notes" - folder.mkdir() - (folder / "README.md").write_text("hi") - - with pytest.raises(BundleError, match="no Harbor task folder"): - pack_bundle([folder], tmp_path / "bundle.zip") - - -def test_a_path_that_is_not_a_folder_is_refused(tmp_path: Path) -> None: - file = tmp_path / "task.toml" - file.write_text("") - - with pytest.raises(BundleError, match="is not a folder"): - pack_bundle([file], tmp_path / "bundle.zip") diff --git a/hud/integrations/harbor/tests/test_integration.py b/hud/integrations/harbor/tests/test_integration.py deleted file mode 100644 index 48ec0daf7..000000000 --- a/hud/integrations/harbor/tests/test_integration.py +++ /dev/null @@ -1,935 +0,0 @@ -"""Docker coverage for Harbor phase behavior. - -The tests build adapted images and require network access:: - - uv run pytest -m integration -""" - -from __future__ import annotations - -import asyncio -import shutil -import subprocess -import sys -from pathlib import Path -from typing import TYPE_CHECKING, cast - -import pytest - -from hud.agents.base import Agent -from hud.eval import DockerRuntime, Shared, Taskset -from hud.eval.runtime import DockerBindMount -from hud.integrations import harbor - -from .conftest import make_harbor_task - -if TYPE_CHECKING: - from hud.capabilities import MCPClient, SSHClient - from hud.eval.run import Run - -TASKS = Path(__file__).parent / "tasks" -REPO = Path(__file__).resolve().parents[4] - -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif(sys.platform == "win32", reason="adapted images are Linux containers"), -] - - -def _adapt(path: Path, *, hud_requirement: str = "hud") -> Taskset: - result = harbor.adapt(path, hud_requirement=hud_requirement) - assert result.failures == () - return result.taskset - - -@pytest.fixture(scope="module", autouse=True) -def docker_daemon() -> None: - if shutil.which("docker") is None: - pytest.skip("needs a running Docker daemon") - try: - probe = subprocess.run(["docker", "info"], capture_output=True, check=False, timeout=20) - except subprocess.TimeoutExpired: - pytest.skip("Docker daemon did not answer") - if probe.returncode != 0: - pytest.skip("needs a running Docker daemon") - - -class Oracle(Agent): - """Run each fixture's reference solution.""" - - def __init__(self, solutions: dict[str, str]) -> None: - super().__init__() - self.solutions = solutions - - async def __call__(self, run: Run) -> None: - setup = next( - step.task_call - for step in run.trace.steps - if step.task_call is not None and step.task_call.phase == "setup" - ) - assert isinstance(setup.arguments, dict) - task = setup.arguments.get("task") - assert isinstance(task, dict) - task_id = task.get("id") - assert isinstance(task_id, str) - ssh = cast("SSHClient", await run.client.open("ssh/2")) - if task_id == "agent-lifecycle": - for index in range(40): - result = await ssh.conn.run(f"printf '%s' {index}", check=False) - assert result.exit_status == 0, f"command session {index} failed: {result.stderr!r}" - assert result.stdout == str(index), f"command session {index} lost output" - await ssh.conn.run("cat > session-input", input="written", check=True) - written = await ssh.conn.run("cat session-input", check=True) - assert written.stdout == "written" - if task_id == "hello-mcp": - mcp = cast("MCPClient", await run.client.open("mcp-server")) - assert {tool.name for tool in await mcp.list_tools()} == {"get_secret"} - assert run.client.manifest is not None - capability = next( - cap for cap in run.client.manifest.bindings if cap.name == "mcp-server" - ) - script = f""" -import json -from pathlib import Path -from urllib.request import Request, urlopen - -url = {capability.url!r} -session_id = None - -def post(payload): - global session_id - headers = {{ - "Accept": "application/json, text/event-stream", - "Content-Type": "application/json", - }} - if session_id is not None: - headers["Mcp-Session-Id"] = session_id - request = Request(url, data=json.dumps(payload).encode(), headers=headers) - with urlopen(request) as response: - body = response.read().decode() - returned_session = response.headers.get("Mcp-Session-Id") - if returned_session: - session_id = returned_session - data = [line[6:] for line in body.splitlines() if line.startswith("data: ")] - return json.loads(data[-1] if data else body) if body else None - -post({{ - "jsonrpc": "2.0", - "id": 1, - "method": "initialize", - "params": {{ - "protocolVersion": "2025-06-18", - "capabilities": {{}}, - "clientInfo": {{"name": "hud-test", "version": "1"}}, - }}, -}}) -post({{"jsonrpc": "2.0", "method": "notifications/initialized"}}) -result = post({{ - "jsonrpc": "2.0", - "id": 2, - "method": "tools/call", - "params": {{"name": "get_secret", "arguments": {{}}}}, -}}) -Path("/app/secret.txt").write_text(result["result"]["content"][0]["text"]) -""" - result = await ssh.conn.run("python3 -", input=script, check=True) - if result.exit_status is None: - raise RuntimeError("workspace MCP client closed without an exit status") - run.trace.content = "called get_secret from inside the workspace" - return - result = await ssh.conn.run(self.solutions[task_id], check=False) - if result.exit_status is None: - run.trace.content = ( - "solution SSH channel closed without an exit status:\n" - f"{result.stdout}\n{result.stderr}" - ) - raise RuntimeError("solution SSH channel closed without an exit status") - if result.exit_status != 0: - run.trace.content = f"solution failed:\n{result.stdout}\n{result.stderr}" - raise RuntimeError(f"solution exited with status {result.exit_status}") - run.trace.content = "solution completed" - - -async def _grade_every_task(dataset: Path, wheel: Path) -> dict[str, Run]: - def load_solutions() -> dict[str, str]: - return { - task.name: (task / "solution" / "solve.sh").read_text("utf-8") - for task in sorted(dataset.iterdir()) - if (task / "task.toml").is_file() - } - - solutions = await asyncio.to_thread(load_solutions) - taskset = _adapt(dataset, hud_requirement=str(wheel)) - job = await taskset.run( - Oracle(solutions), - runtime=DockerRuntime(), - max_concurrent=1, - ) - return {cast("str", run.slug): run for run in job.runs} - - -@pytest.fixture(scope="module") -def wheel(tmp_path_factory: pytest.TempPathFactory) -> Path: - workdir = tmp_path_factory.mktemp("harbor") - wheels = workdir / "wheels" - subprocess.run( - ["uv", "build", "--wheel", "--out-dir", str(wheels)], - cwd=REPO, - check=True, - capture_output=True, - ) - return next(iter(wheels.glob("*.whl"))) - - -@pytest.fixture(scope="module") -def graded(tmp_path_factory: pytest.TempPathFactory, wheel: Path) -> dict[str, Run]: - """Adapt and grade the inline-verifier fixtures once for this module.""" - dataset = tmp_path_factory.mktemp("harbor-inline") / "harbor-harness" - shutil.copytree(TASKS, dataset, ignore=shutil.ignore_patterns("sidecar-reachability")) - return asyncio.run(_grade_every_task(dataset, wheel)) - - -@pytest.fixture(scope="module") -def separately_graded(tmp_path_factory: pytest.TempPathFactory, wheel: Path) -> dict[str, Run]: - """Adapt and grade the separate-verifier fixture.""" - dataset = tmp_path_factory.mktemp("harbor-separate") / "harbor-harness" - dataset.mkdir() - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - declaration = task / "task.toml" - declaration.write_text( - declaration.read_text("utf-8") + "\n[verifier.environment]\ncpus = 1\n", - encoding="utf-8", - ) - return asyncio.run(_grade_every_task(dataset, wheel)) - - -@pytest.mark.parametrize( - "task_id", - [ - "phase-boundary", - "agent-lifecycle", - "verifier-lifecycle", - "hello-mcp", - ], -) -def test_harbor_phase_behavior(graded: dict[str, Run], task_id: str) -> None: - run = graded[task_id] - evaluation = run.evaluation - detail = evaluation.get("content") or "" - info = evaluation.get("info") or {} - detail = "\n".join( - filter( - None, - ( - run.trace.content, - run.trace.error, - detail, - info.get("stdout"), - info.get("stderr"), - ), - ) - ) - assert run.reward == 1.0, f"{task_id} scored {run.reward}; the verifier reported:\n{detail}" - - -def test_separate_verifier_phase_behavior(separately_graded: dict[str, Run]) -> None: - test_harbor_phase_behavior(separately_graded, "sidecar-reachability") - - -def test_sidecar_ports_are_discovered_from_the_built_image( - tmp_path_factory: pytest.TempPathFactory, - wheel: Path, -) -> None: - dataset = tmp_path_factory.mktemp("harbor-sidecar-ports") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - compose = task / "environment/docker-compose.yaml" - authored = compose.read_text("utf-8") - inferred = authored.replace( - " image: ${SIDECAR_IMAGE:-python:3.11-alpine}\n", - " build: ./workspace\n", - ).replace(' expose: ["5678", "5679"]\n', "") - assert inferred != authored - compose.write_text(inferred, encoding="utf-8") - sidecar = task / "environment/workspace" - sidecar.mkdir() - (sidecar / "Dockerfile").write_text( - "FROM python:3.11-alpine\nEXPOSE 5678 5679\n", - encoding="utf-8", - ) - - runs = asyncio.run(_grade_every_task(dataset, wheel)) - - test_harbor_phase_behavior(runs, "sidecar-reachability") - - -def test_separate_verifier_artifact_materialization_is_repeatable( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-verifier-artifacts") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - - async def grade_twice() -> list[Run]: - taskset = _adapt(dataset, hud_requirement=str(wheel)) - job = await taskset.run( - Oracle({"sidecar-reachability": (task / "solution/solve.sh").read_text("utf-8")}), - runtime=Shared(DockerRuntime(), width=1), - group=2, - max_concurrent=1, - ) - return job.runs - - runs = asyncio.run(grade_twice()) - - assert len(runs) == 2 - assert all(run.reward == 1.0 for run in runs) - - -def test_separate_verifier_artifacts_are_accessible_to_child_user( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-child-user") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - dockerfile = task / "tests/Dockerfile" - dockerfile.write_text( - dockerfile.read_text("utf-8").replace("USER verifier\n", "USER root\n"), - encoding="utf-8", - ) - (task / "tests/test.sh").write_text( - "#!/bin/sh\n" - "set -eu\n" - "mkdir -p /logs/verifier\n" - "if su verifier -s /bin/sh -c 'test \"$(cat /root/agent-output.txt)\" = private'; then\n" - " echo 1 > /logs/verifier/reward.txt\n" - "else\n" - " echo 0 > /logs/verifier/reward.txt\n" - "fi\n", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 1.0 - - -def test_fedora_environment_bootstrap( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-fedora") / "harbor-harness" - task = dataset / "fedora-bootstrap" - (task / "environment").mkdir(parents=True) - (task / "tests").mkdir() - (task / "solution").mkdir() - (task / "task.toml").write_text( - '[task]\nname = "fedora-bootstrap"\n\n[verifier]\ntimeout_sec = 30\n', - encoding="utf-8", - ) - (task / "instruction.md").write_text("Verify Fedora bootstrap.\n", encoding="utf-8") - (task / "environment/Dockerfile").write_text( - "FROM fedora:42\nWORKDIR /app\n", - encoding="utf-8", - ) - (task / "tests/test.sh").write_text( - "#!/bin/bash\necho 1 > /logs/verifier/reward.txt\n", - encoding="utf-8", - ) - (task / "solution/solve.sh").write_text("true\n", encoding="utf-8") - - run = asyncio.run(_grade_every_task(dataset, wheel))["fedora-bootstrap"] - - assert run.reward == 1.0 - - -def test_separate_verifier_rejects_artifact_symlinks( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-symlink") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - solution = task / "solution" / "solve.sh" - solution.write_text( - solution.read_text("utf-8") - + "\nrm -f /app/main.html\n" - + "ln -s /verifier/tests /app/main.html\n", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 0.0 - assert "artifact /app/main.html is a symbolic link" in (run.trace.error or "") - - -def test_separate_verifier_rejects_sidecar_symlink_artifact_roots( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-sidecar-symlink") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "task.toml").write_text( - """\ -artifacts = [{ source = "/link", service = "workspace", exclude = ["main.html"] }] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 - -[[verifier.collect]] -service = "workspace" -command = "ln -sfn /app /link" -timeout_sec = 10 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text("true\n", encoding="utf-8") - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 0.0 - assert "artifact /link contains a symbolic link" in (run.trace.error or "") - - -def test_separate_verifier_sees_directory_artifacts_without_excluded_entries( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-artifact-exclude") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "task.toml").write_text( - """\ -artifacts = [ - { source = "/app/outputs", destination = "results", exclude = ["*.tmp", "cache"] }, -] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text( - """\ -#!/bin/sh -set -eu -mkdir -p /app/outputs/cache/nested /app/outputs/logs -echo keep > /app/outputs/keep.txt -echo junk > /app/outputs/junk.tmp -echo junk > /app/outputs/logs/nested.tmp -echo junk > /app/outputs/cache/nested/blob -ln -s /etc/passwd /app/outputs/cache/link -ln -s missing /app/outputs/logs/dangling.tmp -""", - encoding="utf-8", - ) - (task / "tests" / "test.sh").write_text( - """\ -#!/bin/sh -set -u -mkdir -p /logs/verifier -if [ "$(cat /app/outputs/keep.txt 2>/dev/null)" = "keep" ] \\ - && [ -d /app/outputs/logs ] \\ - && [ ! -e /app/outputs/junk.tmp ] \\ - && [ ! -e /app/outputs/logs/nested.tmp ] \\ - && [ ! -L /app/outputs/logs/dangling.tmp ] \\ - && [ ! -e /app/outputs/cache ]; then - echo 1 > /logs/verifier/reward.txt -else - echo "excluded artifact entries leaked into the verifier" >&2 - echo 0 > /logs/verifier/reward.txt -fi -""", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 1.0, run.trace.error - - -def test_separate_verifier_collects_artifacts_written_to_authored_volumes( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-volume-artifacts") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - volumes: - - outputs:/app/outputs -volumes: - outputs: {} -""", - encoding="utf-8", - ) - (task / "task.toml").write_text( - """\ -artifacts = [{ source = "/app/outputs/result.txt", service = "main" }] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text( - "#!/bin/sh\nset -eu\necho from-volume > /app/outputs/result.txt\n", - encoding="utf-8", - ) - (task / "tests" / "test.sh").write_text( - """\ -#!/bin/sh -mkdir -p /logs/verifier -if [ "$(cat /app/outputs/result.txt 2>/dev/null)" = "from-volume" ]; then - echo 1 > /logs/verifier/reward.txt -else - echo 0 > /logs/verifier/reward.txt -fi -""", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 1.0, run.trace.error - - -def test_separate_verifier_rejects_artifacts_beneath_symlinks( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-ancestor-symlink") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - config = task / "task.toml" - config.write_text( - """\ -artifacts = [{ source = "/app/linked/tests", service = "main" }] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - solution = task / "solution" / "solve.sh" - solution.write_text( - "#!/bin/sh\nset -eu\nln -s /verifier /app/linked\n", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 0.0 - assert "artifact /app/linked/tests has a symbolic link in its path" in (run.trace.error or "") - - -def test_separate_verifier_restores_image_paths_between_shared_rollouts( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-verifier-restore") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "task.toml").write_text( - """\ -artifacts = [{ source = "/tests/test.sh", service = "main" }] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text("true\n", encoding="utf-8") - - async def grade_twice() -> list[Run]: - taskset = _adapt(dataset, hud_requirement=str(wheel)) - job = await taskset.run( - Oracle({"sidecar-reachability": "true"}), - runtime=Shared(DockerRuntime(), width=1), - group=2, - max_concurrent=1, - ) - return job.runs - - runs = asyncio.run(grade_twice()) - - assert len(runs) == 2 - for run in runs: - assert any( - step.task_call is not None - and step.task_call.phase == "evaluate" - and step.task_call.name == "verify" - for step in run.trace.steps - ) - - -def test_separate_verifier_honors_the_test_script_shebang( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-verifier-shebang") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "task.toml").write_text( - """\ -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text("true\n", encoding="utf-8") - (task / "tests" / "test.sh").write_text( - """\ -#!/usr/bin/env python3 -from pathlib import Path - -Path("/logs/verifier/reward.txt").write_text("1") -""", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 1.0 - - -def test_env_templates_resolve_from_runtime_env_vars( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - """``${VAR}``/``${VAR:-default}`` env values resolve exactly like Harbor's - host-side resolution, sourced from the runtime's ``env_vars``.""" - dataset = tmp_path_factory.mktemp("harbor-env-templates") / "harbor-harness" - task = make_harbor_task( - dataset, - "env-templates", - task_toml="""\ -[metadata] -category = "systems" - -[environment.env] -GREETING = "${HARBOR_GREETING:-hello}" -JUDGE_KEY = "${HARBOR_JUDGE_KEY}" -EMBEDDED = "Bearer ${HARBOR_JUDGE_KEY}" - -[verifier] -timeout_sec = 120 - -[verifier.env] -VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" -EMPTY_DEFAULT = "${HARBOR_UNSET:-}" -""", - ) - (task / "tests" / "test.sh").write_text( - """\ -#!/bin/bash -fail() { echo "unexpected $1"; echo "0.0" > /logs/verifier/reward.txt; exit 0; } -[ "$GREETING" = "hello" ] || fail "GREETING=$GREETING" -[ "$JUDGE_KEY" = "judge-secret" ] || fail "JUDGE_KEY=$JUDGE_KEY" -[ "$EMBEDDED" = 'Bearer ${HARBOR_JUDGE_KEY}' ] || fail "EMBEDDED=$EMBEDDED" -[ "$VERIFIER_KEY" = "judge-secret" ] || fail "VERIFIER_KEY=$VERIFIER_KEY" -[ "${EMPTY_DEFAULT-unset}" = "" ] || fail "EMPTY_DEFAULT=${EMPTY_DEFAULT-unset}" -echo "1.0" > /logs/verifier/reward.txt -""", - encoding="utf-8", - ) - solution = '[ "$JUDGE_KEY" = "judge-secret" ] && [ "$GREETING" = "hello" ]' - - async def grade() -> Run: - taskset = harbor.adapt(dataset, hud_requirement=str(wheel)).taskset - job = await taskset.run( - Oracle({"env-templates": solution}), - runtime=DockerRuntime( - env_vars={"HARBOR_GREETING": "", "HARBOR_JUDGE_KEY": "judge-secret"} - ), - max_concurrent=1, - ) - (run,) = job.runs - return run - - run = asyncio.run(grade()) - - evaluation = run.evaluation - info = evaluation.get("info") or {} - detail = "\n".join( - filter( - None, - ( - run.trace.content, - run.trace.error, - evaluation.get("content") or "", - info.get("stdout"), - info.get("stderr"), - ), - ) - ) - assert run.reward == 1.0, f"env-templates scored {run.reward}; the verifier reported:\n{detail}" - - -def test_missing_env_template_aborts_startup( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - """A required ``${VAR}`` with no default and no runtime value must abort - environment startup with an error naming the variable, like Harbor.""" - dataset = tmp_path_factory.mktemp("harbor-env-missing") / "harbor-harness" - make_harbor_task( - dataset, - "env-missing", - task_toml="""\ -[metadata] -category = "systems" - -[environment.env] -API_KEY = "${HARBOR_MISSING_KEY}" - -[verifier] -timeout_sec = 120 -""", - ) - - taskset = harbor.adapt(dataset, hud_requirement=str(wheel)).taskset - task = next(iter(taskset)) - assert task.runtime_config is not None - assert task.runtime_config.compose is not None - compose = task.runtime_config.compose.document - assert isinstance(compose, Path) - # Providers yield as soon as the published port exists, before the serve - # process proves itself, so an early abort is only observable from the - # adapted artifact: run its main service in the foreground. - subprocess.run( - ["docker", "compose", "--file", str(compose), "build"], - cwd=compose.parent, - check=True, - capture_output=True, - timeout=600, - ) - command = ["docker", "compose", "--file", str(compose), "run", "--rm", "main"] - try: - serve = subprocess.run(command, capture_output=True, text=True, timeout=60) - except subprocess.TimeoutExpired: - pytest.fail("main service kept serving despite an unresolvable env template") - finally: - subprocess.run( - ["docker", "compose", "--file", str(compose), "down", "--volumes", "--remove-orphans"], - capture_output=True, - check=False, - ) - assert serve.returncode != 0 - assert "HARBOR_MISSING_KEY" in serve.stdout + serve.stderr - - -@pytest.mark.parametrize("mode", ["shared", "separate"]) -@pytest.mark.parametrize("credential", ["missing", "provided", "default"]) -def test_verifier_credentials_are_required_only_for_grading( - tmp_path: Path, wheel: Path, mode: str, credential: str -) -> None: - dataset = tmp_path / "harbor-harness" - template = ( - "${HARBOR_VERIFIER_KEY:-judge-value}" - if credential == "default" - else "${HARBOR_VERIFIER_KEY}" - ) - task = make_harbor_task( - dataset, - "verifier-credentials", - task_toml=f"""[verifier] -environment_mode = "{mode}" -timeout_sec = 30 - -[verifier.env] -JUDGE_KEY = "{template}" -""", - dockerfile="FROM python:3.11-slim\nWORKDIR /workspace\n", - ) - (task / "tests/test.sh").write_text( - '#!/bin/sh\nset -eu\n[ "$JUDGE_KEY" = "judge-value" ]\n' - "echo 1 > /logs/verifier/reward.txt\n", - encoding="utf-8", - ) - if mode == "separate": - (task / "tests/Dockerfile").write_text( - "FROM python:3.11-slim\nCOPY . /tests\n", encoding="utf-8" - ) - - async def grade() -> Run: - taskset = _adapt(dataset, hud_requirement=str(wheel)) - job = await taskset.run( - Oracle({"verifier-credentials": '[ "${JUDGE_KEY-unset}" = unset ]'}), - runtime=DockerRuntime( - env_vars={"HARBOR_VERIFIER_KEY": "judge-value"} if credential == "provided" else {} - ), - max_concurrent=1, - ) - (run,) = job.runs - return run - - run = asyncio.run(grade()) - - assert run.trace.content == "solution completed" - if credential == "missing": - assert run.reward != 1.0 - assert "HARBOR_VERIFIER_KEY" in (run.trace.error or str(run.evaluation)) - else: - assert run.reward == 1.0, run.trace.error or run.evaluation - - -def test_main_image_ports_preserve_the_outer_control_channel( - tmp_path_factory: pytest.TempPathFactory, - wheel: Path, -) -> None: - dataset = tmp_path_factory.mktemp("harbor-main-ports") / "harbor-harness" - task = make_harbor_task( - dataset, - "main-ports", - dockerfile="""\ -FROM debian:bookworm-slim -WORKDIR /app -COPY entrypoint.sh /usr/local/bin/start-environment -RUN chmod +x /usr/local/bin/start-environment && printf nested > /app/marker -EXPOSE 3128 3129 8765 8080 -ENTRYPOINT ["/usr/local/bin/start-environment"] -""", - task_toml="""\ -[environment.healthcheck] -command = "curl -fsS http://127.0.0.1:8765/marker && curl -fsS http://127.0.0.1:8080/marker" -interval_sec = 0.1 -timeout_sec = 5 -retries = 50 - -[verifier] -timeout_sec = 30 - -[[verifier.collect]] -service = "observer" -command = ''' -python - <<'PY' -from urllib.request import urlopen -assert urlopen('http://main:8080/marker').read() == b'nested' -PY -''' -timeout_sec = 10 -""", - ) - (task / "environment/entrypoint.sh").write_text( - "#!/bin/sh\nset -eu\n" - "python3 -m http.server 8765 --directory /app >/tmp/inner.log 2>&1 &\n" - "python3 -m http.server 8080 --directory /app >/tmp/app.log 2>&1 &\n" - 'exec "$@"\n', - encoding="utf-8", - ) - (task / "environment/docker-compose.yaml").write_text( - "services:\n main: {}\n observer:\n image: python:3.11-alpine\n" - ' command: ["python", "-m", "http.server", "9000"]\n expose: [9000]\n', - encoding="utf-8", - ) - (task / "solution").mkdir() - (task / "solution/solve.sh").write_text( - "curl -fsS http://127.0.0.1:8765/marker > /app/result\n", - encoding="utf-8", - ) - (task / "tests/test.sh").write_text( - '#!/bin/sh\nset -eu\ntest "$(cat /app/result)" = nested\n' - "echo 1 > /logs/verifier/reward.txt\n", - encoding="utf-8", - ) - - runs = asyncio.run(_grade_every_task(dataset, wheel)) - - test_harbor_phase_behavior(runs, "main-ports") - - -def test_adapter_install_ignores_vendor_uv_configuration( - tmp_path_factory: pytest.TempPathFactory, - wheel: Path, -) -> None: - """The private HUD install does not consume or rewrite the task's uv settings.""" - dataset = tmp_path_factory.mktemp("harbor-vendor-uv") / "harbor-harness" - task = make_harbor_task( - dataset, - "vendor-uv", - dockerfile="FROM python:3.11-slim\nWORKDIR /workspace\nCOPY pyproject.toml .\n", - ) - config = '[tool.uv]\nrequired-version = "==0.0.0"\n[tool.uv.pip]\nno-index = true\n' - config_path = task / "environment/pyproject.toml" - config_path.write_text(config, encoding="utf-8") - (adapted,) = _adapt(dataset, hud_requirement=str(wheel)) - assert adapted.runtime_config is not None - assert adapted.runtime_config.compose is not None - compose = adapted.runtime_config.compose.document - context = adapted.runtime_config.compose.root - assert isinstance(compose, Path) - assert isinstance(context, Path) - - build = subprocess.run( - ["docker", "compose", "--file", str(compose), "build"], - cwd=compose.parent, - capture_output=True, - text=True, - check=False, - timeout=600, - ) - - assert build.returncode == 0, build.stdout[-4000:] + build.stderr[-4000:] - assert config_path.read_text(encoding="utf-8") == config - copied_config = context / "compose-project/environment/pyproject.toml" - assert copied_config.read_text(encoding="utf-8") == config - - -def test_agent_sessions_run_the_managed_cli_bundle( - tmp_path_factory: pytest.TempPathFactory, - wheel: Path, -) -> None: - """A CLI bundle the runtime mounts at /usr/local/lib/agents runs in agent sessions.""" - root = tmp_path_factory.mktemp("harbor-managed-agents") - binary = root / "codex" / "bin" / "codex" - binary.parent.mkdir(parents=True) - binary.write_text("#!/bin/sh\necho managed-codex\n", encoding="utf-8") - binary.chmod(0o755) - dataset = root / "harbor-harness" - task = make_harbor_task( - dataset, - "managed-agents", - dockerfile="FROM python:3.11-slim\nWORKDIR /workspace\n", - ) - (task / "tests/test.sh").write_text( - '#!/bin/sh\nset -eu\ntest "$(cat /workspace/result)" = managed-codex\n' - "echo 1 > /logs/verifier/reward.txt\n", - encoding="utf-8", - ) - - async def grade() -> Run: - job = await _adapt(dataset, hud_requirement=str(wheel)).run( - Oracle({"managed-agents": "/usr/local/lib/agents/codex/bin/codex > result"}), - runtime=DockerRuntime( - bind_mounts=(DockerBindMount(root / "codex", "/usr/local/lib/agents/codex"),), - ), - max_concurrent=1, - ) - (run,) = job.runs - return run - - run = asyncio.run(grade()) - - assert run.trace.content == "solution completed" - assert run.reward == 1.0, run.trace.error or run.evaluation diff --git a/hud/patches/tests/__init__.py b/hud/patches/tests/__init__.py deleted file mode 100644 index 3c9f639f5..000000000 --- a/hud/patches/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Tests for HUD third-party patches.""" - -from __future__ import annotations diff --git a/hud/patches/tests/test_warnings.py b/hud/patches/tests/test_warnings.py deleted file mode 100644 index 5fca3356b..000000000 --- a/hud/patches/tests/test_warnings.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Regression tests for the ``authlib.jose`` deprecation-warning suppression. - -``authlib.deprecate`` runs ``warnings.simplefilter("always", AuthlibDeprecationWarning)`` -at import time, prepending an "always" filter to ``warnings.filters``. If -``suppress_known_import_warnings`` installs its "ignore" filter before authlib's -module is imported, authlib's filter lands ahead of ours and wins (the machinery -applies the first matching filter), so ``authlib.jose module is deprecated`` leaks -on every CLI launch. The fix imports ``authlib.deprecate`` first so our filter stays -ahead, and scopes the filter narrowly so nothing else is silenced. - -Filters are process-global and the breakage is purely about import order, so the -checks run in a fresh subprocess. The suppression module is loaded in isolation to -keep the interpreter free of unrelated filters that a full ``import hud`` registers. - -The warning is emitted directly via ``authlib.deprecate.deprecate`` (what -``authlib.jose`` does internally) rather than by importing ``authlib.jose``, so the -test behaves identically regardless of the resolved authlib version -- the jose -import only emits the warning from ``authlib>=1.7`` -- and therefore regardless of -platform. ``fastmcp`` pins only ``authlib>=1.6.5``, so that version (and thus whether -the warning appears at all) floats per machine; the suppression itself is pure stdlib -``warnings`` and is OS-independent. -""" - -from __future__ import annotations - -import subprocess -import sys -from pathlib import Path - -# Loads the suppression helper in a pristine interpreter without importing the rest -# of ``hud`` (which would register unrelated filters). A custom showwarning() runs -# only for warnings that pass the active filters, so the captured list reflects -# exactly what survives the filter -- deterministic across Python versions. -_SETUP = """ -import importlib.util -import sys -import warnings - -assert "authlib.deprecate" not in sys.modules, "test requires a pristine interpreter" - -spec = importlib.util.spec_from_file_location("_hud_patch_warnings", sys.argv[1]) -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) - -shown = [] -warnings.showwarning = lambda message, *args, **kwargs: shown.append(str(message)) -""" - -_SILENCES_JOSE = ( - _SETUP - + """ -module.suppress_known_import_warnings() - -from authlib.deprecate import deprecate - -# Exactly what authlib.jose raises at import time via authlib.deprecate.deprecate(). -deprecate("authlib.jose module is deprecated, please use joserfc instead.", version="2.0.0") - -leaked = [m for m in shown if "authlib.jose module is deprecated" in m] -assert not leaked, f"authlib.jose deprecation warning was not suppressed: {leaked}" -print("SUPPRESSED") -""" -) - -_LEAVES_OTHERS = ( - _SETUP - + """ -warnings.simplefilter("always") # baseline: nothing is hidden unless our filter says so -module.suppress_known_import_warnings() - -from authlib.deprecate import AuthlibDeprecationWarning - -warnings.warn(AuthlibDeprecationWarning("authlib.jose module is deprecated")) -warnings.warn(AuthlibDeprecationWarning("authlib.oauth2 client is deprecated")) -warnings.warn(DeprecationWarning("an unrelated deprecation")) -warnings.warn(UserWarning("an unrelated user warning")) - -assert not any("authlib.jose module is deprecated" in m for m in shown), shown -assert any("authlib.oauth2 client is deprecated" in m for m in shown), shown -assert any("an unrelated deprecation" in m for m in shown), shown -assert any("an unrelated user warning" in m for m in shown), shown -print("SPECIFIC") -""" -) - - -def _run_pristine(code: str) -> subprocess.CompletedProcess[str]: - warnings_module = Path(__file__).resolve().parents[1] / "warnings.py" - return subprocess.run( - [sys.executable, "-c", code, str(warnings_module)], - capture_output=True, - text=True, - timeout=60, - ) - - -def test_suppress_known_import_warnings_silences_authlib_jose_deprecation() -> None: - result = _run_pristine(_SILENCES_JOSE) - - assert result.returncode == 0, f"stdout={result.stdout!r}\nstderr={result.stderr!r}" - assert "SUPPRESSED" in result.stdout - - -def test_suppress_known_import_warnings_leaves_other_warnings_untouched() -> None: - result = _run_pristine(_LEAVES_OTHERS) - - assert result.returncode == 0, f"stdout={result.stdout!r}\nstderr={result.stderr!r}" - assert "SPECIFIC" in result.stdout diff --git a/hud/telemetry/tests/test_exporter.py b/hud/telemetry/tests/test_exporter.py deleted file mode 100644 index 5f5566890..000000000 --- a/hud/telemetry/tests/test_exporter.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Tests for telemetry exporter with mock backend.""" - -from __future__ import annotations - -import json -import subprocess -import sys -from typing import Any -from unittest.mock import patch - -import pytest - -from hud.settings import settings -from hud.telemetry.exporter import _do_upload, flush, queue_span - - -@pytest.fixture(autouse=True) -def drain_exporter(): - """Drain the background worker before and after each test.""" - assert flush(timeout=1.0) - yield - assert flush(timeout=1.0) - - -class _RecordingUpload: - """Captures (task_run_id, spans, api_key) for each upload.""" - - def __init__(self) -> None: - self.calls: list[tuple[str, list[dict[str, Any]], str]] = [] - - def __call__( - self, - task_run_id: str, - spans: list[dict[str, Any]], - telemetry_url: str, - api_key: str, - ) -> None: - self.calls.append((task_run_id, spans, api_key)) - - -def _configure( - monkeypatch: pytest.MonkeyPatch, - *, - api_key: str | None = "test-key", - enabled: bool = True, -) -> None: - """Pin the settings fields the exporter reads, on the real object. - - Field patches (not a whole-object MagicMock) keep every other setting - real — including the root conftest's isolation pins, which a mock object - would replace with truthy auto-attributes. - """ - monkeypatch.setattr(settings, "api_key", api_key) - monkeypatch.setattr(settings, "telemetry_enabled", enabled) - monkeypatch.setattr(settings, "hud_telemetry_url", "https://api.hud.ai") - - -class TestDoUpload: - def test_upload_posts_to_trace_endpoint(self): - with patch("hud.telemetry.exporter.make_request_sync") as mock_request: - _do_upload( - task_run_id="test-task-123", - spans=[{"name": "test.span"}], - telemetry_url="https://api.hud.ai", - api_key="test-key", - ) - - mock_request.assert_called_once() - kwargs = mock_request.call_args.kwargs - assert kwargs["method"] == "POST" - assert "test-task-123" in kwargs["url"] - assert kwargs["api_key"] == "test-key" - assert kwargs["json"] == {"telemetry": [{"name": "test.span"}]} - - def test_upload_swallows_request_errors(self): - with patch("hud.telemetry.exporter.make_request_sync", side_effect=Exception("boom")): - _do_upload("test-task-123", [{"name": "test.span"}], "https://api.hud.ai", "test-key") - - -class TestQueueSpan: - @pytest.mark.parametrize( - ("api_key", "enabled", "attributes"), - [ - (None, True, {"hud.task_run_id": "123"}), - ("test-key", False, {"hud.task_run_id": "123"}), - ("test-key", True, {}), - ], - ) - def test_span_is_dropped(self, monkeypatch, api_key, enabled, attributes): - upload = _RecordingUpload() - _configure(monkeypatch, api_key=api_key, enabled=enabled) - with patch("hud.telemetry.exporter._do_upload", side_effect=upload): - queue_span({"name": "test", "attributes": attributes}) - assert flush(timeout=1.0) - - assert upload.calls == [] - - def test_spans_upload_in_one_batch_per_trace(self, monkeypatch): - upload = _RecordingUpload() - _configure(monkeypatch) - with patch("hud.telemetry.exporter._do_upload", side_effect=upload): - queue_span({"name": "span-1", "attributes": {"hud.task_run_id": "task-1"}}) - queue_span({"name": "span-2", "attributes": {"hud.task_run_id": "task-1"}}) - queue_span({"name": "span-3", "attributes": {"hud.task_run_id": "task-2"}}) - assert flush(timeout=1.0) - - by_task = {task_run_id: spans for task_run_id, spans, _ in upload.calls} - assert [span["name"] for span in by_task["task-1"]] == ["span-1", "span-2"] - assert [span["name"] for span in by_task["task-2"]] == ["span-3"] - - def test_upload_uses_settings_api_key(self, monkeypatch): - upload = _RecordingUpload() - _configure(monkeypatch) - with patch("hud.telemetry.exporter._do_upload", side_effect=upload): - queue_span({"name": "test", "attributes": {"hud.task_run_id": "task-1"}}) - assert flush(timeout=1.0) - - assert [api_key for _, _, api_key in upload.calls] == ["test-key"] - - -class TestLocalExport: - def test_span_written_to_local_dir(self, monkeypatch, tmp_path): - _configure(monkeypatch, api_key=None, enabled=False) - monkeypatch.setattr(settings, "telemetry_local_dir", str(tmp_path)) - - queue_span({"name": "s", "trace_id": "t-1", "attributes": {"hud.task_run_id": "task-1"}}) - - lines = (tmp_path / "t-1.jsonl").read_text(encoding="utf-8").splitlines() - assert json.loads(lines[0])["name"] == "s" - - def test_disabled_uploads_write_to_home_spans(self, monkeypatch, tmp_path): - _configure(monkeypatch, api_key=None, enabled=False) - monkeypatch.setattr(settings, "telemetry_local_dir", None) - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - - queue_span({"name": "s", "trace_id": "t-1", "attributes": {"hud.task_run_id": "task-1"}}) - - lines = (tmp_path / ".hud" / "spans" / "t-1.jsonl").read_text(encoding="utf-8").splitlines() - assert json.loads(lines[0])["name"] == "s" - - def test_non_str_local_dir_fails_loudly(self, monkeypatch): - _configure(monkeypatch, api_key=None, enabled=False) - monkeypatch.setattr(settings, "telemetry_local_dir", object()) - - with pytest.raises(TypeError, match="telemetry_local_dir"): - queue_span({"name": "s", "attributes": {"hud.task_run_id": "task-1"}}) - - -class TestFlush: - def test_flush_is_noop_when_idle(self): - assert flush(timeout=1.0) - - def test_flush_drains_queued_spans(self, monkeypatch): - upload = _RecordingUpload() - _configure(monkeypatch) - with patch("hud.telemetry.exporter._do_upload", side_effect=upload): - queue_span({"name": "final-span", "attributes": {"hud.task_run_id": "task-1"}}) - assert flush(timeout=1.0) - - assert [span["name"] for _, spans, _ in upload.calls for span in spans] == ["final-span"] - - def test_interpreter_exit_uploads_spans_queued_without_a_flush(self, tmp_path): - uploads = tmp_path / "uploads.jsonl" - script = f""" -import json -from hud.settings import settings -from hud.telemetry import exporter - -def record(**kwargs): - with open({str(uploads)!r}, "a") as out: - out.write(json.dumps(kwargs["json"]) + "\\n") - -settings.api_key = "test-key" -settings.telemetry_enabled = True -settings.telemetry_local_dir = None -exporter.make_request_sync = record -exporter.queue_span({{"name": "last-span", "attributes": {{"hud.task_run_id": "task-1"}}}}) -""" - subprocess.run([sys.executable, "-c", script], check=True, timeout=60) - - uploaded = [json.loads(line) for line in uploads.read_text().splitlines()] - assert [span["name"] for body in uploaded for span in body["telemetry"]] == ["last-span"] diff --git a/hud/telemetry/tests/test_instrument.py b/hud/telemetry/tests/test_instrument.py deleted file mode 100644 index 72737b601..000000000 --- a/hud/telemetry/tests/test_instrument.py +++ /dev/null @@ -1,449 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any -from unittest.mock import patch - -import pytest - -from hud.telemetry.context import set_trace_context -from hud.telemetry.instrument import _serialize_value, instrument -from hud.types import MCPToolResult - - -def test_serialize_value_simple_types(): - """Test _serialize_value with simple types.""" - assert _serialize_value("string") == "string" - assert _serialize_value(42) == 42 - assert _serialize_value(3.14) == 3.14 - assert _serialize_value(True) is True - assert _serialize_value(None) is None - - -def test_serialize_value_list(): - """Test _serialize_value with lists.""" - result = _serialize_value([1, 2, 3]) - assert result == [1, 2, 3] - - -def test_serialize_value_list_truncation(): - """Test _serialize_value truncates long lists.""" - long_list = list(range(20)) - result = _serialize_value(long_list, max_items=5) - assert result == [0, 1, 2, 3, 4] - - -def test_serialize_value_tuple(): - """Test _serialize_value with tuples.""" - result = _serialize_value((1, 2, 3)) - assert result == [1, 2, 3] # Converted to list by JSON - - -def test_serialize_value_tuple_truncation(): - """Test _serialize_value truncates long tuples.""" - long_tuple = tuple(range(20)) - result = _serialize_value(long_tuple, max_items=5) - assert result == [0, 1, 2, 3, 4] - - -def test_serialize_value_dict(): - """Test _serialize_value with dicts.""" - result = _serialize_value({"key": "value"}) - assert result == {"key": "value"} - - -def test_serialize_value_dict_truncation(): - """Test _serialize_value truncates large dicts.""" - large_dict = {f"key{i}": i for i in range(20)} - result = _serialize_value(large_dict, max_items=5) - assert isinstance(result, dict) - assert len(result) == 5 - - -def test_serialize_value_complex_object(): - """Test _serialize_value with custom objects.""" - - @dataclass - class CustomObj: - name: str - value: int - - obj = CustomObj(name="test", value=42) - result = _serialize_value(obj) - assert isinstance(result, dict) - assert result["name"] == "test" - assert result["value"] == 42 - - -def test_serialize_value_fallback(): - """Test _serialize_value fallback for non-serializable objects.""" - - class WeirdObj: - def __init__(self): - raise Exception("Can't access") - - obj = WeirdObj.__new__(WeirdObj) - result = _serialize_value(obj) - # The result is a string representation of the object - assert isinstance(result, str) - assert "WeirdObj" in result - - -def test_serialize_value_pydantic_model_dumps_generically(): - """Domain models serialize through the generic pydantic path (no special-casing).""" - result = _serialize_value(MCPToolResult(content=[], isError=False)) - assert isinstance(result, dict) - assert result["isError"] is False - assert result["content"] == [] - - -@pytest.mark.asyncio -async def test_instrument_async_basic(): - """Test instrument decorator on async function.""" - - @instrument - async def test_func(x: int, y: int) -> int: - return x + y - - result = await test_func(2, 3) - assert result == 5 - - -@pytest.mark.asyncio -async def test_instrument_async_with_params(): - """Test instrument with custom parameters.""" - - @instrument(name="custom_name") - async def test_func(x: int) -> int: - return x * 2 - - result = await test_func(5) - assert result == 10 - - -@pytest.mark.asyncio -async def test_instrument_async_with_exception(): - """Test instrument handles exceptions.""" - - @instrument - async def test_func(): - raise ValueError("Test error") - - with pytest.raises(ValueError, match="Test error"): - await test_func() - - -@pytest.mark.asyncio -async def test_instrument_async_no_record_args(): - """Test instrument with record_args=False.""" - - @instrument(record_args=False) - async def test_func(x: int) -> int: - return x - - result = await test_func(42) - assert result == 42 - - -@pytest.mark.asyncio -async def test_instrument_async_no_record_result(): - """Test instrument with record_result=False.""" - - @instrument(record_result=False) - async def test_func() -> str: - return "test" - - result = await test_func() - assert result == "test" - - -def test_instrument_sync_basic(): - """Test instrument decorator on sync function.""" - - @instrument - def test_func(x: int, y: int) -> int: - return x + y - - result = test_func(2, 3) - assert result == 5 - - -def test_instrument_sync_with_params(): - """Test instrument on sync function with parameters.""" - - @instrument(name="sync_custom") - def test_func(x: int) -> int: - return x * 2 - - result = test_func(5) - assert result == 10 - - -def test_instrument_sync_with_exception(): - """Test instrument handles exceptions in sync functions.""" - - @instrument - def test_func(): - raise ValueError("Sync error") - - with pytest.raises(ValueError, match="Sync error"): - test_func() - - -def test_instrument_sync_no_record_args(): - """Test instrument sync with record_args=False.""" - - @instrument(record_args=False) - def test_func(x: int) -> int: - return x - - result = test_func(42) - assert result == 42 - - -def test_instrument_sync_no_record_result(): - """Test instrument sync with record_result=False.""" - - @instrument(record_result=False) - def test_func() -> str: - return "test" - - result = test_func() - assert result == "test" - - -def test_instrument_already_instrumented(): - """Test that instrumenting already instrumented function is skipped.""" - - @instrument - def test_func(): - return "original" - - # Try to instrument again - test_func2 = instrument(test_func) - - # Should be the same function - assert test_func2 is test_func - - -def test_instrument_marks_as_instrumented(): - """Test that instrument marks functions correctly.""" - - @instrument - def test_func(): - return True - - assert hasattr(test_func, "_hud_instrumented") - assert test_func._hud_instrumented is True - assert hasattr(test_func, "_hud_original") - - -@pytest.mark.asyncio -async def test_instrument_async_complex_result(): - """Test instrument with complex result object.""" - - @instrument - async def test_func() -> dict[str, Any]: - return {"nested": {"data": [1, 2, 3]}, "count": 3} - - result = await test_func() - assert result["count"] == 3 - - -def test_instrument_sync_complex_result(): - """Test instrument sync with complex result.""" - - @dataclass - class Result: - value: int - name: str - - @instrument - def test_func() -> Result: - return Result(value=42, name="test") - - result = test_func() - assert result.value == 42 - - -@pytest.mark.asyncio -async def test_instrument_async_with_self_param(): - """Test instrument properly handles 'self' parameter.""" - - class TestClass: - @instrument - async def method(self, x: int) -> int: - return x * 2 - - obj = TestClass() - result = await obj.method(5) - assert result == 10 - - -def test_instrument_sync_with_cls_param(): - """Test instrument properly handles 'cls' parameter.""" - - class TestClass: - @classmethod - @instrument - def method(cls, x: int) -> int: - return x * 3 - - result = TestClass.method(4) - assert result == 12 - - -@pytest.mark.asyncio -async def test_instrument_async_serialization_error(): - """Test instrument handles serialization errors gracefully.""" - - class UnserializableArg: - def __getattribute__(self, name): - raise Exception("Can't serialize") - - @instrument - async def test_func(arg): - return "success" - - # Should not raise, just skip serialization - result = await test_func(UnserializableArg()) - assert result == "success" - - -def test_instrument_function_without_signature(): - """Test instrument on functions without inspectable signature.""" - # Built-in functions don't have signatures - instrumented_len = instrument(len) - result = instrumented_len([1, 2, 3]) - assert result == 3 - - -@pytest.mark.asyncio -async def test_instrument_async_result_serialization_error(): - """Test instrument handles result serialization errors.""" - - class UnserializableResult: - def __iter__(self): - raise Exception("Can't iterate") - - @instrument - async def test_func(): - return UnserializableResult() - - # Should not raise, just skip result recording - result = await test_func() - assert isinstance(result, UnserializableResult) - - -def test_instrument_without_parentheses(): - """Test using @instrument without parentheses.""" - - @instrument - def test_func(x: int) -> int: - return x + 1 - - assert test_func(5) == 6 - - -def test_instrument_with_parentheses(): - """Test using @instrument() with parentheses.""" - - @instrument() - def test_func(x: int) -> int: - return x + 1 - - assert test_func(5) == 6 - - -@pytest.mark.asyncio -async def test_instrument_emits_debug_span_under_trace_context(): - """Spans carry the namespaced run id, no schema tag, and request/result events.""" - captured: list[dict[str, Any]] = [] - - @instrument(name="diag") - async def fn(x: int) -> str: - return "ok" - - with ( - patch("hud.telemetry.instrument.queue_span", side_effect=captured.append), - set_trace_context("run-123"), - ): - assert await fn(2) == "ok" - - (span,) = captured - assert span["name"] == "diag" - assert span["attributes"]["hud.task_run_id"] == "run-123" - assert "hud.schema" not in span["attributes"] - assert span["status_code"] == "OK" - assert [event["name"] for event in span["events"]] == ["hud.request", "hud.result"] - assert span["events"][0]["attributes"]["hud.payload"] == {"x": 2} - - -@pytest.mark.asyncio -async def test_instrument_records_exception_event(): - captured: list[dict[str, Any]] = [] - - @instrument(name="boom") - async def fn() -> None: - raise ValueError("bad") - - with ( - patch("hud.telemetry.instrument.queue_span", side_effect=captured.append), - set_trace_context("run-123"), - pytest.raises(ValueError, match="bad"), - ): - await fn() - - (span,) = captured - assert span["status_code"] == "ERROR" - assert span["status_message"] == "ValueError: bad" - assert span["events"][-1]["name"] == "exception" - - -@pytest.mark.asyncio -async def test_instrument_emits_nothing_without_trace_context(): - captured: list[dict[str, Any]] = [] - - @instrument - async def fn() -> int: - return 1 - - with patch("hud.telemetry.instrument.queue_span", side_effect=captured.append): - assert await fn() == 1 - - assert captured == [] - - -@pytest.mark.asyncio -async def test_instrument_async_with_defaults(): - """Test instrument with function that has default arguments.""" - - @instrument - async def test_func(x: int, y: int = 10) -> int: - return x + y - - assert await test_func(5) == 15 - assert await test_func(5, 20) == 25 - - -def test_instrument_sync_with_kwargs(): - """Test instrument with keyword arguments.""" - - @instrument - def test_func(x: int, **kwargs: Any) -> dict[str, Any]: - return {"x": x, **kwargs} - - result = test_func(1, a=2, b=3) - assert result == {"x": 1, "a": 2, "b": 3} - - -@pytest.mark.asyncio -async def test_instrument_async_with_varargs(): - """Test instrument with *args.""" - - @instrument - async def test_func(*args) -> int: - return sum(args) - - result = await test_func(1, 2, 3, 4) - assert result == 10 diff --git a/hud/telemetry/tests/test_robot_video.py b/hud/telemetry/tests/test_robot_video.py deleted file mode 100644 index c900775d6..000000000 --- a/hud/telemetry/tests/test_robot_video.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Tests for the robot trace video encoder.""" - -from __future__ import annotations - -import io -import threading - -import av -import numpy as np - -from hud.telemetry.robot.video import SegmentEncoder - - -def test_a_lossless_encoder_keeps_every_frame_while_its_queue_is_full() -> None: - frames = 60 - segments: list[bytes] = [] - released = threading.Event() - - def on_segment(_index: int, data: bytes) -> None: - released.wait() # stall the encoder so its one-frame queue stays full - segments.append(data) - - encoder = SegmentEncoder("cam", on_segment, fps=10, max_queued_frames=1, lossless=True) - threading.Timer(0.3, released.set).start() - for value in range(frames): - encoder.submit(np.full((16, 16, 3), value, dtype=np.uint8)) - encoder.finalize() - - with av.open(io.BytesIO(b"".join(segments)), mode="r") as container: - assert sum(1 for _ in container.decode(video=0)) == frames diff --git a/hud/tests/test_graders.py b/hud/tests/test_graders.py deleted file mode 100644 index 4b521cc49..000000000 --- a/hud/tests/test_graders.py +++ /dev/null @@ -1,648 +0,0 @@ -"""Tests for hud.graders: comparison helpers, combine, graders.""" - -from __future__ import annotations - -import json -import os -import warnings -from types import SimpleNamespace -from typing import Any -from unittest.mock import AsyncMock - -import pytest - -from hud.graders import ( - BashGrader, - EvaluationResult, - Grader, - LLMJudgeGrader, - SubScore, - combine, - combine_all, - combine_any, - contains, - contains_all, - contains_any, - exact_match, - f1_score, - normalize, - numeric_match, -) - - -class TestResultShapes: - def test_subscore_score_aliases_value(self) -> None: - s = SubScore(name="acc", value=0.75, weight=1.0) - assert s.score == 0.75 - - def test_evaluation_result_from_float(self) -> None: - r = EvaluationResult.from_float(0.25) - assert r.reward == 0.25 - assert r.done is True - - def test_evaluation_result_warns_when_subscores_disagree_with_reward(self) -> None: - with pytest.warns(UserWarning): - EvaluationResult(reward=1.0, subscores=[SubScore(name="a", value=0.5, weight=1.0)]) - - def test_subscore_summary_omits_recursive_info(self) -> None: - node = SubScore( - name="any", - value=1.0, - children=[SubScore(name="tests", value=1.0, info={"exit_code": 0})], - ) - assert node.model_dump(mode="json")["children"][0]["info"] == {"exit_code": 0} - - dumped = node.to_summary() - assert dumped["children"][0]["name"] == "tests" - assert "info" not in dumped - assert "info" not in dumped["children"][0] - - def test_evaluation_result_serializes_recursive_info(self) -> None: - result = EvaluationResult( - reward=1.0, - subscores=[ - SubScore( - name="judge", - value=1.0, - children=[ - SubScore( - name="Mentions Paris", - value=1.0, - info={"reason": "says Paris"}, - ) - ], - info={"model": "judge-model"}, - ) - ], - ) - dumped = result.model_dump(mode="json") - assert dumped["subscores"][0]["info"] == {"model": "judge-model"} - assert dumped["subscores"][0]["children"][0]["info"] == {"reason": "says Paris"} - - def test_subscore_accepts_metadata_alias(self) -> None: - subscore = SubScore.model_validate( - {"name": "legacy", "value": 1.0, "metadata": {"source": "legacy"}} - ) - assert subscore.info == {"source": "legacy"} - assert subscore.metadata == subscore.info - assert subscore.model_dump(mode="json")["info"] == {"source": "legacy"} - subscore.metadata = {"source": "updated"} - assert subscore.info == {"source": "updated"} - - -class TestNormalize: - def test_lowercases(self) -> None: - assert normalize("Hello World") == "hello world" - - def test_strips_punctuation(self) -> None: - assert normalize("answer: 42!") == "answer 42" - - def test_strips_articles(self) -> None: - assert normalize("The answer is a test") == "answer is test" - - def test_collapses_whitespace(self) -> None: - assert normalize(" lots of space ") == "lots of space" - - def test_non_string_input(self) -> None: - assert normalize(42) == "42" - - -class TestExactMatch: - def test_match_normalized(self) -> None: - assert exact_match("The Answer!", "answer") == 1.0 - - def test_no_match(self) -> None: - assert exact_match("Germany", "france") == 0.0 - - def test_punctuation_stripped(self) -> None: - assert exact_match("Paris.", "Paris") == 1.0 - - def test_articles_stripped(self) -> None: - assert exact_match("The capital is Paris", "capital is Paris") == 1.0 - - def test_no_normalize(self) -> None: - assert exact_match("Paris", "Paris", normalize_text=False) == 1.0 - assert exact_match("The Paris!", "Paris", normalize_text=False) == 0.0 - - def test_non_string_answer(self) -> None: - assert exact_match(42, "42") == 1.0 - - def test_empty_strings(self) -> None: - assert exact_match("", "") == 1.0 - - def test_different_content(self) -> None: - assert exact_match("hello", "world") == 0.0 - - -class TestContains: - def test_contains_substring(self) -> None: - assert contains("The capital of France is Paris", "paris") == 1.0 - - def test_not_contains(self) -> None: - assert contains("The capital of France is Paris", "berlin") == 0.0 - - def test_case_sensitive(self) -> None: - assert contains("Paris", "paris", case_sensitive=True) == 0.0 - assert contains("Paris", "Paris", case_sensitive=True) == 1.0 - - -class TestContainsAny: - def test_matches_one(self) -> None: - assert contains_any("I like Toyota cars", ["toyota", "honda", "nissan"]) == 1.0 - - def test_matches_none(self) -> None: - assert contains_any("I like BMW cars", ["toyota", "honda", "nissan"]) == 0.0 - - def test_empty_list(self) -> None: - assert contains_any("anything", []) == 0.0 - - def test_case_sensitive(self) -> None: - assert contains_any("Toyota", ["toyota"], case_sensitive=True) == 0.0 - - -class TestContainsAll: - def test_all_present(self) -> None: - assert contains_all("Toyota and Honda are Japanese", ["toyota", "honda"]) == 1.0 - - def test_some_missing(self) -> None: - assert contains_all("Toyota is Japanese", ["toyota", "honda"]) == 0.0 - - def test_empty_list(self) -> None: - assert contains_all("anything", []) == 1.0 - - -class TestNumericMatch: - def test_exact_integer(self) -> None: - assert numeric_match("The answer is 42", 42) == 1.0 - - def test_exact_float(self) -> None: - assert numeric_match("Result: 3.14", 3.14) == 1.0 - - def test_no_number(self) -> None: - assert numeric_match("no numbers", 42) == 0.0 - - def test_wrong_number(self) -> None: - assert numeric_match("The answer is 41", 42) == 0.0 - - def test_tolerance(self) -> None: - assert numeric_match("The answer is 41.5", 42, tolerance=1.0) == 1.0 - assert numeric_match("The answer is 40", 42, tolerance=1.0) == 0.0 - - def test_negative_number(self) -> None: - assert numeric_match("Temperature is -5 degrees", -5) == 1.0 - - def test_extracts_first_number(self) -> None: - assert numeric_match("There are 3 items and 5 left", 3) == 1.0 - - -class TestF1Score: - def test_exact_match(self) -> None: - assert f1_score("Paris", "Paris") == pytest.approx(1.0) - - def test_partial_overlap(self) -> None: - # pred=["capital", "is", "paris", "france"], ref=["paris"] - # common=1, precision=1/4, recall=1/1, f1=2*(0.25*1)/(0.25+1)=0.4 - assert f1_score("The capital is Paris, France", "Paris") == pytest.approx(0.4) - - def test_no_overlap(self) -> None: - assert f1_score("Berlin", "Paris") == pytest.approx(0.0) - - def test_empty_prediction(self) -> None: - assert f1_score("", "Paris") == pytest.approx(0.0) - - def test_empty_reference(self) -> None: - assert f1_score("Paris", "") == pytest.approx(0.0) - - def test_multi_word_match(self) -> None: - assert f1_score("New York City", "New York City") == pytest.approx(1.0) - - def test_superset_answer(self) -> None: - # pred has all ref tokens plus extras -> recall=1.0, precision<1.0 - result = f1_score("I believe the answer is 42 degrees", "42 degrees") - assert 0.0 < result < 1.0 - - def test_subset_answer(self) -> None: - # pred has fewer tokens than ref -> precision=1.0, recall<1.0 - result = f1_score("42", "42 degrees celsius") - assert 0.0 < result < 1.0 - - def test_normalization_applied(self) -> None: - assert f1_score("The PARIS!", "paris") == pytest.approx(1.0) - - -class TestCombineParallelism: - async def test_combine_sync_subscores(self) -> None: - result = await combine( - SubScore(name="a", value=1.0, weight=0.5), - SubScore(name="b", value=0.0, weight=0.5), - ) - assert result.reward == pytest.approx(0.5) - - async def test_combine_with_awaitables(self) -> None: - import asyncio - - order: list[str] = [] - - async def slow_check_a() -> SubScore: - order.append("a_start") - await asyncio.sleep(0.05) - order.append("a_end") - return SubScore(name="a", value=1.0, weight=0.5) - - async def slow_check_b() -> SubScore: - order.append("b_start") - await asyncio.sleep(0.05) - order.append("b_end") - return SubScore(name="b", value=0.0, weight=0.5) - - result = await combine(slow_check_a(), slow_check_b()) - assert result.reward == pytest.approx(0.5) - assert order.index("b_start") < order.index("a_end") - - async def test_combine_mixed(self) -> None: - import asyncio - - async def async_score() -> SubScore: - await asyncio.sleep(0.01) - return SubScore(name="async", value=1.0, weight=0.5) - - result = await combine( - SubScore(name="sync", value=0.0, weight=0.5), - async_score(), - ) - assert result.reward == pytest.approx(0.5) - - -#: ``BashGrader`` shells out to ``/bin/bash``; skip its tests where it's absent (Windows). -_HAS_BASH = os.path.exists("/bin/bash") - - -class TestCombine: - async def test_combine_returns_evaluation_result(self) -> None: - result = await combine(SubScore(name="alpha", value=1.0, weight=1.0)) - assert isinstance(result, EvaluationResult) - assert result.reward == 1.0 - assert result.done is True - - async def test_combine_normalizes_positive_weights(self) -> None: - result = await combine( - SubScore(name="alpha", value=1.0, weight=2.0), - SubScore(name="beta", value=0.0, weight=1.0), - ) - assert result.reward == pytest.approx(2.0 / 3.0) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - assert by_name["alpha"].weight == pytest.approx(2.0 / 3.0) - assert by_name["beta"].weight == pytest.approx(1.0 / 3.0) - - async def test_combine_preserves_negative_penalties(self) -> None: - result = await combine( - SubScore(name="correct", value=1.0, weight=1.0), - SubScore(name="penalty", value=1.0, weight=-0.2), - ) - assert result.reward == pytest.approx(0.8) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - assert by_name["correct"].weight == pytest.approx(1.0) - assert by_name["penalty"].weight == pytest.approx(-0.2) - - async def test_combine_duplicate_names_are_deduped(self) -> None: - result = await combine( - SubScore(name="same", value=1.0, weight=0.5), - SubScore(name="same", value=0.0, weight=0.5), - ) - assert result.subscores is not None - assert [subscore.name for subscore in result.subscores] == ["same-1", "same-2"] - - async def test_combine_duplicate_names_avoid_existing_suffix_collisions(self) -> None: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = await combine( - SubScore(name="x-1", value=1.0, weight=0.3), - SubScore(name="x", value=1.0, weight=0.4), - SubScore(name="x", value=0.0, weight=0.6), - ) - - assert result.subscores is not None - assert [subscore.name for subscore in result.subscores] == ["x-1", "x-2", "x-3"] - assert set(result.info) == set() - assert not [ - warning for warning in caught if "Duplicate subscore names" in str(warning.message) - ] - - async def test_combine_propagates_info(self) -> None: - info = {"stdout": "ok"} - result = await combine(SubScore(name="grader", value=1.0, weight=1.0, info=info)) - assert result.info == {} - assert result.subscores is not None - assert result.subscores[0].info == info - assert result.model_dump(mode="json")["subscores"][0]["info"] == info - - async def test_combine_keeps_rubric_children_on_subscores(self) -> None: - verdicts = [ - SubScore(name="Mentions Paris", value=1.0, info={"reason": "says Paris"}), - SubScore(name="Names the river", value=0.0, info={"reason": "no river"}), - ] - result = await combine( - SubScore(name="judge", value=0.5, weight=0.5, children=verdicts), - SubScore(name="tests", value=0.0, weight=0.5), - ) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - assert by_name["judge"].children == verdicts - assert by_name["tests"].children is None - - async def test_combine_preserves_combinator_children(self) -> None: - result = await combine( - combine_any( - weight=0.5, - subscores=[ - SubScore(name="pytest", value=1.0), - SubScore(name="make", value=0.0), - ], - ), - SubScore(name="format", value=1.0, weight=0.5), - ) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - any_node = by_name["any"] - assert any_node.children is not None - assert [child.name for child in any_node.children] == ["pytest", "make"] - assert by_name["format"].children is None - - async def test_combine_preserves_negative_reward_without_validator_warning(self) -> None: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = await combine( - SubScore(name="correct", value=0.0, weight=1.0), - SubScore(name="penalty", value=1.0, weight=-0.2), - ) - - assert result.reward == pytest.approx(-0.2) - assert not [ - warning for warning in caught if "Subscores don't match reward" in str(warning.message) - ] - - -class TestGrader: - async def test_grade_wraps_float_and_stores_parameters(self) -> None: - class DummyGrader(Grader): - name = "DummyGrader" - - @classmethod - async def compute_score(cls, **kwargs: object) -> Any: - return 0.75 - - subscore = await DummyGrader.grade(weight=0.4, marker="ok", payload=object()) - assert isinstance(subscore, SubScore) - assert subscore.name == "DummyGrader" - assert subscore.value == pytest.approx(0.75) - assert subscore.weight == pytest.approx(0.4) - assert subscore.info is not None - assert subscore.info["_parameters"]["marker"] == "ok" - assert subscore.info["_parameters"]["payload"] == "" - - async def test_grade_coerces_legacy_tuple_metadata(self) -> None: - class TupleGrader(Grader): - name = "tuple" - - @classmethod - async def compute_score(cls, **kwargs: object) -> Any: - return 0.75, {"source": "released-contract"} - - subscore = await TupleGrader.grade(weight=0.4, marker="ok") - assert subscore.name == "tuple" - assert subscore.value == pytest.approx(0.75) - assert subscore.weight == pytest.approx(0.4) - assert subscore.info is not None - assert subscore.info["source"] == "released-contract" - assert subscore.info["_parameters"]["marker"] == "ok" - - async def test_grade_uses_grader_name_without_override(self) -> None: - class RubricGrader(Grader): - name = "rubric" - - @classmethod - async def compute_score(cls, **kwargs: object) -> SubScore: - return SubScore(name="specific-rubric", value=1.0) - - subscore = await RubricGrader.grade(weight=0.7) - assert subscore.name == "rubric" - assert subscore.weight == pytest.approx(0.7) - - async def test_grade_stamps_name_and_weight_on_returned_subscore(self) -> None: - class RubricGrader(Grader): - name = "rubric" - - @classmethod - async def compute_score(cls, **kwargs: object) -> SubScore: - return SubScore( - name="ignored", - value=1.0, - info={"reason": "did the thing", "extra": "kept"}, - ) - - subscore = await RubricGrader.grade(weight=0.7, name="my-rubric") - assert subscore.name == "my-rubric" - assert subscore.weight == pytest.approx(0.7) - assert subscore.info is not None - assert subscore.info["reason"] == "did the thing" - assert subscore.info["extra"] == "kept" - assert "_parameters" in subscore.info - - -class TestBooleanCombinators: - def test_combine_any_picks_max(self) -> None: - combined = combine_any( - weight=1.0, - subscores=[ - SubScore(name="a", value=1.0, weight=0.5), - SubScore(name="b", value=0.0, weight=0.5), - ], - ) - assert combined.name == "any" - assert combined.value == 1.0 - - def test_combine_any_keeps_inputs_as_children(self) -> None: - rubric = SubScore( - name="judge", - value=1.0, - weight=0.5, - children=[SubScore(name="c1", value=1.0, info={"reason": "met"})], - ) - combined = combine_any( - weight=1.0, - subscores=[rubric, SubScore(name="tests", value=0.0, weight=0.5)], - ) - assert combined.children is not None - assert combined.children[0].children == rubric.children - assert combined.children[1].children is None - - def test_combine_any_dedupes_child_names(self) -> None: - combined = combine_any( - weight=1.0, - subscores=[ - SubScore(name="BashGrader", value=1.0, weight=0.5, info={"exit_code": 0}), - SubScore(name="BashGrader", value=0.0, weight=0.5, info={"exit_code": 1}), - ], - ) - assert combined.children is not None - assert [child.name for child in combined.children] == ["BashGrader-1", "BashGrader-2"] - assert [child.info for child in combined.children] == [ - {"exit_code": 0}, - {"exit_code": 1}, - ] - assert combined.info is None - - def test_combine_all_picks_min(self) -> None: - combined = combine_all( - weight=1.0, - subscores=[ - SubScore(name="a", value=1.0, weight=0.5), - SubScore(name="b", value=0.0, weight=0.5), - ], - name="tests_all", - ) - assert combined.name == "tests_all" - assert combined.value == 0.0 - - def test_nested_combinators_build_a_tree(self) -> None: - combined = combine_all( - weight=1.0, - subscores=[ - combine_any( - weight=1.0, - subscores=[ - SubScore(name="pytest", value=0.0), - SubScore(name="make", value=1.0), - ], - ), - SubScore(name="lint", value=1.0), - ], - ) - assert combined.value == 1.0 - assert combined.children is not None - inner = combined.children[0] - assert inner.children is not None - assert [child.name for child in inner.children] == ["pytest", "make"] - - -class _FakeGatewayClient: - """Judges MET when the criterion text (inside the user prompt) contains 'met:'.""" - - def __init__(self) -> None: - self.chat = SimpleNamespace(completions=SimpleNamespace(create=self._create)) - - async def _create(self, **kwargs: Any) -> Any: - user_prompt: str = kwargs["messages"][1]["content"] - status = "MET" if "met:" in user_prompt else "UNMET" - content = json.dumps({"criterion_status": status, "explanation": "because"}) - return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=content))]) - - -class TestLLMJudgeGrader: - async def test_compute_score_returns_rubric_node(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "hud.utils.gateway.build_gateway_client", lambda provider: _FakeGatewayClient() - ) - subscore = await LLMJudgeGrader.compute_score( - answer="some answer", - criteria=["met: mentions Paris", ("names the river", 3.0)], - ) - assert subscore.value == pytest.approx(0.25) - assert subscore.children is not None - assert [(c.name, c.value, c.weight, c.info) for c in subscore.children] == [ - ("met: mentions Paris", 1.0, 1.0, {"reason": "because"}), - ("names the river", 0.0, 3.0, {"reason": "because"}), - ] - - async def test_compute_score_applies_negative_criterion_penalty( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr( - "hud.utils.gateway.build_gateway_client", lambda provider: _FakeGatewayClient() - ) - subscore = await LLMJudgeGrader.compute_score( - answer="some answer", - criteria=[("met: positive", 1.0), ("met: error", -0.5)], - ) - assert subscore.value == pytest.approx(0.5) - - async def test_grade_puts_verdicts_on_children(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "hud.utils.gateway.build_gateway_client", lambda provider: _FakeGatewayClient() - ) - subscore = await LLMJudgeGrader.grade( - weight=1.0, answer="some answer", criteria=["met: correct"] - ) - assert subscore.children == [ - SubScore( - name="met: correct", - value=1.0, - weight=1.0, - info={"reason": "because"}, - ) - ] - assert subscore.info is not None - assert subscore.info["model"] == "claude-haiku-4-5" - - -@pytest.mark.skipif(not _HAS_BASH, reason="/bin/bash not available (e.g. Windows)") -class TestBashGrader: - async def test_compute_score_requires_command(self) -> None: - with pytest.raises(ValueError, match="requires command"): - await BashGrader.compute_score() - - async def test_compute_score_for_passing_command(self) -> None: - subscore = await BashGrader.compute_score(command="echo hello") - assert subscore.value == 1.0 - assert subscore.info is not None - assert subscore.info["exit_code"] == 0 - assert "hello" in subscore.info["stdout"] - - async def test_compute_score_for_failing_command(self) -> None: - subscore = await BashGrader.compute_score(command="echo oops >&2 && false") - assert subscore.value == 0.0 - assert subscore.info is not None - assert subscore.info["exit_code"] != 0 - assert "oops" in subscore.info["stderr"] - - async def test_compute_score_timeout(self) -> None: - subscore = await BashGrader.compute_score(command="sleep 2", timeout_seconds=1) - assert subscore.value == 0.0 - assert subscore.info is not None - assert subscore.info["timed_out"] is True - assert subscore.info["timeout"] == 1 - - async def test_compute_score_timeout_keeps_partial_output( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - from hud.graders import bash as bash_mod - from hud.utils.process import ProcessResult - - process = SimpleNamespace( - complete=AsyncMock( - return_value=ProcessResult(None, b"progress\n", b"still working\n", timed_out=True) - ) - ) - monkeypatch.setattr(bash_mod, "create_process_group_exec", AsyncMock(return_value=process)) - - subscore = await BashGrader.compute_score(command="slow", timeout_seconds=1) - - assert subscore.info is not None - assert subscore.info["stdout"] == "progress\n" - assert subscore.info["stderr"] == "still working\n" - - async def test_grade_and_combine_compose(self) -> None: - result = await combine( - BashGrader.grade(weight=0.5, command="true"), - BashGrader.grade(weight=0.5, command="false"), - ) - assert result.reward == pytest.approx(0.5) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - assert by_name["BashGrader-1"].info is not None - assert by_name["BashGrader-1"].info["exit_code"] == 0 - assert by_name["BashGrader-2"].info is not None - assert by_name["BashGrader-2"].info["exit_code"] != 0 - assert result.info == {} diff --git a/hud/tests/test_init.py b/hud/tests/test_init.py deleted file mode 100644 index 67ac56186..000000000 --- a/hud/tests/test_init.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Tests for hud.__init__ module.""" - -from __future__ import annotations - - -class TestHudInit: - """Tests for the hud package initialization.""" - - def test_version_import_success(self): - """Test that version is imported successfully.""" - import hud - - # Version should be available - assert hasattr(hud, "__version__") - assert isinstance(hud.__version__, str) - - def test_all_exports_available(self): - """Test that all exported functions are available.""" - import hud - - expected_exports = [ - "Chat", - "DockerRuntime", - "Environment", - "Grade", - "Job", - "HUDRuntime", - "HostedRuntime", - "Run", - "Runtime", - "RuntimeConfig", - "RuntimeGPU", - "RuntimeLimits", - "RuntimeResources", - "RuntimeTPU", - "LocalRuntime", - "SubprocessRuntime", - "SyncPlan", - "Task", - "Taskset", - "connect", - "instrument", - ] - - for export in expected_exports: - assert hasattr(hud, export), f"Missing export: {export}" diff --git a/hud/tests/test_init_module.py b/hud/tests/test_init_module.py deleted file mode 100644 index 83437c458..000000000 --- a/hud/tests/test_init_module.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Tests for hud/__init__.py module initialization.""" - -from __future__ import annotations - - -class TestInitModule: - """Test the hud module initialization.""" - - def test_version_exposed(self): - """Test that the package exposes its version.""" - import hud - - assert hasattr(hud, "__version__") - assert isinstance(hud.__version__, str) - - def test_all_exports(self): - """Test that __all__ contains expected exports.""" - import hud - - expected = [ - "Chat", - "ComposeProject", - "Connection", - "DockerRuntime", - "Environment", - "Grade", - "Job", - "HUDRuntime", - "HostedRuntime", - "Run", - "Runtime", - "RuntimeConfig", - "RuntimeGPU", - "RuntimeLimits", - "RuntimeResources", - "RuntimeTPU", - "LocalRuntime", - "SubprocessRuntime", - "SyncPlan", - "Task", - "Taskset", - "Trace", - "TrainingClient", - "__version__", - "connect", - "instrument", - ] - - assert set(hud.__all__) == set(expected) - - # Verify all exported items are actually available - for item in hud.__all__: - assert hasattr(hud, item) diff --git a/hud/tests/test_robot.py b/hud/tests/test_robot.py deleted file mode 100644 index f4500d09d..000000000 --- a/hud/tests/test_robot.py +++ /dev/null @@ -1,373 +0,0 @@ -"""End-to-end gates for the ``robot`` capability — the flows the docs promise. - -Every test drives a real rollout: an env-side bridge serving the ``openpi/0`` -WebSocket and its JSON-RPC control channel, an agent-side ``RobotAgent`` dialing -it through ``RobotClient``, graded by the rollout engine. So a change that breaks -the wire, the contract split, or the slot token fails here rather than in a GPU -benchmark. The flows (see ``docs/v6/advanced/robots.mdx``): - -- a single-env custom bridge, where the slot token is optional; -- a vectorized sim, where the per-episode token pins each rollout to its slot; -- ``env.gym`` behind a pre-written contract — the shape ``hud-evals/robot-template`` - ships (two cameras + an 8-D state, LIBERO-style). - -The policies here are stubs; the plumbing is not. -""" - -from __future__ import annotations - -import asyncio -import json -from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime -from contextlib import asynccontextmanager -from typing import TYPE_CHECKING, Any - -import numpy as np -import pytest - -from hud.agents.robot import Adapter, Model, RobotAgent -from hud.environment import Environment -from hud.environment.robot import RobotBridge, RobotEndpoint -from hud.eval import LocalRuntime, Shared, Task, Taskset, rollout - -if TYPE_CHECKING: - from numpy.typing import NDArray - -if TYPE_CHECKING: - from collections.abc import AsyncIterator - from pathlib import Path - - from hud.eval import Provider - -#: Ticks a stub sim runs before terminating — every gate here is a whole rollout. -EPISODE_TICKS = 3 - -#: A minimal contract, exactly as documented: one camera, a state vector, an action. -STUB_CONTRACT = { - "control_rate": 5, - "features": { - "observation/image": {"role": "observation", "type": "rgb"}, - "observation/state": {"role": "observation", "names": ["slot", "tick"]}, - "action": {"role": "action", "names": ["a0", "a1"]}, - }, -} - - -# ─── the sim side ────────────────────────────────────────────────────── - - -class _StubSim(RobotBridge): - """One camera + a 2-D state, terminating after ``EPISODE_TICKS`` ticks. - - Slot ``i``'s state carries ``i + 1``, and its grade is the sum of the actions - that slot received — so with an echoing policy the grade names the slot the - connection actually drove. - """ - - def __init__(self, num_envs: int = 1) -> None: - super().__init__() - self.num_envs = num_envs - self.contract = STUB_CONTRACT - self.tick = 0 - self.executed = np.zeros(num_envs) - - def reset(self, **task_args: Any) -> str: - self.tick = 0 - self.executed[:] = 0.0 - return f"stub task: {task_args.get('goal', 'none')}" - - def step(self, action: NDArray[Any]) -> None: - self.tick += 1 - self.executed += np.asarray(action)[:, 0] # one row per slot - - def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: - slots = np.arange(1, self.num_envs + 1, dtype=np.float32) - data = { - "observation/image": np.zeros((self.num_envs, 16, 16, 3), dtype=np.uint8), - "observation/state": np.stack( - [slots, np.full(self.num_envs, self.tick, dtype=np.float32)], axis=1 - ), - } - return data, np.full(self.num_envs, self.tick >= EPISODE_TICKS) - - def result_slots(self) -> list[dict[str, Any]]: - # Per-slot grades, never one batch-wide scalar: the token has to resolve. - return [ - {"score": float(echoed), "success": bool(echoed), "total_reward": float(echoed)} - for echoed in self.executed - ] - - -@asynccontextmanager -async def _sim_env(sim: RobotBridge) -> AsyncIterator[tuple[Environment, RobotEndpoint]]: - """The docs' custom-bridge env, with *sim* served by this process. - - ``RobotEndpoint.remote`` attaches to a sim someone else runs — here the test - itself, so the whole control plane is real without a spawned sim program. - Yields the env and its endpoint; each test declares the template it is about. - """ - await sim.start() - control = await sim.serve_control() - env = Environment("stub-sim") - endpoint = RobotEndpoint.remote("127.0.0.1", control.sockets[0].getsockname()[1]).attach(env) - - @env.initialize - async def _up() -> None: - await endpoint.start() - for cap in await endpoint.capabilities(): - env.add_capability(cap) - - @env.shutdown - async def _down() -> None: - await endpoint.stop() - - try: - await env.start() # publish the capability now; serving re-enters as a no-op - yield env, endpoint - finally: - await env.stop() - control.close() - await sim.stop() - - -def _serving(env: Environment) -> Provider: - """Placement that serves this one env — the substrate a vectorized sim shares.""" - return LocalRuntime(env) - - -# ─── the agent side ──────────────────────────────────────────────────── - - -class _EchoAdapter(Adapter): - """Wires from the contract alone, and records what it wired for the gate.""" - - def __init__(self) -> None: - super().__init__() - self.wired: list[dict[str, Any]] = [] - - def adapt_observation(self, obs: dict[str, Any], prompt: str) -> dict[str, Any]: - data = obs["data"] - batch = {"state": data[self.state_key], "image": data[self.image_keys[0]], "task": prompt} - self.wired.append(batch) - return batch - - -class _EchoModel(Model): - """Echoes the state's slot id, in the action width its checkpoint was trained for.""" - - def __init__(self, action_dim: int) -> None: - self.action_dim = action_dim - - def infer(self, batch: Any) -> Any: - slot = float(np.asarray(batch["state"]).reshape(-1)[0]) - return np.full((1, 1, self.action_dim), slot, dtype=np.float32) # [N, T, A] - - -class _EchoAgent(RobotAgent): - """Stub harness subclass: the model/adapter pair, base owns the loop.""" - - max_steps = EPISODE_TICKS + 2 # the env must end the episode, not this cap - - def __init__(self, adapter: _EchoAdapter, *, action_dim: int = 2) -> None: - super().__init__() - self.model = _EchoModel(action_dim) - self.adapter = adapter - - -class _EarlyStopAgent(_EchoAgent): - """Stop after one action through the public rollout hook.""" - - def should_stop(self, obs: dict[str, Any], *, step: int, max_steps: int) -> bool: - return step >= 1 or super().should_stop(obs, step=step, max_steps=max_steps) - - -# ─── gates ───────────────────────────────────────────────────────────── - - -async def test_single_env_rollout_drives_the_policy_and_grades_the_episode() -> None: - sim = _StubSim() - adapter = _EchoAdapter() - - async with _sim_env(sim) as (env, endpoint): - - @env.template() - async def episode(goal: str = "lift") -> AsyncGenerator[Any, Any]: - ep = await endpoint.reset(goal=goal) - yield {"prompt": ep["prompt"]} # one slot: the token is optional - yield await endpoint.result() - - run = await rollout( - Task(env="stub-sim", id="episode", args={"goal": "lift"}), - _EchoAgent(adapter), - runtime=_serving(env), - ) - - assert run.reward == EPISODE_TICKS # every tick's echoed action reached the sim - assert sim.tick == EPISODE_TICKS # the env terminated the episode, not max_steps - assert run.prompt == "stub task: lift" # the sim's prompt, built from the task arg - # The contract is the whole wiring: its camera arrives HWC uint8, its state as named. - assert [batch["task"] for batch in adapter.wired] == [run.prompt] * EPISODE_TICKS - assert adapter.wired[0]["image"].shape == (16, 16, 3) - assert adapter.wired[0]["image"].dtype == np.uint8 - assert adapter.wired[0]["state"].tolist() == [1.0, 0.0] # slot 1, before any step - - -async def test_agent_stop_hook_ends_the_rollout_before_the_env_terminates() -> None: - sim = _StubSim() - - async with _sim_env(sim) as (env, endpoint): - - @env.template() - async def episode() -> AsyncGenerator[Any, Any]: - ep = await endpoint.reset() - yield {"prompt": ep["prompt"]} - yield await endpoint.result() - - run = await rollout( - Task(env="stub-sim", id="episode"), - _EarlyStopAgent(_EchoAdapter()), - runtime=_serving(env), - ) - - assert sim.tick == 1 - assert run.reward == 1 - - -async def test_the_slot_token_pins_each_rollout_to_its_own_slot() -> None: - sim = _StubSim(num_envs=2) - # Both rollouts must hold a slot before either acts: a sequential pair would each - # get their own global reset (and slot 0), which is not the case under test. - claimed = asyncio.Barrier(2) - - async with _sim_env(sim) as (env, endpoint): - - @env.template() - async def episode(goal: str = "lift") -> AsyncGenerator[Any, Any]: - ep = await endpoint.reset(goal=goal) - await claimed.wait() - # Vectorized: the token is this episode's binding, and grades its slot alone. - yield {"prompt": ep["prompt"], "bindings": {"robot": {"token": ep["token"]}}} - yield await endpoint.result(token=ep["token"]) - - job = await Taskset("stub", [Task(env="stub-sim", id="episode")]).run( - _EchoAgent(_EchoAdapter()), - runtime=Shared(_serving(env), width=2), - group=2, - max_concurrent=2, - ) - - # Slot i echoes i + 1 per tick, so distinct grades mean neither rollout read or - # graded the other's slot — a dropped or crossed token grades them the same. - assert sorted(run.reward for run in job.runs) == [EPISODE_TICKS, 2 * EPISODE_TICKS] - - -async def test_a_vectorized_sim_refuses_a_tokenless_grade() -> None: - sim = _StubSim(num_envs=2) - - async with _sim_env(sim) as (_env, endpoint): - first = await endpoint.reset(goal="lift") - second = await endpoint.reset(goal="lift") - assert first["token"] != second["token"] # one token per slot, not per batch - - # Two claimed slots: which one to grade is ambiguous, so say so instead of guessing. - with pytest.raises(RuntimeError, match="tokenless claim"): - await endpoint.result() - - -# ─── the robot-template shape: env.gym behind a pre-written contract ─── - -#: ``hud-evals/robot-template``'s ``contract_lerobot.json``: two cameras, an 8-D -#: LIBERO state, a 7-D action. The agent wires the policy from this alone. -TEMPLATE_CONTRACT = { - "control_rate": 80, - "features": { - "observation/image": {"role": "observation", "type": "rgb"}, - "observation/wrist_image": {"role": "observation", "type": "rgb"}, - "observation/state": {"role": "observation", "names": [f"state_{i}" for i in range(8)]}, - "action": {"role": "action", "names": [f"act_{i}" for i in range(7)]}, - }, -} - -#: Stands in for the template's ``lerobot_sim.py`` — LIBERO's shape without LIBERO. -TEMPLATE_SIM = '''\ -"""Module-level ``make_env`` for ``env.gym``; suite/id are build args.""" - -from types import SimpleNamespace - -import numpy as np - - -class _Libero: - """Two cameras + an 8-D state, 7-D action, env-reported success on the last tick.""" - - action_space = SimpleNamespace(shape=(7,), dtype=np.float32) - task_description = "pick up the black bowl" # becomes the episode prompt - - def reset(self, seed=None, options=None): - self.tick = 0 - return self._observation(), {} - - def step(self, action): - self.tick += 1 - done = self.tick >= 3 # inside the agent's max_steps - return self._observation(), 1.0, done, False, {"success": done} - - def close(self): - pass - - def _observation(self): - frame = np.zeros((32, 32, 3), dtype=np.uint8) - return { - "observation/image": frame, - "observation/wrist_image": frame, - "observation/state": np.zeros(8, dtype=np.float32), - } - - -def make_env(task_suite="libero_spatial", task_id=0): - return _Libero() -''' - -#: The template's ``environment/env.py``, unchanged but for the sim it imports. -TEMPLATE_ENV = '''\ -"""LIBERO as one declarative HUD env — the task is a template arg.""" - -from pathlib import Path - -from template_sim import make_env - -from hud import Environment - -env = Environment(name="libero") -sim = env.gym(make_env, contract=str(Path(__file__).parent / "contract.json")) - - -@env.template() -async def episode(task_suite: str = "libero_spatial", task_id: int = 0, seed: int = 0): - """One LIBERO episode; the sim provides the instruction as the prompt.""" - ep = await sim.reset(task_suite=task_suite, task_id=task_id, seed=seed) - yield {"prompt": ep["prompt"]} # single env: the slot token is optional - yield await sim.result() -''' - - -async def test_a_gym_sim_behind_a_prewritten_contract_runs_the_template_flow( - tmp_path: Path, -) -> None: - (tmp_path / "template_sim.py").write_text(TEMPLATE_SIM, encoding="utf-8") - (tmp_path / "contract.json").write_text(json.dumps(TEMPLATE_CONTRACT), encoding="utf-8") - (tmp_path / "env.py").write_text(TEMPLATE_ENV, encoding="utf-8") - adapter = _EchoAdapter() - - run = await rollout( - Task(env="libero", id="episode", args={"task_id": 0}), - _EchoAgent(adapter, action_dim=7), - runtime=LocalRuntime(tmp_path / "env.py"), - ) - - assert run.reward == 1.0 # the sim's own success outranks accumulated reward - assert run.prompt == "pick up the black bowl" # the sim's instruction, not the task args - # A pre-written contract is loaded verbatim, cameras in file order, state after them. - assert adapter.image_keys == ["observation/image", "observation/wrist_image"] - assert adapter.state_key == "observation/state" - assert adapter.wired[0]["state"].shape == (8,) diff --git a/hud/tests/test_settings.py b/hud/tests/test_settings.py deleted file mode 100644 index ad75e85f8..000000000 --- a/hud/tests/test_settings.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Tests for settings module.""" - -from __future__ import annotations - -from hud.settings import Settings, get_settings, settings - - -def test_get_settings(): - """Test that get_settings returns the singleton settings instance.""" - result = get_settings() - assert isinstance(result, Settings) - assert result is settings # Should be the same singleton instance - - -def test_service_url_defaults(): - expected = { - "hud_telemetry_url": "https://telemetry.hud.ai/v3/api", - "hud_api_url": "https://api.hud.ai", - "hud_web_url": "https://hud.ai", - "hud_gateway_url": "https://inference.hud.ai", - "hud_runtime_url": "https://mcp.hud.ai", - "hud_rl_url": "https://rl.hud.ai", - } - - for name, default in expected.items(): - assert Settings.model_fields[name].default == default - - -def test_file_tracking_is_enabled_by_default(): - assert Settings.model_fields["file_tracking_enabled"].default is True - - -def test_file_tracking_can_be_disabled_by_env(monkeypatch): - monkeypatch.setenv("HUD_FILE_TRACKING_ENABLED", "false") - - assert Settings().file_tracking_enabled is False - - -def test_default_project_accepts_a_name_or_id(monkeypatch): - monkeypatch.setenv("HUD_DEFAULT_PROJECT", "browser-evals") - - assert Settings().default_project == "browser-evals" - - -def test_span_dir_defaults_to_home_when_uploads_are_off(monkeypatch, tmp_path): - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "false") - monkeypatch.setenv("HUD_TELEMETRY_LOCAL_DIR", "") - monkeypatch.setattr("hud.settings.Path.home", lambda: tmp_path) - - assert Settings().span_dir == str(tmp_path / ".hud" / "spans") - - -def test_span_dir_explicit_local_dir_wins_when_uploads_are_off(monkeypatch): - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "false") - monkeypatch.setenv("HUD_TELEMETRY_LOCAL_DIR", "./spans") - - assert Settings().span_dir == "./spans" - - -def test_span_dir_is_unset_when_uploads_are_on(monkeypatch): - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "true") - monkeypatch.setenv("HUD_TELEMETRY_LOCAL_DIR", "") - - assert Settings().span_dir is None - - -def test_cli_analytics_is_independent_of_trace_telemetry(monkeypatch): - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "true") - monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "false") - configured = Settings() - - assert configured.telemetry_enabled is True - assert configured.cli_analytics_enabled is False - - -def test_settings_singleton(): - """Test that settings is a singleton.""" - s1 = get_settings() - s2 = get_settings() - assert s1 is s2 - assert s1 is settings diff --git a/hud/tests/test_trace.py b/hud/tests/test_trace.py deleted file mode 100644 index 666999232..000000000 --- a/hud/tests/test_trace.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Core trajectory contract tests: ``Trace`` invariants + step span emission.""" - -from __future__ import annotations - -from typing import Any -from unittest.mock import patch - -from mcp import types as mcp_types - -from hud.telemetry.context import set_trace_context -from hud.types import Step, Trace - - -def test_trace_final_returns_newest_non_none_answer(): - """final() asks newest-first; None means "no answer", falsy answers win.""" - trace = Trace() - trace.record(Step(source="agent", extra={"note": "first"})) - trace.record(Step(source="agent", extra={"note": ""})) - trace.record(Step(source="tool")) - - assert trace.final(lambda s: s.extra.get("note")) == "" - assert trace.final(lambda s: s.error) is None - - -def test_trace_collect_gathers_answers_in_step_order(): - """collect() keeps step order and skips steps that answer None.""" - trace = Trace() - trace.record(Step(source="agent", extra={"n": 1})) - trace.record(Step(source="tool")) - trace.record(Step(source="agent", extra={"n": 2})) - - assert trace.collect(lambda s: s.extra.get("n")) == [1, 2] - - -def test_trace_append_numbers_steps(): - """Trace.append assigns sequential 1-based step ids.""" - trace = Trace() - trace.record(Step(source="user")) - trace.record(Step(source="agent")) - assert len(trace) == 2 - assert [step.step_id for step in trace.steps] == [1, 2] - - -def test_trace_validator_numbers_preloaded_steps(): - """Steps passed to the constructor are renumbered on validation.""" - trace = Trace(steps=[Step(source="user"), Step(source="agent"), Step(source="tool")]) - assert [step.step_id for step in trace.steps] == [1, 2, 3] - - -def test_trace_error_surfaces_last_step_error(): - """Trace.error reads the most recent step error; is_error follows status.""" - trace = Trace() - assert trace.error is None - assert trace.is_error is False - - trace.record(Step(source="tool", error="first")) - trace.record(Step(source="agent")) - trace.record(Step(source="system", error="second")) - trace.status = "error" - - assert trace.error == "second" - assert trace.is_error is True - - -def test_step_emit_wraps_step_in_schema_tagged_span(): - captured: list[dict[str, Any]] = [] - step = Step( - source="user", - messages=[ - mcp_types.PromptMessage( - role="user", - content=mcp_types.TextContent(type="text", text="do the thing"), - ), - ], - ) - - with ( - patch("hud.types.queue_span", side_effect=captured.append), - set_trace_context("run-1"), - ): - step.emit() - - (span,) = captured - assert span["name"] == "step.user" - assert span["attributes"]["hud.schema"] == "hud.step.v1" - assert span["attributes"]["hud.task_run_id"] == "run-1" - payload = span["attributes"]["hud.payload"] - assert payload["source"] == "user" - assert payload["messages"][0]["content"]["text"] == "do the thing" - assert span["status_code"] == "OK" - - -def test_step_emit_marks_error_status(): - captured: list[dict[str, Any]] = [] - - with ( - patch("hud.types.queue_span", side_effect=captured.append), - set_trace_context("run-1"), - ): - Step(source="system", error="boom").emit() - - (span,) = captured - assert span["status_code"] == "ERROR" - assert span["status_message"] == "boom" - - -def test_step_emit_without_context_is_noop(): - captured: list[dict[str, Any]] = [] - - with patch("hud.types.queue_span", side_effect=captured.append): - Step(source="system", error="boom").emit() - - assert captured == [] - - -def test_trace_record_emits_and_stamps_end(): - """record = number + stamp end + append + emit, in one call.""" - captured: list[dict[str, Any]] = [] - trace = Trace() - - with ( - patch("hud.types.queue_span", side_effect=captured.append), - set_trace_context("run-1"), - ): - trace.record(Step(source="user")) - trace.record(Step(source="agent", ended_at="2026-05-14T20:00:05Z")) - - assert [span["attributes"]["hud.payload"]["step_id"] for span in captured] == [1, 2] - assert trace.steps[0].ended_at is not None # stamped at record time - assert trace.steps[1].ended_at == "2026-05-14T20:00:05Z" # explicit timing kept - assert captured[1]["end_time"] == "2026-05-14T20:00:05Z" diff --git a/hud/tests/test_types.py b/hud/tests/test_types.py deleted file mode 100644 index 00107901d..000000000 --- a/hud/tests/test_types.py +++ /dev/null @@ -1,164 +0,0 @@ -from __future__ import annotations - -from unittest.mock import patch - -from mcp.types import ImageContent, TextContent - -from hud.types import MCPToolCall, MCPToolResult - - -def test_mcp_tool_call_str_long_args(): - """Test MCPToolCall __str__ truncates long arguments.""" - tool_call = MCPToolCall( - name="test_tool", - arguments={"very": "long" * 30 + " argument string that should be truncated"}, - ) - result = str(tool_call) - assert "..." in result - assert len(result) < 100 - - -def test_mcp_tool_call_str_invalid_json_args(): - """Test MCPToolCall __str__ handles non-JSON-serializable arguments.""" - tool_call = MCPToolCall(name="test_tool", arguments={"func": lambda x: x}) - result = str(tool_call) - assert "test_tool" in result - - -def test_mcp_tool_call_rich(): - """Test MCPToolCall __rich__ calls hud_console.""" - with patch("hud.utils.hud_console.hud_console") as mock_console: - mock_console.format_tool_call.return_value = "formatted" - tool_call = MCPToolCall(name="test", arguments={}) - result = tool_call.__rich__() - assert result == "formatted" - mock_console.format_tool_call.assert_called_once() - - -def test_mcp_tool_call_annotation_in_model_dump(): - """model_dump() includes annotation when set.""" - tool_call = MCPToolCall(name="click", arguments={"x": 100}, annotation="Navigate to login page") - data = tool_call.model_dump() - assert data["annotation"] == "Navigate to login page" - - -def test_mcp_tool_call_annotation_roundtrip(): - """Annotation survives serialize -> deserialize roundtrip.""" - original = MCPToolCall(name="click", arguments={"x": 100}, annotation="Step 1: open menu") - data = original.model_dump(mode="json") - restored = MCPToolCall(**data) - assert restored.annotation == "Step 1: open menu" - assert restored.name == original.name - assert restored.arguments == original.arguments - - -def test_mcp_tool_call_annotation_none_excluded(): - """model_dump(exclude_none=True) omits annotation when None.""" - tool_call = MCPToolCall(name="click", arguments={}) - data = tool_call.model_dump(exclude_none=True) - assert "annotation" not in data - - -def test_mcp_tool_call_annotation_defaults_to_none(): - """MCPToolCall without explicit annotation defaults to None.""" - tool_call = MCPToolCall(name="click", arguments={"x": 1}) - assert tool_call.annotation is None - - -def test_mcp_tool_call_str_with_annotation(): - """__str__ appends annotation comment when set.""" - tool_call = MCPToolCall(name="click", arguments={"x": 1}, annotation="Open the sidebar") - result = str(tool_call) - assert result.endswith(" # Open the sidebar") - assert "click" in result - - -def test_mcp_tool_call_str_without_annotation(): - """__str__ has no annotation comment when annotation is None.""" - tool_call = MCPToolCall(name="click", arguments={"x": 1}) - result = str(tool_call) - assert "#" not in result - - -def test_mcp_tool_call_rich_with_annotation(): - """__rich__ includes escaped annotation in bright_black markup.""" - with patch("hud.utils.hud_console.hud_console") as mock_console: - mock_console.format_tool_call.return_value = "formatted" - tool_call = MCPToolCall(name="test", arguments={}, annotation="has [brackets] & stuff") - result = tool_call.__rich__() - assert "[bright_black]" in result - assert "has \\[brackets] & stuff" in result - - -def test_mcp_tool_result_text_content(): - """Test MCPToolResult with text content.""" - result = MCPToolResult( - content=[TextContent(text="Test output", type="text")], - isError=False, - ) - assert "Test output" in str(result) - assert "✓" in str(result) - - -def test_mcp_tool_result_multiline_text(): - """Test MCPToolResult with multiline text uses first line.""" - result = MCPToolResult( - content=[TextContent(text="First line\nSecond line\nThird line", type="text")], - isError=False, - ) - assert "First line" in result._get_content_summary() - assert "Second line" not in result._get_content_summary() - - -def test_mcp_tool_result_image_content(): - """Test MCPToolResult with image content.""" - result = MCPToolResult( - content=[ImageContent(data="base64data", mimeType="image/png", type="image")], - isError=False, - ) - summary = result._get_content_summary() - assert "Image" in summary or "📷" in summary - - -def test_mcp_tool_result_structured_content(): - """Test MCPToolResult with structured content.""" - result = MCPToolResult( - content=[], - structuredContent={"key": "value", "nested": {"data": 123}}, - isError=False, - ) - summary = result._get_content_summary() - assert "key" in summary - - -def test_mcp_tool_result_structured_content_non_serializable(): - """Test MCPToolResult with non-JSON-serializable structured content.""" - result = MCPToolResult( - content=[], - structuredContent={"func": lambda x: x}, - isError=False, - ) - summary = result._get_content_summary() - assert summary # Should have some string representation - - -def test_mcp_tool_result_error(): - """Test MCPToolResult when isError is True.""" - result = MCPToolResult( - content=[TextContent(text="Error message", type="text")], - isError=True, - ) - assert "✗" in str(result) - - -def test_mcp_tool_result_rich(): - """Test MCPToolResult __rich__ calls hud_console.""" - with patch("hud.utils.hud_console.hud_console") as mock_console: - mock_console.format_tool_result.return_value = "formatted" - result = MCPToolResult( - content=[TextContent(text="Test", type="text")], - isError=False, - ) - rich_output = result.__rich__() - assert rich_output == "formatted" - mock_console.format_tool_result.assert_called_once() diff --git a/hud/tests/test_version.py b/hud/tests/test_version.py deleted file mode 100644 index c569e4d2a..000000000 --- a/hud/tests/test_version.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import annotations - -from importlib.metadata import version - - -def test_import(): - """Test that the package can be imported.""" - import hud - - assert hud.__version__ == version("hud") diff --git a/hud/train/tests/test_client.py b/hud/train/tests/test_client.py deleted file mode 100644 index e166a8bff..000000000 --- a/hud/train/tests/test_client.py +++ /dev/null @@ -1,112 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Any - -import pytest - -from hud.eval.run import Run -from hud.train import TrainingClient -from hud.utils.gateway import list_gateway_models - -_MODEL_ID = "00000000-0000-0000-0000-000000000001" - - -async def test_training_rejects_timed_out_run_without_explicit_reward() -> None: - run = Run.failed("rollout timed out") - run.trace.stop_reason = "timeout" - - with pytest.raises(ValueError, match="explicit training reward"): - await TrainingClient("test-model").forward_backward([run], group_size=1) - - -async def test_training_rejects_incomplete_run_groups() -> None: - runs = [Run(None, "", {}) for _ in range(4)] - for index, run in enumerate(runs): - run.trace.trace_id = str(index) - run.group_id = "a" if index < 3 else "b" - - with pytest.raises(ValueError, match="incomplete GRPO groups"): - await TrainingClient("test-model").forward_backward(runs, group_size=2) - - -async def test_slug_resolves_through_the_clients_own_credentials( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr("hud.settings.settings.api_key", None) - list_gateway_models.cache_clear() - seen: list[tuple[str, str | None]] = [] - - def catalog(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - seen.append((url, kwargs.get("api_key"))) - return { - "items": [{"id": _MODEL_ID, "model_name": "owner/model"}], - "total": 1, - } - - monkeypatch.setattr("hud.utils.platform.make_request_sync", catalog) - client = TrainingClient("owner/model", api_key="team-key", api_url="https://api.example") - try: - url = await client._train_url("optim_step") - finally: - list_gateway_models.cache_clear() - assert url.startswith(f"{client._base_url}/v1/models/{_MODEL_ID}/") - assert seen and seen[0][0].startswith("https://api.example/v2/models") - assert seen[0][1] == "team-key" - - -def _grpo_runs(groups: list[str], rewards: list[float]) -> list[Run]: - runs = [Run(None, "", {}) for _ in rewards] - for index, (run, group_id, reward) in enumerate(zip(runs, groups, rewards, strict=True)): - run.trace.trace_id = str(index) - run.group_id = group_id - run.grade.reward = reward - return runs - - -async def _forward_backward(runs: list[Run], monkeypatch: pytest.MonkeyPatch) -> None: - """Run forward_backward against a stubbed training service (a UUID model id - resolves without a catalog lookup).""" - - async def fake_request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - return {"metrics": {}, "num_datums": len(runs)} - - monkeypatch.setattr("hud.train.base.make_request", fake_request) - await TrainingClient(_MODEL_ID).forward_backward(runs, group_size=2) - - -async def test_training_warns_when_no_group_has_reward_spread( - caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch -) -> None: - # Identical rewards within every group zero every GRPO advantage, so the - # pass accumulates nothing and the later optim_step fails confusingly. - runs = _grpo_runs(["a", "a", "b", "b"], [1.0, 1.0, 0.0, 0.0]) - - with caplog.at_level(logging.WARNING, logger="hud.train.client"): - await _forward_backward(runs, monkeypatch) - - assert "no gradient" in caplog.text - - -async def test_training_groups_rewards_by_group_id_not_position( - caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch -) -> None: - # An interleaved batch is valid (groups are keyed by group_id), and its - # positional slices have spread even when the real groups have none. - runs = _grpo_runs(["a", "b", "a", "b"], [1.0, 0.0, 1.0, 0.0]) - - with caplog.at_level(logging.WARNING, logger="hud.train.client"): - await _forward_backward(runs, monkeypatch) - - assert "no gradient" in caplog.text - - -async def test_training_is_quiet_when_a_group_has_reward_spread( - caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch -) -> None: - runs = _grpo_runs(["a", "a", "b", "b"], [0.0, 1.0, 0.0, 1.0]) - - with caplog.at_level(logging.WARNING, logger="hud.train.client"): - await _forward_backward(runs, monkeypatch) - - assert "no gradient" not in caplog.text diff --git a/hud/utils/modules.py b/hud/utils/modules.py index 585ffef38..0f3ffd922 100644 --- a/hud/utils/modules.py +++ b/hud/utils/modules.py @@ -30,7 +30,7 @@ def _module_name(file: Path) -> tuple[str, Path]: directory = directory.parent if file.parent != directory: package_root = directory / parts[0] - for name, module in tuple(sys.modules.items()): + for name, module in sys.modules.copy().items(): if name != parts[0] and not name.startswith(f"{parts[0]}."): continue locations = (*getattr(module, "__path__", ()), getattr(module, "__file__", None)) diff --git a/hud/utils/process.py b/hud/utils/process.py index bbda6db86..6962f1e36 100644 --- a/hud/utils/process.py +++ b/hud/utils/process.py @@ -316,7 +316,8 @@ async def _terminate_process_group( term_deadline = loop.time() + term_timeout + settle_time try: os.killpg(proc.pid, signal.SIGTERM) - except ProcessLookupError: + # macOS answers EPERM, not ESRCH, for a group whose leader exited but is not yet reaped. + except (ProcessLookupError, PermissionError): if proc.returncode is None: with contextlib.suppress(TimeoutError): await asyncio.wait_for(proc.wait(), term_timeout) diff --git a/hud/utils/tests/test_exceptions.py b/hud/utils/tests/test_exceptions.py deleted file mode 100644 index 96dae8d95..000000000 --- a/hud/utils/tests/test_exceptions.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Tests for the HUD SDK exception hierarchy.""" - -from __future__ import annotations - -import httpx - -from hud.utils.exceptions import ( - HudAuthenticationError, - HudException, - HudRequestError, -) -from hud.utils.hints import ( - HUD_API_KEY_MISSING, - PRO_PLAN_REQUIRED, - RATE_LIMIT_HIT, -) - - -class TestHudException: - def test_message_and_str(self): - error = HudException("Something broke") - assert error.message == "Something broke" - assert str(error) == "Something broke" - assert error.hints == [] - - def test_response_json_in_str(self): - error = HudException("Bad payload", response_json={"detail": "nope"}) - assert str(error) == "Bad payload | Response: {'detail': 'nope'}" - - def test_subclass_default_hints(self): - error = HudAuthenticationError("API key missing") - assert error.hints == [HUD_API_KEY_MISSING] - assert error.hints[0].title == "HUD API key required" - # Hint copy evolved; keep the assertion robust to minor copy changes - tips = error.hints[0].tips - assert tips and "Set HUD_API_KEY" in tips[0] - - -class TestHudRequestError: - """Test HudRequestError specific behavior.""" - - def test_401_adds_auth_hint(self): - error = HudRequestError("Unauthorized", status_code=401) - assert HUD_API_KEY_MISSING in error.hints - - def test_403_adds_auth_hint(self): - error = HudRequestError("Forbidden", status_code=403) - assert HUD_API_KEY_MISSING in error.hints - - def test_403_pro_plan_message_sets_pro_hint(self): - """403 with Pro wording should map to PRO_PLAN_REQUIRED, not auth.""" - error = HudRequestError("Feature requires Pro plan", status_code=403) - assert PRO_PLAN_REQUIRED in error.hints - assert HUD_API_KEY_MISSING not in error.hints - - def test_403_pro_plan_detail_sets_pro_hint(self): - """403 with detail indicating Pro should map to PRO_PLAN_REQUIRED.""" - error = HudRequestError( - "Forbidden", - status_code=403, - response_json={"detail": "Requires Pro plan"}, - ) - assert PRO_PLAN_REQUIRED in error.hints - assert HUD_API_KEY_MISSING not in error.hints - - def test_429_adds_rate_limit_hint(self): - error = HudRequestError("Too Many Requests", status_code=429) - assert RATE_LIMIT_HIT in error.hints - - def test_other_status_no_default_hints(self): - error = HudRequestError("Server Error", status_code=500) - assert error.hints == [] - - def test_explicit_hints_override_defaults(self): - from hud.utils.hints import Hint - - custom_hint = Hint(title="Custom Error", message="This is a custom hint") - error = HudRequestError("Unauthorized", status_code=401, hints=[custom_hint]) - assert error.hints == [custom_hint] - assert HUD_API_KEY_MISSING not in error.hints - - def test_from_httpx_error(self): - request = httpx.Request("GET", "https://api.test.com") - response = httpx.Response(404, json={"detail": "Not found"}, request=request) - httpx_error = httpx.HTTPStatusError("Not found", request=request, response=response) - - error = HudRequestError.from_httpx_error(httpx_error, context="Testing") - - assert error.status_code == 404 - assert "Testing" in str(error) - assert "Not found" in str(error) - assert error.response_json == {"detail": "Not found"} - - def test_string_representation(self): - error = HudRequestError( - "Request failed", status_code=404, response_json={"error": "Not found"} - ) - - error_str = str(error) - assert "Request failed" in error_str - assert "Status: 404" in error_str - assert "Response JSON: {'error': 'Not found'}" in error_str diff --git a/hud/utils/tests/test_gateway.py b/hud/utils/tests/test_gateway.py deleted file mode 100644 index f452994f6..000000000 --- a/hud/utils/tests/test_gateway.py +++ /dev/null @@ -1,319 +0,0 @@ -from __future__ import annotations - -import asyncio -from functools import partial -from typing import TYPE_CHECKING, cast -from unittest.mock import AsyncMock, MagicMock - -import httpx -import pytest - -from hud.settings import settings -from hud.telemetry.context import set_trace_context -from hud.utils import gateway -from hud.utils.exceptions import HudAuthenticationError - -if TYPE_CHECKING: - from google.genai import Client as GenaiClient - from openai import AsyncOpenAI - - -@pytest.fixture(autouse=True) -def _gateway_settings(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", "sk-hud-test") - monkeypatch.setattr(settings, "hud_gateway_url", "https://gateway.test") - - -@pytest.mark.asyncio -async def test_openai_client_resolves_trace_id_per_request( - monkeypatch: pytest.MonkeyPatch, -) -> None: - seen_headers: dict[str, str] = {} - - async def handler(request: httpx.Request) -> httpx.Response: - trace_id = request.headers["Trace-Id"] - await asyncio.sleep(0) - seen_headers[request.url.path] = trace_id - return httpx.Response(200, json={"object": "list", "data": []}) - - transport = httpx.MockTransport(handler) - real_client_factory = gateway.DefaultAsyncHttpxClient - monkeypatch.setattr( - gateway, - "DefaultAsyncHttpxClient", - partial(real_client_factory, transport=transport), - ) - client = cast("AsyncOpenAI", gateway.build_gateway_client("openai")) - - async def request_in_trace(trace_id: str) -> None: - with set_trace_context(trace_id): - await client.get(f"/models/{trace_id}", cast_to=object) - - try: - await asyncio.gather( - request_in_trace("11111111-1111-4111-8111-111111111111"), - request_in_trace("22222222-2222-4222-8222-222222222222"), - ) - finally: - await client.close() - - assert seen_headers == { - "/models/11111111-1111-4111-8111-111111111111": ("11111111-1111-4111-8111-111111111111"), - "/models/22222222-2222-4222-8222-222222222222": ("22222222-2222-4222-8222-222222222222"), - } - - -@pytest.mark.asyncio -async def test_openai_client_trace_context_overrides_empty_header( - monkeypatch: pytest.MonkeyPatch, -) -> None: - seen_trace_id: str | None = None - - async def handler(request: httpx.Request) -> httpx.Response: - nonlocal seen_trace_id - seen_trace_id = request.headers["Trace-Id"] - return httpx.Response(200, json={"object": "list", "data": []}) - - transport = httpx.MockTransport(handler) - real_client_factory = gateway.DefaultAsyncHttpxClient - monkeypatch.setattr( - gateway, - "DefaultAsyncHttpxClient", - partial(real_client_factory, transport=transport), - ) - client = cast("AsyncOpenAI", gateway.build_gateway_client("openai")) - trace_id = "11111111-1111-4111-8111-111111111111" - - try: - with set_trace_context(trace_id): - await client.get( - "/models/explicit", - cast_to=object, - options={"headers": {"Trace-Id": ""}}, - ) - finally: - await client.close() - - assert seen_trace_id == trace_id - - -@pytest.mark.asyncio -async def test_openai_client_sends_child_and_parent_trace_ids( - monkeypatch: pytest.MonkeyPatch, -) -> None: - seen_headers: dict[str, str] = {} - - async def handler(request: httpx.Request) -> httpx.Response: - seen_headers["Trace-Id"] = request.headers["Trace-Id"] - seen_headers["X-HUD-Parent-Trace-Id"] = request.headers["X-HUD-Parent-Trace-Id"] - return httpx.Response(200, json={"object": "list", "data": []}) - - transport = httpx.MockTransport(handler) - real_client_factory = gateway.DefaultAsyncHttpxClient - monkeypatch.setattr( - gateway, - "DefaultAsyncHttpxClient", - partial(real_client_factory, transport=transport), - ) - client = cast("AsyncOpenAI", gateway.build_gateway_client("openai")) - - try: - with set_trace_context("child", parent_trace_id="parent"): - await client.get("/models/nested", cast_to=object) - finally: - await client.close() - - assert seen_headers == { - "Trace-Id": "child", - "X-HUD-Parent-Trace-Id": "parent", - } - - -_BEDROCK_ARN = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/anthropic.claude" - - -def test_list_gateway_models_reads_every_page(monkeypatch: pytest.MonkeyPatch) -> None: - gateway.list_gateway_models.cache_clear() - rows = [{"id": f"m{i}", "model_name": f"m{i}"} for i in range(103)] - - def get(path: str, *, params: dict[str, int]) -> dict[str, object]: - assert path == "/models" - start = params["offset"] - return {"items": rows[start : start + params["limit"]], "total": len(rows)} - - platform = MagicMock() - platform.get.side_effect = get - monkeypatch.setattr(gateway.PlatformClient, "from_settings", lambda: platform) - try: - models = gateway.list_gateway_models() - finally: - gateway.list_gateway_models.cache_clear() - - assert [m.model_name for m in models] == [f"m{i}" for i in range(103)] - assert platform.get.call_count == 2 - - -def test_model_recency_prefers_release_over_catalog_date() -> None: - released = gateway.GatewayModelInfo( - id="released", created_at="2026-09-01T00:00:00Z", released_at="2026-01-01T00:00:00Z" - ) - added = gateway.GatewayModelInfo(id="added", created_at="2026-06-01T00:00:00Z") - undated = gateway.GatewayModelInfo(id="undated") - assert sorted([added, released, undated], key=lambda m: m.recency) == [ - undated, - released, - added, - ] - - -def test_bedrock_arn_model_gets_a_bedrock_client(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "aws_access_key_id", "AKIATEST") - monkeypatch.setattr(settings, "aws_secret_access_key", "secret") - monkeypatch.setattr(settings, "aws_region", "us-east-1") - bedrock = MagicMock(return_value=object()) - monkeypatch.setattr("anthropic.AsyncAnthropicBedrock", bedrock) - gateway_client = MagicMock() - monkeypatch.setattr(gateway, "build_gateway_client", gateway_client) - - client = gateway.build_model_client("anthropic", model=_BEDROCK_ARN) - - assert client is bedrock.return_value - bedrock.assert_called_once_with( - aws_access_key="AKIATEST", aws_secret_key="secret", aws_region="us-east-1" - ) - gateway_client.assert_not_called() - - -def test_bedrock_arn_model_requires_aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "aws_access_key_id", None) - monkeypatch.setattr(settings, "aws_secret_access_key", None) - monkeypatch.setattr(settings, "aws_region", None) - - with pytest.raises(HudAuthenticationError, match="AWS Bedrock"): - gateway.build_model_client("anthropic", model=_BEDROCK_ARN) - - -def test_bedrock_arn_model_cannot_be_forced_through_the_gateway( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "aws_access_key_id", "AKIATEST") - monkeypatch.setattr(settings, "aws_secret_access_key", "secret") - monkeypatch.setattr(settings, "aws_region", "us-east-1") - bedrock = MagicMock() - monkeypatch.setattr("anthropic.AsyncAnthropicBedrock", bedrock) - - with pytest.raises(ValueError, match="cannot use the HUD gateway"): - gateway.build_model_client("anthropic", model=_BEDROCK_ARN, gateway=True) - bedrock.assert_not_called() - - -def test_provider_key_wins_over_hud_key(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "anthropic_api_key", "sk-ant-test") - direct = MagicMock(return_value=object()) - monkeypatch.setattr("anthropic.AsyncAnthropic", direct) - gateway_client = MagicMock() - monkeypatch.setattr(gateway, "build_gateway_client", gateway_client) - - client = gateway.build_model_client("anthropic", model="claude-sonnet-4-6") - - assert client is direct.return_value - direct.assert_called_once_with(api_key="sk-ant-test") - gateway_client.assert_not_called() - - -def test_hud_key_alone_routes_through_gateway(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "anthropic_api_key", None) - gateway_client = MagicMock(return_value=object()) - monkeypatch.setattr(gateway, "build_gateway_client", gateway_client) - - client = gateway.build_model_client("anthropic", model="claude-sonnet-4-6") - - assert client is gateway_client.return_value - gateway_client.assert_called_once_with("anthropic") - - -def test_no_key_at_all_is_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", None) - monkeypatch.setattr(settings, "openai_api_key", None) - with pytest.raises(HudAuthenticationError, match="No API key for openai"): - gateway.build_model_client("openai") - - -@pytest.mark.asyncio -async def test_anthropic_client_receives_trace_aware_http_client( - monkeypatch: pytest.MonkeyPatch, -) -> None: - client_factory = MagicMock(return_value=object()) - monkeypatch.setattr("anthropic.AsyncAnthropic", client_factory) - - gateway.build_gateway_client("anthropic") - - kwargs = client_factory.call_args.kwargs - http_client = kwargs["http_client"] - assert http_client.event_hooks["request"] - await http_client.aclose() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("initial_status", [200, 429, 503]) -async def test_gemini_async_request_includes_trace_id( - monkeypatch: pytest.MonkeyPatch, - initial_status: int, -) -> None: - sleep = AsyncMock() - monkeypatch.setattr(asyncio, "sleep", sleep) - seen_trace_id: str | None = None - attempts = 0 - - async def handler(request: httpx.Request) -> httpx.Response: - nonlocal seen_trace_id, attempts - seen_trace_id = request.headers["Trace-Id"] - attempts += 1 - if attempts == 1 and initial_status != 200: - return httpx.Response( - initial_status, json={"error": {"code": initial_status, "message": "busy"}} - ) - return httpx.Response( - 200, - json={ - "candidates": [ - { - "content": {"parts": [{"text": "ok"}], "role": "model"}, - "finishReason": "STOP", - "index": 0, - } - ], - "modelVersion": "gemini-test", - "usageMetadata": { - "candidatesTokenCount": 1, - "promptTokenCount": 1, - "totalTokenCount": 2, - }, - }, - ) - - monkeypatch.setattr( - gateway.httpx, - "AsyncHTTPTransport", - MagicMock(return_value=httpx.MockTransport(handler)), - ) - client = cast("GenaiClient", gateway.build_gateway_client("gemini")) - trace_id = "11111111-1111-4111-8111-111111111111" - - try: - with set_trace_context(trace_id): - response = await client.aio.models.generate_content( - model="gemini-test", - contents="hi", - ) - finally: - await client.aio.aclose() - - assert response.text == "ok" - assert seen_trace_id == trace_id - assert attempts == (1 if initial_status == 200 else 2) - if initial_status != 200: - sleep.assert_awaited_once() - assert 10 <= sleep.await_args_list[0].args[0] <= 11 - else: - sleep.assert_not_awaited() diff --git a/hud/utils/tests/test_hints.py b/hud/utils/tests/test_hints.py deleted file mode 100644 index 94a07e264..000000000 --- a/hud/utils/tests/test_hints.py +++ /dev/null @@ -1,167 +0,0 @@ -from __future__ import annotations - -from unittest.mock import MagicMock, patch - -from hud.utils.hints import ( - CLIENT_NOT_INITIALIZED, - ENV_VAR_MISSING, - HUD_API_KEY_MISSING, - INVALID_CONFIG, - MCP_SERVER_ERROR, - RATE_LIMIT_HIT, - TOOL_NOT_FOUND, - Hint, - render_hints, -) - - -def test_hint_objects_basic(): - assert HUD_API_KEY_MISSING.title and isinstance(HUD_API_KEY_MISSING.tips, list) - assert RATE_LIMIT_HIT.code == "RATE_LIMIT" - assert TOOL_NOT_FOUND.title.startswith("Tool") - assert CLIENT_NOT_INITIALIZED.message - assert ENV_VAR_MISSING.command_examples is not None - - -def test_all_hint_constants(): - """Test that all predefined hint constants have required fields.""" - hints = [ - HUD_API_KEY_MISSING, - RATE_LIMIT_HIT, - TOOL_NOT_FOUND, - CLIENT_NOT_INITIALIZED, - INVALID_CONFIG, - ENV_VAR_MISSING, - MCP_SERVER_ERROR, - ] - - for hint in hints: - assert hint.title - assert hint.message - assert hint.code - - -def test_hint_creation(): - """Test creating a custom Hint.""" - hint = Hint( - title="Test Hint", - message="This is a test", - tips=["Tip 1", "Tip 2"], - docs_url="https://example.com", - command_examples=["command 1"], - code="TEST_CODE", - context=["test", "custom"], - ) - - assert hint.title == "Test Hint" - assert hint.message == "This is a test" - assert hint.tips and len(hint.tips) == 2 - assert hint.docs_url == "https://example.com" - assert hint.command_examples and len(hint.command_examples) == 1 - assert hint.code == "TEST_CODE" - assert hint.context and "test" in hint.context - - -def test_hint_minimal(): - """Test creating a minimal Hint with only required fields.""" - hint = Hint(title="Minimal", message="Just basics") - - assert hint.title == "Minimal" - assert hint.message == "Just basics" - assert hint.tips is None - assert hint.docs_url is None - assert hint.command_examples is None - assert hint.code is None - assert hint.context is None - - -def test_render_hints_none(): - """Test that render_hints handles None gracefully.""" - # Should not raise - render_hints(None) - - -def test_render_hints_empty_list(): - """Test that render_hints handles empty list gracefully.""" - # Should not raise - render_hints([]) - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_with_tips(mock_console): - """Test rendering hints with tips.""" - render_hints([HUD_API_KEY_MISSING]) - - # Should call warning for title/message - mock_console.warning.assert_called() - # Should call info for tips - assert mock_console.info.call_count >= 1 - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_with_command_examples(mock_console): - """Test rendering hints with command examples.""" - render_hints([ENV_VAR_MISSING]) - - # Should call command_example - mock_console.command_example.assert_called() - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_with_docs_url(mock_console): - """Test rendering hints with documentation URL.""" - hint = Hint( - title="Test", - message="Test message", - docs_url="https://docs.example.com", - ) - - render_hints([hint]) - - # Should call link for docs URL - mock_console.link.assert_called_with("https://docs.example.com") - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_same_title_and_message(mock_console): - """Test rendering hints when title equals message.""" - hint = Hint(title="Same", message="Same") - - render_hints([hint]) - - # Should only call warning once with just the message - mock_console.warning.assert_called_once_with("Same") - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_different_title_and_message(mock_console): - """Test rendering hints when title differs from message.""" - hint = Hint(title="Title", message="Different message") - - render_hints([hint]) - - # Should call warning with both title and message - mock_console.warning.assert_called_once() - call_args = mock_console.warning.call_args[0][0] - assert "Title" in call_args - assert "Different message" in call_args - - -def test_render_hints_with_custom_design(): - """Test rendering hints with custom design object.""" - custom_design = MagicMock() - - hint = Hint(title="Test", message="Message") - # Should not raise when custom design is provided - render_hints([hint], design=custom_design) - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_handles_exception(mock_console): - """Test that render_hints handles exceptions gracefully.""" - mock_console.warning.side_effect = Exception("Test error") - - hint = Hint(title="Test", message="Message") - - # Should not raise, just log warning - render_hints([hint]) diff --git a/hud/utils/tests/test_hud_console.py b/hud/utils/tests/test_hud_console.py deleted file mode 100644 index b9ea2dcc4..000000000 --- a/hud/utils/tests/test_hud_console.py +++ /dev/null @@ -1,165 +0,0 @@ -"""``HUDConsole`` — smoke-exercise the output methods + check the pure formatters. - -These mostly assert "doesn't raise" (output goes to a Rich console), which still -exercises the formatting branches; the ``format_*`` helpers return values -we can assert directly. -""" - -from __future__ import annotations - -import logging -from typing import Any - -import pytest -import typer -from prompt_toolkit.application import create_app_session -from prompt_toolkit.input.defaults import create_pipe_input -from prompt_toolkit.key_binding import KeyBindings -from prompt_toolkit.output import DummyOutput - -from hud.utils.hud_console import HUDConsole - - -def _select_with_keystrokes( - keys: str, choices: list[str | dict[str, Any]], *, spaced: bool = False -) -> str: - """Drive ``HUDConsole.select`` with scripted keystrokes through a prompt_toolkit - pipe input (no real TTY), returning the selected value.""" - with create_pipe_input() as inp: - inp.send_text(keys) - with create_app_session(input=inp, output=DummyOutput()): - return HUDConsole().select("pick", choices, spaced=spaced) - - -def test_output_methods_do_not_raise() -> None: - c = HUDConsole() - c.header("Title") - c.section_title("Section") - c.success("ok") - c.error("bad") - c.warning("warn") - c.info("info") - c.print("plain") - c.dim_info("key", "value") - c.link("https://example.com") - c.progress_message("working") - c.hint("a hint") - c.status_item("label", "value") - c.command_example("hud eval tasks.json") - c.key_value_table({"key": "value"}) - c.render_support_hint() - - -def test_debug_respects_logger_level() -> None: - logger = logging.getLogger("test_hud_console_debug") - c = HUDConsole(logger=logger) - logger.setLevel(logging.DEBUG) - c.debug("debug visible") - logger.setLevel(logging.WARNING) - c.debug("debug hidden") # no-op when not verbose - - -def test_format_helpers_return_strings() -> None: - c = HUDConsole() - assert isinstance(c.format_tool_call("bash", {"command": "ls"}), str) - assert isinstance(c.format_tool_result("output text"), str) - assert isinstance(c.format_tool_result("error text", is_error=True), str) - - -def test_render_exception_does_not_raise() -> None: - c = HUDConsole() - try: - raise ValueError("boom") - except ValueError as exc: - c.render_exception(exc) - - -def test_render_exception_request_error_details() -> None: - from hud.utils.exceptions import HudRequestError - - c = HUDConsole() - c.render_exception(HudRequestError("nope", status_code=403, response_text="forbidden")) - - -def _capture_select_choices(monkeypatch: pytest.MonkeyPatch, returns: str) -> dict[str, Any]: - """Patch the ``questionary`` boundary so ``HUDConsole.select`` runs without a - TTY, recording the choice list it builds.""" - import questionary - - captured: dict[str, Any] = {} - - class _App: - def __init__(self) -> None: - self.key_bindings = KeyBindings() - - class _Stub: - def __init__(self) -> None: - self.application = _App() - - def ask(self) -> str: - return returns - - def _fake_select(message: str, **kwargs: Any) -> _Stub: - captured["choices"] = kwargs["choices"] - return _Stub() - - monkeypatch.setattr(questionary, "select", _fake_select) - return captured - - -def test_select_spaced_interleaves_blank_separators(monkeypatch: pytest.MonkeyPatch) -> None: - from questionary import Choice, Separator - - captured = _capture_select_choices(monkeypatch, returns="b") - - result = HUDConsole().select( - "pick", - [ - {"name": "A", "value": "a"}, - {"name": "B", "value": "b"}, - {"name": "C", "value": "c"}, - ], - spaced=True, - ) - - assert result == "b" - choices = captured["choices"] - # A blank separator sits between consecutive choices, but never leads or trails. - # (questionary's Separator subclasses Choice, so compare exact types here.) - assert [type(x).__name__ for x in choices] == [ - "Choice", - "Separator", - "Choice", - "Separator", - "Choice", - ] - assert sum(type(x) is Choice for x in choices) == 3 - assert sum(isinstance(x, Separator) for x in choices) == 2 - - -def test_select_without_spaced_has_no_separators(monkeypatch: pytest.MonkeyPatch) -> None: - from questionary import Separator - - captured = _capture_select_choices(monkeypatch, returns="a") - - HUDConsole().select("pick", [{"name": "A", "value": "a"}, {"name": "B", "value": "b"}]) - - assert not any(isinstance(x, Separator) for x in captured["choices"]) - - -def test_select_escape_cancels() -> None: - # questionary cancels on Ctrl+C only; we additionally bind Esc to cancel, - # which surfaces as a typer.Exit through select's None handling. - with pytest.raises(typer.Exit): - _select_with_keystrokes("\x1b", [{"name": "A", "value": "a"}, {"name": "B", "value": "b"}]) - - -def test_select_arrow_then_enter_selects_skipping_separators() -> None: - # Down + Enter lands on the second real choice; the blank separator between - # spaced choices is skipped, and the Esc binding doesn't break arrow keys. - result = _select_with_keystrokes( - "\x1b[B\r", - [{"name": "A", "value": "a"}, {"name": "B", "value": "b"}, {"name": "C", "value": "c"}], - spaced=True, - ) - assert result == "b" diff --git a/hud/utils/tests/test_naming.py b/hud/utils/tests/test_naming.py deleted file mode 100644 index 5edf49344..000000000 --- a/hud/utils/tests/test_naming.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations - -from hud.utils.naming import normalize_environment_name - - -def test_normalize_environment_name() -> None: - assert normalize_environment_name("terminal-bench") == "terminal-bench" - assert normalize_environment_name("My Cool_Bench") == "my-cool-bench" - assert normalize_environment_name("bench@2.0!") == "bench20" - assert normalize_environment_name("--hello--") == "hello" - assert normalize_environment_name("a---b") == "a-b" - assert normalize_environment_name("@#$") == "environment" - assert normalize_environment_name("", default="converted") == "converted" diff --git a/hud/utils/tests/test_platform.py b/hud/utils/tests/test_platform.py deleted file mode 100644 index f767b31cb..000000000 --- a/hud/utils/tests/test_platform.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Generic platform transport in ``hud.utils.platform``.""" - -from __future__ import annotations - -from typing import Any - -import pytest - -from hud.utils.exceptions import HudAuthenticationError -from hud.utils.platform import PlatformClient - - -def test_url_prefixes_version_segment_and_joins_params() -> None: - """Feature modules pass version-free paths; the client prepends the - canonical ``/v2`` namespace (the default ``api_prefix``).""" - platform = PlatformClient("https://api.example/", "key") - - assert platform.base_url == "https://api.example/v2" - assert platform.url("/tasks/upload") == "https://api.example/v2/tasks/upload" - assert platform.url("/registry/envs", {"limit": 5}) == ( - "https://api.example/v2/registry/envs?limit=5" - ) - - -def test_url_encodes_repeated_query_parameters() -> None: - """List-valued FastAPI query parameters are emitted as repeated keys.""" - platform = PlatformClient("https://api.example", "key") - - assert platform.url( - "/qa-agents/results/resources", - {"subject_type": "taskset", "subject_ids": ["first", "second"]}, - ) == ( - "https://api.example/v2/qa-agents/results/resources?" - "subject_type=taskset&subject_ids=first&subject_ids=second" - ) - - -def test_http_methods_route_through_shared_requests(monkeypatch: pytest.MonkeyPatch) -> None: - calls: list[dict[str, object]] = [] - - def fake_request( - method: str, url: str, json: object = None, **kwargs: object - ) -> dict[str, Any]: - calls.append({"method": method, "url": url, "json": json, "api_key": kwargs.get("api_key")}) - return {"ok": True} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - platform = PlatformClient("https://api.example", "key") - - assert platform.get("/x", params={"a": 1}) == {"ok": True} - assert platform.post("/y", json={"b": 2}) == {"ok": True} - assert platform.patch("/z", json={"c": 3}) == {"ok": True} - assert platform.delete("/z") == {"ok": True} - assert calls == [ - {"method": "GET", "url": "https://api.example/v2/x?a=1", "json": None, "api_key": "key"}, - {"method": "POST", "url": "https://api.example/v2/y", "json": {"b": 2}, "api_key": "key"}, - {"method": "PATCH", "url": "https://api.example/v2/z", "json": {"c": 3}, "api_key": "key"}, - {"method": "DELETE", "url": "https://api.example/v2/z", "json": None, "api_key": "key"}, - ] - - -def test_requests_without_api_key_raise_authentication_error() -> None: - platform = PlatformClient("https://api.example", "") - - with pytest.raises(HudAuthenticationError): - platform.get("/tasks") - - -def test_from_settings_prepends_canonical_version(monkeypatch: pytest.MonkeyPatch) -> None: - from hud import settings as settings_module - - monkeypatch.setattr(settings_module.settings, "hud_api_url", "https://api.example") - monkeypatch.setattr(settings_module.settings, "api_key", "key") - - platform = PlatformClient.from_settings() - assert platform.url("/tasks/upload") == "https://api.example/v2/tasks/upload" - - -@pytest.mark.parametrize("api_key", [None, ""]) -def test_from_settings_requires_api_key( - monkeypatch: pytest.MonkeyPatch, api_key: str | None -) -> None: - from hud import settings as settings_module - - monkeypatch.setattr(settings_module.settings, "api_key", api_key) - - with pytest.raises(HudAuthenticationError, match="HUD_API_KEY is required"): - PlatformClient.from_settings() diff --git a/hud/utils/tests/test_process.py b/hud/utils/tests/test_process.py deleted file mode 100644 index 1fdbca0bc..000000000 --- a/hud/utils/tests/test_process.py +++ /dev/null @@ -1,148 +0,0 @@ -"""What a managed process group reports when it ends in each of its ways.""" - -from __future__ import annotations - -import asyncio -import os -import shlex -import signal -import sys -from io import StringIO -from types import SimpleNamespace -from typing import cast - -import pytest - -from hud.utils.process import ProcessGroup, ProcessResult, create_process_group_exec, stream_output - -pytestmark = [ - pytest.mark.asyncio, - pytest.mark.skipif(sys.platform == "win32", reason="POSIX process groups"), -] - - -async def _run(script: str, max_wait: float | None = None) -> ProcessResult: - group = await create_process_group_exec( - "sh", - "-c", - script, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - return await group.complete(max_wait=max_wait) - - -async def test_a_finished_process_is_not_a_timeout_because_a_child_holds_its_pipes() -> None: - """Starting a service and exiting is how a verifier ends: it has already - written its verdict. Waiting on the inherited pipes instead of on the - process reports that as a timeout, and scores a passing task zero.""" - result = await _run("echo verdict-written; sleep 30 & exit 0", max_wait=10) - - assert result.timed_out is False - assert result.returncode == 0 - assert b"verdict-written" in result.stdout - - -async def test_a_process_that_overran_keeps_what_it_printed() -> None: - """The output is the evidence of how far it got; reporting only that the - deadline passed throws away the one thing that explains the timeout.""" - result = await _run("echo progress-so-far; sleep 30", max_wait=1) - - assert result.timed_out is True - assert b"progress-so-far" in result.stdout - - -async def test_a_chatty_process_does_not_block_on_a_full_pipe() -> None: - """Reading only after exit deadlocks once the pipe buffer fills (~64KB).""" - result = await asyncio.wait_for(_run("yes hud | head -c 500000; exit 0", max_wait=30), 30) - - assert result.timed_out is False - assert len(result.stdout) == 500000 - - -async def test_stream_output_drains_lines_larger_than_the_reader_limit() -> None: - source = asyncio.StreamReader() - source.feed_data(b"x" * 100_000) - source.feed_eof() - output = StringIO() - - await stream_output(source, output) - - assert output.getvalue() == "x" * 100_000 - - -async def test_stream_output_preserves_utf8_split_across_chunks() -> None: - async def chunks(): - yield b"price: \xe2" - yield b"\x82\xac10" - - output = StringIO() - - await stream_output(chunks(), output) - - assert output.getvalue() == "price: €10" - - -async def test_stream_output_keeps_draining_after_sink_failure() -> None: - class FailedOutput(StringIO): - def write(self, value: str) -> int: - raise OSError("terminal closed") - - source = asyncio.StreamReader() - source.feed_data(b"x" * 200_000) - source.feed_eof() - - await stream_output(source, FailedOutput()) - - assert source.at_eof() - - -async def test_a_child_outside_the_group_cannot_hold_completion_open() -> None: - child = shlex.quote(sys.executable) - result = await asyncio.wait_for( - _run(f"{child} -c 'import os, time; os.setsid(); time.sleep(2)' & echo retained"), - 1, - ) - - assert result.returncode == 0 - assert result.stdout == b"retained\n" - - -async def test_a_cancelled_call_leaves_nothing_running() -> None: - """A cancelled rollout unwinds through here, and the group is this call's - to release however it exits.""" - group = await create_process_group_exec( - "sh", - "-c", - "sleep 30", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - task = asyncio.create_task(group.complete()) - await asyncio.sleep(0.2) - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - - assert group.returncode is not None - - -async def test_teardown_survives_a_process_group_becoming_unsignalable( - monkeypatch: pytest.MonkeyPatch, -) -> None: - signals: list[int] = [] - - def killpg(_process_group: int, sent: int) -> None: - signals.append(sent) - if sent in (0, signal.SIGKILL): - raise PermissionError - - monkeypatch.setattr(os, "killpg", killpg) - process = cast( - "asyncio.subprocess.Process", - SimpleNamespace(pid=1234, returncode=0), - ) - - await ProcessGroup(process, term_timeout=0).terminate() - - assert signals == [signal.SIGTERM, 0, signal.SIGKILL] diff --git a/hud/utils/tests/test_requests.py b/hud/utils/tests/test_requests.py deleted file mode 100644 index 08b95fb4e..000000000 --- a/hud/utils/tests/test_requests.py +++ /dev/null @@ -1,337 +0,0 @@ -"""Tests for the HTTP request utilities in the HUD API.""" - -from __future__ import annotations - -import uuid -from http import HTTPStatus -from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, Mock, patch - -import httpx -import pytest - -from hud.utils.exceptions import ( - HudAuthenticationError, - HudDeprecationWarning, - HudNetworkError, - HudRequestError, - HudTimeoutError, -) -from hud.utils.requests import ( - make_request, - make_request_sync, -) - -if TYPE_CHECKING: - from collections.abc import Callable - - -@pytest.mark.parametrize("asynchronous", [False, True]) -async def test_requests_retry_transient_responses_with_shared_backoff(asynchronous): - calls = 0 - - def handle(request): - nonlocal calls - calls += 1 - return httpx.Response(503 if calls < 3 else 200, json={"ok": True}) - - transport = httpx.MockTransport(handle) - with ( - patch("hud.utils.requests.time.sleep") as sync_sleep, - patch("hud.utils.requests.asyncio.sleep", new_callable=AsyncMock) as async_sleep, - ): - if asynchronous: - async with httpx.AsyncClient(transport=transport) as client: - result = await make_request( - "GET", "https://test/data", api_key="key", client=client - ) - else: - with httpx.Client(transport=transport) as client: - result = make_request_sync("GET", "https://test/data", api_key="key", client=client) - sleep = async_sleep if asynchronous else sync_sleep - assert [call.args[0] for call in sleep.call_args_list] == [2.0, 4.0] - assert result == {"ok": True} - assert calls == 3 - - -@pytest.mark.parametrize("asynchronous", [False, True]) -@pytest.mark.parametrize("status", [402, 429]) -async def test_requests_preserve_status_hints_without_retry(asynchronous, status): - from hud.utils.hints import CREDITS_EXHAUSTED, RATE_LIMIT_HIT - - transport = httpx.MockTransport(lambda request: httpx.Response(status, json={"detail": "stop"})) - with pytest.raises(HudRequestError) as error: - if asynchronous: - async with httpx.AsyncClient(transport=transport) as client: - await make_request("GET", "https://test/data", api_key="key", client=client) - else: - with httpx.Client(transport=transport) as client: - make_request_sync("GET", "https://test/data", api_key="key", client=client) - assert error.value.hints == [CREDITS_EXHAUSTED if status == 402 else RATE_LIMIT_HIT] - - -def test_requests_warn_once_per_deprecation_notice(): - successor = f"https://api.test/v2/{uuid.uuid4()}" - headers = { - "Deprecation": "@1790985600", - "Sunset": "Sat, 10 Oct 2026 00:00:00 GMT", - "Link": f'<{successor}>; rel="successor-version"', - } - transport = httpx.MockTransport(lambda request: httpx.Response(200, json={}, headers=headers)) - with httpx.Client(transport=transport) as client, pytest.warns(HudDeprecationWarning) as caught: - for url in ("https://api.test/v2/old/1", "https://api.test/v2/old/2"): - make_request_sync("GET", url, api_key="key", client=client) - - assert [str(warning.message) for warning in caught] == [ - "GET /v2/old/1 is deprecated by the HUD API and stops being served on " - f"Sat, 10 Oct 2026 00:00:00 GMT; its replacement is {successor}. " - "If a hud command printed this, upgrade hud." - ] - - -def _create_mock_response( - status_code: int = 200, - json_data: dict[str, Any] | None = None, - raise_exception: Exception | None = None, -) -> Callable[[httpx.Request], httpx.Response]: - """Create a mock response handler for httpx.MockTransport.""" - - def handler(request: httpx.Request) -> httpx.Response: - if "Authorization" not in request.headers: - return httpx.Response(HTTPStatus.UNAUTHORIZED, json={"error": "Unauthorized"}) - - if raise_exception: - raise raise_exception - - return httpx.Response(status_code, json=json_data or {"result": "success"}, request=request) - - return handler - - -@pytest.mark.asyncio -async def test_make_request_success(): - """Test successful async request.""" - expected_data = {"id": "123", "name": "test"} - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(200, expected_data)) - ) - result = await make_request( - "GET", "https://api.test.com/data", api_key="test-key", client=async_client - ) - assert result == expected_data - - -@pytest.mark.asyncio -async def test_make_request_no_api_key(): - """Test request without API key.""" - with pytest.raises(HudAuthenticationError): - await make_request("GET", "https://api.test.com/data", api_key=None) - - -@pytest.mark.asyncio -async def test_make_request_http_error(): - """Test HTTP error handling.""" - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(404, {"error": "Not found"})) - ) - - with pytest.raises(HudRequestError) as excinfo: - await make_request( - "GET", "https://api.test.com/data", api_key="test-key", client=async_client - ) - - assert "404" in str(excinfo.value) - - -@pytest.mark.asyncio -async def test_make_request_network_error(): - """Test network error handling with retry exhaustion.""" - request_error = httpx.RequestError( - "Connection error", request=httpx.Request("GET", "https://api.test.com") - ) - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(raise_exception=request_error)) - ) - - with patch("hud.utils.requests.asyncio.sleep", AsyncMock()) as mock_retry: - mock_retry.return_value = None - - with pytest.raises(HudNetworkError) as excinfo: - await make_request( - "GET", - "https://api.test.com/data", - api_key="test-key", - max_retries=2, - retry_delay=0.01, - client=async_client, - ) - - assert "Connection error" in str(excinfo.value) - - -@pytest.mark.asyncio -async def test_make_request_timeout(): - """Test timeout error handling.""" - timeout_error = httpx.TimeoutException( - "Request timed out", request=httpx.Request("GET", "https://api.test.com") - ) - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(raise_exception=timeout_error)) - ) - - with pytest.raises(HudTimeoutError) as excinfo: - await make_request( - "GET", "https://api.test.com/data", api_key="test-key", client=async_client - ) - - assert "timed out" in str(excinfo.value) - - -@pytest.mark.asyncio -async def test_make_request_unexpected_error(): - """Test handling of unexpected errors.""" - unexpected_error = ValueError("Unexpected error") - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(raise_exception=unexpected_error)) - ) - with pytest.raises(HudRequestError) as excinfo: - await make_request( - "GET", "https://api.test.com/data", api_key="test-key", client=async_client - ) - - assert "Unexpected error" in str(excinfo.value) - - -@pytest.mark.asyncio -async def test_make_request_auto_client_creation(): - """Test automatic client creation when not provided.""" - with patch("hud.utils.requests._create_default_async_client") as mock_create_client: - mock_client = AsyncMock() - mock_client.request.return_value = httpx.Response( - 200, json={"result": "success"}, request=httpx.Request("GET", "https://api.test.com") - ) - mock_client.aclose = AsyncMock() - mock_create_client.return_value = mock_client - - result = await make_request("GET", "https://api.test.com/data", api_key="test-key") - - assert result == {"result": "success"} - mock_client.aclose.assert_awaited_once() - - -def test_make_request_sync_success(): - """Test successful sync request.""" - expected_data = {"id": "123", "name": "test"} - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(200, expected_data)) - ) - - result = make_request_sync( - "GET", "https://api.test.com/data", api_key="test-key", client=sync_client - ) - - assert result == expected_data - - -def test_make_request_sync_no_api_key(): - """Test sync request without API key.""" - with pytest.raises(HudAuthenticationError): - make_request_sync("GET", "https://api.test.com/data", api_key=None) - - -def test_make_request_sync_http_error(): - """Test HTTP error handling.""" - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(404, {"error": "Not found"})) - ) - with pytest.raises(HudRequestError) as excinfo: - make_request_sync( - "GET", "https://api.test.com/data", api_key="test-key", client=sync_client - ) - - assert "404" in str(excinfo.value) - - -def test_make_request_sync_network_error(): - """Test network error handling with retry exhaustion.""" - request_error = httpx.RequestError( - "Connection error", request=httpx.Request("GET", "https://api.test.com") - ) - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(raise_exception=request_error)) - ) - with patch("time.sleep", lambda _: None): - with pytest.raises(HudNetworkError) as excinfo: - make_request_sync( - "GET", - "https://api.test.com/data", - api_key="test-key", - max_retries=2, - retry_delay=0.01, - client=sync_client, - ) - - assert "Connection error" in str(excinfo.value) - - -def test_make_request_sync_timeout(): - """Test timeout error handling.""" - timeout_error = httpx.TimeoutException( - "Request timed out", request=httpx.Request("GET", "https://api.test.com") - ) - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(raise_exception=timeout_error)) - ) - with pytest.raises(HudTimeoutError) as excinfo: - make_request_sync( - "GET", "https://api.test.com/data", api_key="test-key", client=sync_client - ) - - assert "timed out" in str(excinfo.value) - - -def test_make_request_sync_unexpected_error(): - """Test handling of unexpected errors.""" - unexpected_error = ValueError("Unexpected error") - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(raise_exception=unexpected_error)) - ) - - with pytest.raises(HudRequestError) as excinfo: - make_request_sync( - "GET", "https://api.test.com/data", api_key="test-key", client=sync_client - ) - - assert "Unexpected error" in str(excinfo.value) - - -def test_make_request_sync_auto_client_creation(): - """Test automatic client creation when not provided.""" - with patch("hud.utils.requests._create_default_sync_client") as mock_create_client: - mock_client = Mock() - mock_client.request.return_value = httpx.Response( - 200, json={"result": "success"}, request=httpx.Request("GET", "https://api.test.com") - ) - mock_client.close = Mock() - mock_create_client.return_value = mock_client - - result = make_request_sync("GET", "https://api.test.com/data", api_key="test-key") - - assert result == {"result": "success"} - mock_client.close.assert_called_once() - - -def test_make_request_sync_empty_204() -> None: - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(204, request=request) - - sync_client = httpx.Client(transport=httpx.MockTransport(handler)) - assert ( - make_request_sync( - "DELETE", - "https://api.test.com/data", - api_key="test-key", - client=sync_client, - ) - == {} - ) diff --git a/hud/utils/tests/test_serialization.py b/hud/utils/tests/test_serialization.py deleted file mode 100644 index 1dfb92f25..000000000 --- a/hud/utils/tests/test_serialization.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from hud.utils.serialization import json_safe_dict, json_safe_value - - -def test_json_safe_value_serializes_dataclass() -> None: - @dataclass - class Demo: - name: str - count: int - - result = json_safe_value(Demo(name="test", count=2)) - assert result == {"name": "test", "count": 2} - - -def test_json_safe_value_falls_back_for_unserializable_object() -> None: - class Weird: - def __init__(self) -> None: - raise RuntimeError("boom") - - value = Weird.__new__(Weird) - result = json_safe_value(value) - assert isinstance(result, str) - assert "Weird" in result - - -def test_json_safe_dict_serializes_each_value() -> None: - data = {"number": 1, "items": [1, 2, 3]} - assert json_safe_dict(data) == data diff --git a/pyproject.toml b/pyproject.toml index b4ccdbfe5..3a7ad57eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,6 @@ exclude = [ "cookbooks/", "environments/", "hud-python/", - "hud/integrations/harbor/tests/", "**/checkpoints/", "**/*.safetensors", "**/*.ckpt", @@ -96,7 +95,6 @@ include = [ "/pyproject.toml" ] exclude = [ - "*/tests/", "*/__pycache__/", "**/*.safetensors", "**/*.ckpt", @@ -128,7 +126,7 @@ bedrock = [ # Development dependencies - includes testing, linting, and automation tools dev = [ "hud[agents]", # Include agents for dev - "hud[robot]", # the robot end-to-end gates (hud/tests/test_robot.py) run the real wire + "hud[robot]", # the robot scenarios in tests/robot run the real wire "dotenv>=0.9.9", # Testing and linting "ruff==0.16.1", @@ -136,15 +134,14 @@ dev = [ "pytest-asyncio", "pytest-mock", "pytest-cov", + "pytest-xdist", + "pytest-timeout", + "dirty-equals", "ty==0.0.67", ] # Alias for backwards compatibility agent = ["hud[agents]"] -browseruse = [ - "browser-use>=0.11.13", -] - # Robot capability (openpi/0 protocol wire codec + bridges + agent harness) robot = [ "numpy>=1.24", @@ -172,7 +169,7 @@ train = [ [tool.uv] -required-version = "==0.12.2" +required-version = ">=0.12.2" [tool.ruff] target-version = "py311" @@ -230,7 +227,7 @@ docstring-code-format = true runtime-evaluated-base-classes = ["pydantic.BaseModel"] [tool.ty.src] -include = ["hud"] +include = ["hud", "tests"] [tool.ty.rules] blanket-ignore-comment = "error" @@ -244,9 +241,6 @@ strict-equality-semantics = true [tool.coverage.run] source = ["hud"] -omit = [ - "*/tests/*", -] [tool.coverage.report] exclude_lines = [ @@ -263,15 +257,19 @@ exclude_lines = [ ] show_missing = true fail_under = 58 -omit = [ - "*/tests/*", -] [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" -testpaths = ["hud"] -addopts = "-m 'not integration'" +testpaths = ["tests"] +addopts = "-m 'not e2e'" +timeout = 120 markers = [ - "integration: marks tests as integration tests (require HUD_API_KEY, network access, or Docker)", + "e2e: needs something outside the test process; runs only when selected with -m", + "docker: needs a running Docker daemon", + "modal: needs a Modal account (a token in the environment or ~/.modal.toml)", + "lerobot: needs the lerobot[dataset] package", + "sandbox: needs Linux, root, and bubblewrap", + "live: needs a real HUD account (HUD_API_KEY) and spends credits", + "hosted: deploys to and runs on the HUD platform", ] diff --git a/scripts/check_tests.py b/scripts/check_tests.py new file mode 100644 index 000000000..3a96ed24e --- /dev/null +++ b/scripts/check_tests.py @@ -0,0 +1,145 @@ +"""Fail when a test reaches into hud internals. + +Tests drive the SDK through its public boundaries and fake only what lies outside +it (see "Testing" in AGENTS.md). This flags, in test modules outside the support package +(``tests/support``, the one place allowed to configure the SDK): + +- patching a hud name: ``monkeypatch.setattr``, ``patch``, ``patch.object`` or + ``mocker.patch`` aimed at a ``"hud..."`` path or at anything imported from hud; +- importing a private name or module from hud; +- touching a private attribute of anything other than ``self`` or ``cls``. + +Files that predate the rule are listed in ``check_tests_baseline.json`` with the +number of findings they still hold. That number may only fall: a file above its +baseline fails, and so does one below it until the baseline is lowered to match. +Every other file must have none. + +Usage: uv run python scripts/check_tests.py [path ...] +""" + +from __future__ import annotations + +import ast +import json +import sys +from collections import Counter +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_PATHS = [ROOT / "tests", *sorted(ROOT.glob("environments/*/tests"))] +SUPPORT = ROOT / "tests" / "support" +BASELINE = Path(__file__).with_name("check_tests_baseline.json") +PATCHERS = {"setattr", "delattr", "patch"} +PATCH_METHODS = {"object", "dict", "multiple"} + + +def _private(name: str) -> bool: + return name.startswith("_") and not (name.startswith("__") and name.endswith("__")) + + +def _is_patch(func: ast.expr) -> bool: + """``setattr``, ``patch`` and ``monkeypatch.setattr`` forms, and ``patch.object`` forms.""" + if isinstance(func, ast.Name): + return func.id in PATCHERS + if not isinstance(func, ast.Attribute): + return False + if func.attr in PATCHERS: + return True + owner = func.value + owner_name = owner.attr if isinstance(owner, ast.Attribute) else getattr(owner, "id", "") + return func.attr in PATCH_METHODS and owner_name == "patch" + + +def _root(node: ast.expr) -> ast.expr: + while isinstance(node, (ast.Attribute, ast.Subscript, ast.Call)): + node = node.func if isinstance(node, ast.Call) else node.value + return node + + +class Checker(ast.NodeVisitor): + def __init__(self, path: Path) -> None: + self.path = path + self.hud_names: set[str] = set() + self.problems: list[str] = [] + + def report(self, node: ast.AST, message: str) -> None: + line = getattr(node, "lineno", 0) + self.problems.append(f"{self.path.relative_to(ROOT)}:{line}: {message}") + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + if alias.name == "hud" or alias.name.startswith("hud."): + self.hud_names.add((alias.asname or alias.name).split(".")[0]) + if any(_private(part) for part in alias.name.split(".")): + self.report(node, f"imports private module {alias.name}") + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + module = node.module or "" + if module == "hud" or module.startswith("hud."): + private_module = any(_private(part) for part in module.split(".")) + for alias in node.names: + self.hud_names.add(alias.asname or alias.name) + if private_module or _private(alias.name): + self.report(node, f"imports private {module}.{alias.name}") + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + if _is_patch(node.func) and node.args: + target = node.args[0] + if isinstance(target, ast.Constant) and isinstance(target.value, str): + if target.value == "hud" or target.value.startswith("hud."): + self.report(node, f"patches {target.value}") + else: + root = _root(target) + if isinstance(root, ast.Name) and root.id in self.hud_names: + self.report(node, f"patches {ast.unparse(target)}, which comes from hud") + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> None: + if _private(node.attr): + root = _root(node) + if not (isinstance(root, ast.Name) and root.id in {"self", "cls"}): + self.report(node, f"touches private attribute {ast.unparse(node)}") + self.generic_visit(node) + + +def check(path: Path) -> list[str]: + checker = Checker(path) + checker.visit(ast.parse(path.read_text("utf-8"), filename=str(path))) + return checker.problems + + +def main(argv: list[str]) -> int: + roots = [Path(arg).resolve() for arg in argv] or DEFAULT_PATHS + files = sorted( + file + for root in roots + for file in ([root] if root.is_file() else root.rglob("*.py")) + if SUPPORT not in file.parents and "tasks" not in file.relative_to(ROOT).parts + ) + baseline: dict[str, int] = json.loads(BASELINE.read_text("utf-8")) if BASELINE.exists() else {} + findings = {str(file.relative_to(ROOT)): check(file) for file in files} + failed = False + for name, problems in findings.items(): + allowed = baseline.get(name, 0) + if len(problems) > allowed: + failed = True + for problem in problems: + print(problem) + print(f"{name}: {len(problems)} reach(es) into hud internals, {allowed} allowed\n") + elif len(problems) < allowed: + failed = True + print(f"{name}: {len(problems)} reach(es) left; lower its baseline from {allowed}\n") + counts = Counter({name: len(problems) for name, problems in findings.items()}) + if failed: + print("See Testing in AGENTS.md; baselines live in scripts/check_tests_baseline.json.") + elif baseline: + print( + f"{sum(counts[name] for name in baseline)} known reach(es) in {len(baseline)} file(s)" + ) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/check_tests_baseline.json b/scripts/check_tests_baseline.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/scripts/check_tests_baseline.json @@ -0,0 +1 @@ +{} diff --git a/scripts/example_lifecycle.py b/scripts/example_lifecycle.py deleted file mode 100644 index 9c26afcfb..000000000 --- a/scripts/example_lifecycle.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Prepare live example tasks and check their traces.""" - -from __future__ import annotations - -import argparse -import json -import os -import subprocess -import sys -import time -from pathlib import Path - -from hud.eval import Taskset - - -def prepare(example: str) -> None: - task = next(iter(Taskset.from_file("tasks.py"))) - Path(".hud").mkdir(exist_ok=True) - if example == "argument-hints": - file_id = os.environ["HUD_CI_DATA_FILE_ID"] - if not file_id: - raise ValueError("HUD_CI_DATA_FILE_ID must reference the shared CI attachment") - task.args["attachments"] = [{"file_id": file_id, "path": "notes.txt"}] - task.args["prompt"] = "Read files/notes.txt first. " + task.args["prompt"] - Taskset(tasks=[task]).to_file(".hud/ci-tasks.json") - - -def check() -> None: - trace_dir = Path(os.environ["HUD_TELEMETRY_LOCAL_DIR"]) - traces = list(trace_dir.glob("*.jsonl")) - assert len(traces) == 1, f"Expected one rollout trace, found {len(traces)}" - steps = [ - span["attributes"]["hud.payload"] - for line in traces[0].read_text().splitlines() - if (span := json.loads(line))["attributes"].get("hud.schema") == "hud.step.v1" - ] - errors = [ - step["error"] - for step in steps - if step["source"] in {"task", "agent", "system"} and step.get("error") - ] - assert not errors, errors - calls = [step["task_call"] for step in steps if step["source"] == "task"] - assert [call["phase"] for call in calls] == ["setup", "evaluate"], calls - assert calls[0]["result"].get("prompt"), "Task setup did not return a prompt" - grade = calls[1]["result"] - assert not grade.get("isError"), grade - assert isinstance(grade["score"], int | float) and 0 <= grade["score"] <= 1, grade - task = next(iter(Taskset.from_file(".hud/ci-tasks.json"))) - if attachments := task.args.get("attachments"): - assert calls[0]["result"]["data_files"] == [ - {"path": "files/notes.txt", "file_id": attachments[0]["file_id"]} - ] - - # Unset local export so `hud trace` must read back from the platform. - env = {key: value for key, value in os.environ.items() if key != "HUD_TELEMETRY_LOCAL_DIR"} - for attempt in range(12): - result = subprocess.run( - [str(Path(sys.executable).with_name("hud")), "trace", "get", traces[0].stem, "--json"], - env=env, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - events = json.loads(result.stdout) if result.returncode == 0 else [] - if any(event["kind"] == "agent_message" for event in events): - (trace_dir / "remote-events.json").write_text(result.stdout) - print(f"Hosted lifecycle verified: {traces[0].stem}, score={grade['score']}") - return - if attempt < 11: - time.sleep(5) - raise AssertionError( - f"No persisted agent events after 12 attempts: {result.stdout}\n{result.stderr}" - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - commands.add_parser("prepare").add_argument("example") - commands.add_parser("check") - args = parser.parse_args() - if args.command == "prepare": - prepare(args.example) - elif args.command == "check": - check() diff --git a/hud/agents/claude/tools/tests/__init__.py b/tests/__init__.py similarity index 100% rename from hud/agents/claude/tools/tests/__init__.py rename to tests/__init__.py diff --git a/hud/agents/tests/__init__.py b/tests/agents/__init__.py similarity index 100% rename from hud/agents/tests/__init__.py rename to tests/agents/__init__.py diff --git a/tests/agents/conftest.py b/tests/agents/conftest.py new file mode 100644 index 000000000..409bb900e --- /dev/null +++ b/tests/agents/conftest.py @@ -0,0 +1,27 @@ +"""A model provider reached directly rather than through the HUD gateway.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from tests.support import FakeServices, Models + +if TYPE_CHECKING: + from collections.abc import Iterator + + +@pytest.fixture(scope="session") +def provider_server() -> Iterator[FakeServices]: + server = FakeServices() + server.start() + yield server + server.stop() + + +@pytest.fixture +def provider(provider_server: FakeServices) -> Models: + """Scripted models served as a provider's own API.""" + provider_server.reset() + return Models(provider_server) diff --git a/tests/agents/test_chat.py b/tests/agents/test_chat.py new file mode 100644 index 000000000..064ea0e25 --- /dev/null +++ b/tests/agents/test_chat.py @@ -0,0 +1,180 @@ +"""What only ``OpenAIChatAgent`` does over Chat Completions: where its requests go, how it +continues a sampled conversation by token ids, how it replays an incomplete call, and the +training sample it records.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.agents import OpenAIChatAgent +from hud.agents.types import OpenAIChatConfig +from tests.fixtures.environments import run_task, workspace_env +from tests.support import Turn, say, steps + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import HudEnv, Models + +MODEL = "qwen3.6-plus" +TOKEN_IDS = {"extra_body": {"return_token_ids": True}} + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="hud-key") + + +@pytest.mark.parametrize( + ("openai_key", "explicit", "route", "credential"), + [ + (None, True, "provider", "explicit"), + (None, False, "gateway", "hud-key"), + ("openai-key", False, "gateway", "hud-key"), + ], + ids=["an-explicit-endpoint", "only-a-hud-key", "an-openai-key-is-not-its-own"], +) +async def test_requests_go_to_an_explicit_endpoint_and_otherwise_to_the_gateway( + openai_key: str | None, + explicit: bool, + route: str, + credential: str, + models: Models, + provider: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(OPENAI_API_KEY=openai_key) + models.script([say("done")]) + provider.script([say("done")]) + endpoint = {"api_key": "explicit", "base_url": provider.url} if explicit else {} + + run = await run_task( + workspace_env(tmp_path / "ws"), OpenAIChatAgent(OpenAIChatConfig(model=MODEL, **endpoint)) + ) + + (request,) = {"gateway": models, "provider": provider}[route].requests() + assert (run.trace.status, request.credential) == ("completed", credential) + + +SAMPLED = {"logprobs": True, "return_token_ids": True} + + +@pytest.mark.parametrize( + ("sampled", "continued"), + [ + ( + [{"prompt_token_ids": [1, 2], "token_ids": [3]}, {"token_ids": [5]}], + [{}, {"prompt_token_ids": [1, 2, 3], "continuation_from": 2}], + ), + ( + [ + {"prompt_token_ids": [1, 2], "token_ids": [3]}, + {"prompt_chunks": [{"type": "text", "text": "hi"}], "token_ids": [4]}, + {"token_ids": [5]}, + ], + [{}, {"prompt_token_ids": [1, 2, 3], "continuation_from": 2}, {}], + ), + ], + ids=["text-prompts-continue-the-kv-cache", "a-multimodal-prompt-starts-over"], +) +async def test_token_ids_continue_the_conversation_from_what_was_sampled( + sampled: list[dict[str, Any]], + continued: list[dict[str, Any]], + models: Models, + tmp_path: Path, +) -> None: + *calls, last = sampled + models.script( + [Turn(shell="true", extra=extra) for extra in calls] + [Turn(text="done", extra=last)] + ) + + await run_task( + workspace_env(tmp_path / "ws"), + OpenAIChatAgent(OpenAIChatConfig(model=MODEL, completion_kwargs=TOKEN_IDS)), + ) + + keys = {"prompt_token_ids", "continuation_from", *SAMPLED} + sent = [{k: v for k, v in r.body.items() if k in keys} for r in models.requests()] + assert sent == [SAMPLED | expected for expected in continued] + + +async def test_the_models_reasoning_is_replayed_in_the_conversation( + models: Models, tmp_path: Path +) -> None: + models.script([Turn(shell="true", reasoning="check first"), say("done")]) + + await run_task(workspace_env(tmp_path / "ws"), OpenAIChatAgent(OpenAIChatConfig(model=MODEL))) + + replayed = models.requests()[-1].body["messages"][1] + assert (replayed["role"], replayed["reasoning_content"]) == ("assistant", "check first") + + +async def test_completion_kwargs_cannot_override_what_the_agent_sends_itself( + models: Models, tmp_path: Path +) -> None: + reserved = {"model": "other", "stream": True, "tools": [], "messages": [], "max_tokens": 50} + models.script([say("done")]) + + await run_task( + workspace_env(tmp_path / "ws"), + OpenAIChatAgent(OpenAIChatConfig(model=MODEL, completion_kwargs=reserved)), + ) + + (request,) = models.requests() + assert (request.body["model"], request.body["stream"], request.body["max_tokens"]) == ( + MODEL, + False, + 50, + ) + assert request.body["messages"] != [] and request.body["tools"] != [] + + +async def test_an_incomplete_call_is_explained_and_replayed_with_empty_arguments( + models: Models, tmp_path: Path +) -> None: + cut_off = Turn( + native=( + { + "id": "call_cut", + "type": "function", + "function": {"name": "bash", "arguments": '{"command": "ec'}, + }, + ) + ) + models.script([cut_off, say("done")]) + + run = await run_task( + workspace_env(tmp_path / "ws"), OpenAIChatAgent(OpenAIChatConfig(model=MODEL)) + ) + + last = models.requests()[-1].body + assert run.trace.status == "completed" + assert last["messages"][1]["tool_calls"][0]["function"]["arguments"] == "{}" + assert "the arguments for this 'bash' call arrived incomplete" in json.dumps(last) + + +async def test_a_completion_with_token_ids_records_its_training_sample( + models: Models, tmp_path: Path +) -> None: + logprobs = [ + {"token": "Bl", "logprob": -0.5, "bytes": None, "top_logprobs": []}, + {"token": "ue.", "logprob": -0.25, "bytes": None, "top_logprobs": []}, + ] + sample = {"prompt_token_ids": [1, 2], "token_ids": [3, 4], "logprobs": {"content": logprobs}} + models.script([Turn(text="Blue.", extra=sample)]) + + run = await run_task( + workspace_env(tmp_path / "ws"), + OpenAIChatAgent(OpenAIChatConfig(model=MODEL, completion_kwargs=TOKEN_IDS)), + ) + + (step,) = [step for step in steps(run.trace_id) if step["source"] == "agent"] + assert step["sample"] == { + "prompt_token_ids": [1, 2], + "output_token_ids": [3, 4], + "output_logprobs": [-0.5, -0.25], + } diff --git a/tests/agents/test_claude.py b/tests/agents/test_claude.py new file mode 100644 index 000000000..71d2da9b6 --- /dev/null +++ b/tests/agents/test_claude.py @@ -0,0 +1,386 @@ +"""What only ``ClaudeAgent`` does: the hosted tools, tool search and computer tool it +offers, how it retries an invalid call or a broken stream, the images it sends back, +Bedrock invocation, and the citations and refusals it records.""" + +from __future__ import annotations + +import base64 +import io +import json +from dataclasses import replace +from typing import TYPE_CHECKING, Any + +import mcp.types as mcp_types +import pytest +from fastmcp import FastMCP +from fastmcp.tools import ToolResult +from PIL import Image + +from hud.agents import ClaudeAgent +from hud.agents.claude import ClaudeToolSearchTool, ClaudeWebFetchTool, ClaudeWebSearchTool +from hud.agents.types import ClaudeConfig +from hud.capabilities import Capability +from tests.fixtures.environments import run_task, workspace_env +from tests.support import Turn, call, fake_screen, mcp_server, say, steps, stream_error + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import FakeServices, HudEnv, Models + +MODEL = "claude-sonnet-4-6" + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="k") + + +def png(width: int, height: int) -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (width, height), "white").save(buffer, format="PNG") + return buffer.getvalue() + + +def upright_after_rotation() -> bytes: + """A 40x20 JPEG whose EXIF orientation says to show it rotated, as 20x40.""" + exif = Image.Exif() + exif[274] = 6 + buffer = io.BytesIO() + Image.new("RGB", (40, 20), "red").save(buffer, format="JPEG", exif=exif) + return buffer.getvalue() + + +async def test_hosted_web_tools_are_offered_to_the_models_they_support( + models: Models, tmp_path: Path +) -> None: + hosted = [ + ClaudeWebSearchTool(), + ClaudeWebFetchTool(citations_enabled=True), + ClaudeWebSearchTool(supported_models=("claude-opus-*",)), + ] + models.script([say("done")]) + + await run_task( + workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model=MODEL, hosted_tools=hosted)) + ) + + (request,) = models.requests() + assert [tool for tool in request.body["tools"] if tool.get("type", "").startswith("web_")] == [ + {"type": "web_search_20250305", "name": "web_search"}, + {"type": "web_fetch_20250910", "name": "web_fetch", "citations": {"enabled": True}}, + ] + + +@pytest.mark.parametrize( + ("threshold", "deferred"), + [(1, {"database__lookup", "search__lookup"}), (2, set())], + ids=["over-its-threshold", "at-its-threshold"], +) +async def test_tool_search_defers_mcp_tools_once_there_are_more_than_its_threshold( + threshold: int, deferred: set[str], models: Models, tmp_path: Path +) -> None: + lookups = FastMCP("lookups") + lookups.tool(name="lookup", description="Look a key up.")(lambda key: key) + models.script([say("done")]) + agent = ClaudeAgent( + ClaudeConfig(model=MODEL, hosted_tools=[ClaudeToolSearchTool(threshold=threshold)]) + ) + + async with ( + mcp_server(lookups, name="database") as database, + mcp_server(lookups, name="search") as search, + ): + await run_task(workspace_env(tmp_path / "ws", capabilities=(database, search)), agent) + + (request,) = models.requests() + assert {tool["name"] for tool in request.body["tools"] if tool.get("defer_loading")} == deferred + + +@pytest.mark.parametrize( + ("model", "beta", "kind", "sized"), + [ + ("claude-sonnet-4-6", "computer-use-2025-11-24", "computer_20251124", True), + ("claude-sonnet-4-5", "computer-use-2025-01-24", "computer_20250124", True), + ("qwen/qwen3.8-max", None, None, False), + ], + ids=["4-6", "4-5", "a-model-without-native-computer-use"], +) +async def test_each_model_gets_the_computer_tool_it_supports_for_the_screen( + model: str, beta: str | None, kind: str | None, sized: bool, models: Models, tmp_path: Path +) -> None: + models.script([say("done")]) + + async with fake_screen(width=200, height=100) as screen: + env = workspace_env(tmp_path / "ws", capabilities=(Capability.rfb(url=screen.url),)) + await run_task(env, ClaudeAgent(ClaudeConfig(model=model))) + + (request,) = models.requests() + (computer,) = [tool for tool in request.body["tools"] if tool["name"] == "computer"] + assert (request.headers.get("anthropic-beta"), computer.get("type")) == (beta, kind) + assert (computer.get("display_width_px"), computer.get("display_height_px")) == ( + (200, 100) if sized else (None, None) + ) + assert sized or "display is 200x100 pixels" in computer["description"] + + +async def test_a_failed_tool_result_reaches_the_model_marked_as_an_error( + models: Models, tmp_path: Path +) -> None: + models.script([call("str_replace_based_edit_tool", command="bogus", path="/x"), say("done")]) + + await run_task(workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model=MODEL))) + + (result,) = models.requests()[-1].body["messages"][-1]["content"] + assert (result["type"], result["is_error"]) == ("tool_result", True) + assert "unknown editor command: 'bogus'" in json.dumps(result["content"]) + + +INVALID_JSON = Turn( + native=({"type": "tool_use", "id": "toolu_bad", "name": "bash", "input": '{"command": ls}'},) +) + + +@pytest.mark.parametrize( + ("invalid", "status", "explained"), + [(1, "completed", False), (2, "completed", True), (3, "error", True)], + ids=["once-is-asked-again", "twice-is-explained-to-the-model", "three-times-fails"], +) +async def test_a_call_with_invalid_json_is_sampled_again_then_explained_then_fails( + invalid: int, status: str, explained: bool, models: Models, tmp_path: Path +) -> None: + models.queue([INVALID_JSON] * invalid + [say("done")]) + + run = await run_task(workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model=MODEL))) + + last = json.dumps(models.requests()[-1].body) + assert run.trace.status == status + assert ("Your previous tool-call arguments were invalid JSON." in last) is explained + + +@pytest.mark.parametrize( + ("errors", "kind", "status", "requests"), + [ + (1, "overloaded_error", "completed", 2), + (3, "gateway_timeout", "error", 3), + (1, "invalid_request_error", "error", 1), + ], + ids=["a-transient-error-is-retried", "three-transient-errors-fail", "a-request-error-fails"], +) +async def test_an_error_inside_the_stream_is_retried_only_when_transient( + errors: int, kind: str, status: str, requests: int, models: Models, tmp_path: Path +) -> None: + models.queue([stream_error(kind, headers={"retry-after-ms": "0"})] * errors + [say("done")]) + + run = await run_task(workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model=MODEL))) + + assert (run.trace.status, len(models.requests())) == (status, requests) + + +@pytest.mark.parametrize( + ("config", "sent", "beta"), + [ + ({}, [["text", "image"], ["text", "image"]], False), + ( + {"max_tool_result_images": 1, "thinking": {"type": "adaptive"}}, + [["text", "text"], ["text", "image"]], + True, + ), + ], + ids=["every-image-by-default", "only-the-newest-image"], +) +async def test_tool_result_images_are_sent_back_up_to_the_configured_count( + config: dict[str, Any], sent: list[list[str]], beta: bool, models: Models, tmp_path: Path +) -> None: + screenshots = FastMCP("screenshots") + + @screenshots.tool + def screenshot() -> ToolResult: + """Capture the current screen.""" + return ToolResult( + content=[ + mcp_types.TextContent(type="text", text="screen"), + mcp_types.ImageContent( + type="image", mimeType="image/png", data=base64.b64encode(png(32, 24)).decode() + ), + ] + ) + + models.script( + [ + replace(call("screenshot"), reasoning="first look"), + replace(call("screenshot"), reasoning="second look"), + say("done"), + ] + ) + agent = ClaudeAgent(ClaudeConfig.model_validate({"model": MODEL, **config})) + + async with mcp_server(screenshots, name="screenshots") as server: + await run_task(workspace_env(tmp_path / "ws", capabilities=(server,)), agent) + + last = models.requests()[-1] + results = last.body["messages"][2::2] + assert [[block["type"] for block in m["content"][0]["content"]] for m in results] == sent + assert ( + "thinking-binding-controls-2026-08-01" in last.headers.get("anthropic-beta", "") + ) is beta + + +@pytest.mark.parametrize( + ("data", "shape"), + [ + (png(40, 30), (40, 30)), + (png(3000, 2000), (3000, 2000)), + (upright_after_rotation(), (20, 40)), + ], + ids=["a-small-png", "a-wide-png", "a-jpeg-turned-upright-by-its-exif"], +) +async def test_an_image_the_editor_views_is_sent_within_claudes_limits_in_its_shape( + data: bytes, shape: tuple[int, int], models: Models, tmp_path: Path +) -> None: + root = tmp_path / "ws" + models.script( + [call("str_replace_based_edit_tool", command="view", path=f"{root}/file"), say("done")] + ) + + await run_task( + workspace_env(root, files={"file": data}), ClaudeAgent(ClaudeConfig(model=MODEL)) + ) + + ((block,),) = [r["content"] for r in models.requests()[-1].body["messages"][-1]["content"]] + with Image.open(io.BytesIO(base64.b64decode(block["source"]["data"]))) as sent: + width, height = sent.size + assert max(width, height) <= 1568 and width * height <= 1_150_000 + assert abs(width / height - shape[0] / shape[1]) < 0.01 + + +async def test_an_image_that_cannot_be_read_is_told_to_the_model_as_an_error( + models: Models, tmp_path: Path +) -> None: + root = tmp_path / "ws" + models.script( + [call("str_replace_based_edit_tool", command="view", path=f"{root}/file"), say("done")] + ) + corrupt = b"\x89PNG\r\n\x1a\ngarbage" + + await run_task( + workspace_env(root, files={"file": corrupt}), ClaudeAgent(ClaudeConfig(model=MODEL)) + ) + + (result,) = models.requests()[-1].body["messages"][-1]["content"] + assert result["is_error"] is True + assert "Cannot view image" in result["content"][0]["text"] + + +async def test_the_newest_user_turn_is_marked_for_prompt_caching( + models: Models, tmp_path: Path +) -> None: + models.script([call("bash", command="true"), say("done")]) + + await run_task(workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model=MODEL))) + + first, second = (request.body["messages"] for request in models.requests()) + assert [block.get("cache_control") for block in first[-1]["content"]] == [{"type": "ephemeral"}] + assert [block.get("cache_control") for block in second[-1]["content"]] == [ + {"type": "ephemeral"} + ] + assert "cache_control" not in json.dumps(second[0]) + + +async def test_a_bedrock_inference_profile_is_invoked_on_bedrock_without_streaming( + services: FakeServices, hud_env: HudEnv, tmp_path: Path +) -> None: + pytest.importorskip("botocore", reason="needs the bedrock extra") + profile = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/claude" + hud_env.set( + AWS_ACCESS_KEY_ID="AKIA", + AWS_SECRET_ACCESS_KEY="secret", + AWS_REGION="us-east-1", + ANTHROPIC_BEDROCK_BASE_URL=services.url("gateway"), + ) + services.route( + "gateway", + "POST", + "/model/{model:path}", + json={ + "id": "msg_bedrock", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [{"type": "text", "text": "from bedrock"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + run = await run_task(workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model=profile))) + + (request,) = services.requests("gateway", "POST") + assert run.trace.content == "from bedrock" + assert request.path == f"/model/{profile}/invoke" + assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256") + assert ("model" in request.json, "stream" in request.json) == (False, False) + + +async def test_document_and_web_citations_land_on_the_agent_step( + models: Models, tmp_path: Path +) -> None: + web = "https://example.com/sky" + citations = [ + { + "type": "char_location", + "cited_text": "the sky is blue", + "document_index": 0, + "document_title": "notes", + "start_char_index": 4, + "end_char_index": 19, + }, + { + "type": "web_search_result_location", + "cited_text": "blue skies", + "url": web, + "title": "Sky", + "encrypted_index": "x", + }, + ] + models.script( + [ + Turn( + reasoning="look it up", + native=({"type": "text", "text": "Blue.", "citations": citations},), + ) + ] + ) + + run = await run_task(workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model=MODEL))) + + (step,) = [step for step in steps(run.trace_id) if step["source"] == "agent"] + assert (step["content"], step["reasoning"]) == ("Blue.", "look it up") + assert [(c["type"], c["source"], c["text"]) for c in step["citations"]] == [ + ("document_citation", "0", "the sky is blue"), + ("web_search_result_location", web, "blue skies"), + ] + + +@pytest.mark.parametrize( + ("explanation", "refusal"), + [ + ( + "This request was blocked under the Usage Policy.", + "This request was blocked under the Usage Policy.", + ), + (None, "This request was refused (cyber)."), + ], + ids=["with-an-explanation", "with-only-a-category"], +) +async def test_a_refusal_lands_on_the_agent_step( + explanation: str | None, refusal: str, models: Models, tmp_path: Path +) -> None: + details = {"type": "refusal", "category": "cyber", "explanation": explanation} + models.script([Turn(finish="refusal", extra={"stop_details": details})]) + + run = await run_task(workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model=MODEL))) + + (step,) = [step for step in steps(run.trace_id) if step["source"] == "agent"] + assert (step["finish_reason"], step["refusal"]) == ("refusal", refusal) diff --git a/tests/agents/test_claude_cli.py b/tests/agents/test_claude_cli.py new file mode 100644 index 000000000..5cdd3156d --- /dev/null +++ b/tests/agents/test_claude_cli.py @@ -0,0 +1,270 @@ +"""What only ``ClaudeCLIAgent`` does: what it hands the ``claude`` CLI (prompt, tools, MCP +servers, where its requests go and which model each tier uses), how the CLI's exit ends +the run, and the computer bridge it serves the CLI. A stand-in ``claude`` on the +workspace's ``PATH`` records what it was handed; the real CLI runs in the live lane.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest +from fastmcp import FastMCP + +from hud.agents import ClaudeCLIAgent +from hud.agents.types import ClaudeCLIConfig +from hud.capabilities import Capability, Connection +from hud.environment.egress import ANY_HOST +from tests.fixtures.environments import run_task, workspace_env +from tests.support import clis, fake_screen, mcp_server + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import HudEnv, Models + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="hud-key") + + +async def test_the_cli_is_given_the_prompt_its_allowed_tools_and_the_environments_servers( + tmp_path: Path, +) -> None: + lookups = FastMCP("tools") + lookups.tool(name="lookup", description="Look a key up.")(lambda key: key) + stub = clis.Stub(tmp_path / "stub", "claude", stdout=clis.answer("claude", "done")) + stub.install() + + async with mcp_server(lookups) as server: + env = workspace_env(tmp_path / "ws", env=stub.shell_env(), capabilities=(server,)) + await run_task(env, ClaudeCLIAgent(), prompt="Fix the bug.") + + ((argv, stdin, files),) = [(c["argv"], c["stdin"], c["files"]) for c in stub.captures()] + allowed = [argv[index + 1] for index, arg in enumerate(argv) if arg == "--allowedTools"] + assert clis.prompt("claude", stdin) == "Fix the bug." + assert allowed == ClaudeCLIConfig().allowed_tools + assert list(json.loads(files[".hud_mcp_config.json"])["mcpServers"]) == ["tools"] + + +@pytest.mark.parametrize( + ("own_key", "gateway", "via_gateway", "credential"), + [ + ("own-key", False, False, "own-key"), + (None, False, True, "hud-key"), + ("own-key", True, True, "hud-key"), + ], + ids=["its-own-key", "only-a-hud-key", "the-gateway-forced-over-its-own-key"], +) +async def test_the_cli_is_pointed_at_the_provider_with_its_own_key_or_else_the_gateway( + own_key: str | None, + gateway: bool, + via_gateway: bool, + credential: str, + models: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(ANTHROPIC_API_KEY=own_key) + stub = clis.Stub(tmp_path / "stub", "claude", stdout=clis.answer("claude", "done")) + stub.install() + + await run_task( + workspace_env(tmp_path / "ws", env=stub.shell_env()), + ClaudeCLIAgent(ClaudeCLIConfig(gateway=gateway)), + ) + + (env,) = [capture["env"] for capture in stub.captures()] + assert env.get("ANTHROPIC_BASE_URL") == (models.url if via_gateway else None) + assert env.get("ANTHROPIC_API_KEY") == credential + assert ("Trace-Id: " in env.get("ANTHROPIC_CUSTOM_HEADERS", "")) is via_gateway + + +async def test_every_model_tier_and_subagents_use_the_configured_model(tmp_path: Path) -> None: + stub = clis.Stub(tmp_path / "stub", "claude", stdout=clis.answer("claude", "done")) + stub.install() + + await run_task( + workspace_env(tmp_path / "ws", env=stub.shell_env()), + ClaudeCLIAgent(ClaudeCLIConfig(model="claude-opus-5", gateway=True)), + ) + + (env,) = [capture["env"] for capture in stub.captures()] + tiers = ( + "ANTHROPIC_MODEL", + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "CLAUDE_CODE_SUBAGENT_MODEL", + ) + assert {env[name] for name in tiers} == {"claude-opus-5"} + + +WRITE = clis.claude_assistant( + {"type": "text", "text": "editing"}, + {"type": "tool_use", "id": "tool-1", "name": "Write", "input": {"file_path": "a.txt"}}, + stop="tool_use", +) +DONE = clis.claude_assistant({"type": "text", "text": "done"}) + + +@pytest.mark.parametrize( + ("stdout", "stderr", "exit", "status", "content", "error"), + [ + ("", "boom", 1, "error", None, "boom"), + (clis.answer("claude", "done"), "transport failed", 1, "error", "done", "transport failed"), + (clis.answer("claude", "done"), "", 1, "completed", "done", None), + (clis.jsonl(DONE), "", 0, "error", "done", "claude CLI exited without a result event"), + ( + clis.jsonl(DONE, clis.claude_result("rate limited upstream", is_error=True)), + "", + 0, + "error", + "rate limited upstream", + "rate limited upstream", + ), + ( + clis.jsonl(WRITE, clis.claude_result()), + "", + 0, + "error", + "done", + "claude CLI exited without results for tool calls: tool-1", + ), + ], + ids=[ + "a-nonzero-exit-without-output", + "a-nonzero-exit-after-a-result-with-stderr", + "a-nonzero-exit-after-a-result-without-stderr", + "a-zero-exit-without-a-result", + "a-result-reporting-an-error", + "a-tool-call-left-without-a-result", + ], +) +async def test_how_the_cli_exits_decides_how_the_run_ends( + stdout: str, + stderr: str, + exit: int, + status: str, + content: str | None, + error: str | None, + tmp_path: Path, +) -> None: + stub = clis.Stub(tmp_path / "stub", "claude", stdout=stdout, stderr=stderr, exit=exit) + stub.install() + + run = await run_task(workspace_env(tmp_path / "ws", env=stub.shell_env()), ClaudeCLIAgent()) + + assert (run.trace.status, run.trace.content) == (status, content) + assert run.trace.error == (None if error is None else f"[agent loop] RuntimeError: {error}") + + +async def test_mcp_servers_reach_the_cli_in_a_config_removed_after_the_run( + tmp_path: Path, +) -> None: + stub = clis.Stub(tmp_path / "stub", "claude", stdout=clis.answer("claude", "done")) + stub.install() + servers = ( + Capability.mcp(name="database", url="http://db.internal:8000/mcp", auth_token="secret"), + Capability.mcp(name="search", url="http://search.internal:8000/sse", transport="sse"), + ) + + await run_task( + workspace_env(tmp_path / "ws", env=stub.shell_env(), capabilities=servers), + ClaudeCLIAgent(), + ) + + (capture,) = stub.captures() + assert capture["argv"][-2:] == ["--mcp-config", ".hud_mcp_config.json"] + assert json.loads(capture["files"][".hud_mcp_config.json"]) == { + "mcpServers": { + "database": { + "type": "http", + "url": "http://db.internal:8000/mcp", + "headers": {"Authorization": "Bearer secret"}, + }, + "search": {"type": "sse", "url": "http://search.internal:8000/sse"}, + } + } + assert list((tmp_path / "ws").iterdir()) == [] + + +async def test_the_cli_drives_the_screen_through_the_computer_bridge(tmp_path: Path) -> None: + actions: list[dict[str, Any]] = [ + {"action": "left_click", "coordinate": [3, 4]}, + {"action": "frobnicate"}, + ] + stub = clis.Stub( + tmp_path / "stub", "claude", stdout=clis.answer("claude", "done"), computer=actions + ) + stub.install() + + async with fake_screen() as screen: + env = workspace_env( + tmp_path / "ws", env=stub.shell_env(), capabilities=(Capability.rfb(url=screen.url),) + ) + run = await run_task(env, ClaudeCLIAgent()) + + (capture,) = stub.captures() + assert run.trace.status == "completed" + assert capture["computer"] == [ + {"isError": False, "content": ["image"]}, + {"isError": True, "content": ["unsupported computer action: 'frobnicate'"]}, + ] + assert [(e.x, e.y, e.buttons) for e in screen.pointer_events()] == [ + (3, 4, 0), + (3, 4, 1), + (3, 4, 0), + ] + + +async def test_an_mcp_server_named_like_the_computer_bridge_is_refused(tmp_path: Path) -> None: + stub = clis.Stub(tmp_path / "stub", "claude", stdout=clis.answer("claude", "done")) + stub.install() + + async with fake_screen() as screen: + clash = ( + Capability.rfb(url=screen.url), + Capability.mcp(name="computer-use", url="http://tools.internal:8000/mcp"), + ) + env = workspace_env(tmp_path / "ws", env=stub.shell_env(), capabilities=clash) + run = await run_task(env, ClaudeCLIAgent()) + + assert (run.trace.error, stub.captures()) == ( + "[agent loop] RuntimeError: duplicate MCP server name 'computer-use'", + [], + ) + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_the_cli_reaches_a_process_bound_connection_without_its_credential( + tmp_path: Path, +) -> None: + connection = Connection( + name="inference", + capability="ssh", + url="https://inference.hud.so/v1", + headers={"Authorization": "Bearer scoped-runtime-token"}, + ) + stub = clis.Stub(tmp_path / "stub", "claude", stdout=clis.answer("claude", "done")) + stub.install() + env = workspace_env( + tmp_path / "ws", + env=stub.shell_env(), + allowed_hosts={ANY_HOST}, + require_isolation=True, + track_files=False, + ) + + run = await run_task(env, ClaudeCLIAgent(), connections=(connection,)) + + (capture,) = stub.captures() + assert run.trace.status == "completed" + assert (capture["env"]["ANTHROPIC_BASE_URL"], capture["env"]["ANTHROPIC_AUTH_TOKEN"]) == ( + connection.client_url, + "hud-process-bound", + ) + assert not {"ANTHROPIC_API_KEY", "ANTHROPIC_CUSTOM_HEADERS"} & capture["env"].keys() + assert "scoped-runtime-token" not in str(capture) and "hud-key" not in str(capture) diff --git a/tests/agents/test_cli_agents.py b/tests/agents/test_cli_agents.py new file mode 100644 index 000000000..3adf296b9 --- /dev/null +++ b/tests/agents/test_cli_agents.py @@ -0,0 +1,155 @@ +"""What every coding-CLI agent does with its CLI: run it in the workspace, record what it +ran, end the run when it fails, refuse what it cannot route, and stop it at the agent +timeout. A stand-in on the workspace's ``PATH`` plays each CLI; every CLI family in +``AgentType`` is a row.""" + +from __future__ import annotations + +import asyncio +import time +from typing import TYPE_CHECKING + +import pytest + +from hud.agents.types import ToolStep +from hud.eval import LocalRuntime, Shared, Task, rollout +from hud.types import AgentType +from tests.fixtures.environments import run_task, workspace_env +from tests.support import clis + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import HudEnv + +#: (agent type, the executable it launches) +CLIS = [("claude_cli", "claude"), ("codex_cli", "codex")] +EVERY_CLI = pytest.mark.parametrize(("agent_type", "cli"), CLIS, ids=[row[0] for row in CLIS]) + + +def test_every_cli_family_is_a_row() -> None: + assert {row[0] for row in CLIS} == {kind.value for kind in AgentType if kind.is_cli} + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="hud-key") + + +@EVERY_CLI +async def test_a_command_the_cli_ran_acts_on_the_workspace_and_lands_on_the_trace( + agent_type: str, cli: str, tmp_path: Path +) -> None: + stub = clis.Stub( + tmp_path / "stub", + cli, + commands=["touch made"], + stdout=clis.shell(cli, "touch made", "done"), + ) + stub.install() + + run = await run_task( + workspace_env(tmp_path / "ws", env=stub.shell_env()), AgentType(agent_type).cls() + ) + + assert (run.trace.status, run.trace.content) == ("completed", "done") + assert (tmp_path / "ws" / "made").exists() + assert sum(isinstance(step, ToolStep) for step in run.trace.steps) == 1 + + +@EVERY_CLI +async def test_one_agent_drives_concurrent_rollouts_each_with_its_own_cli( + agent_type: str, cli: str, tmp_path: Path +) -> None: + stub = clis.Stub(tmp_path / "stub", cli, stdout=clis.answer(cli, "done")) + stub.install() + env, agent = workspace_env(tmp_path / "ws", env=stub.shell_env()), AgentType(agent_type).cls() + + runs = await asyncio.gather( + run_task(env, agent, prompt="first"), run_task(env, agent, prompt="second") + ) + + assert [run.trace.status for run in runs] == ["completed", "completed"] + assert sorted(clis.prompt(cli, capture["stdin"]) for capture in stub.captures()) == [ + "first", + "second", + ] + + +@EVERY_CLI +async def test_a_failure_the_cli_reports_ends_the_run_with_its_error( + agent_type: str, cli: str, tmp_path: Path +) -> None: + stub = clis.Stub(tmp_path / "stub", cli, stdout=clis.fault(cli, "model unavailable")) + stub.install() + + run = await run_task( + workspace_env(tmp_path / "ws", env=stub.shell_env()), AgentType(agent_type).cls() + ) + + assert run.trace.status == "error" + assert "model unavailable" in (run.trace.error or "") + + +@EVERY_CLI +async def test_routing_through_the_gateway_without_a_hud_key_is_refused_before_the_cli_runs( + agent_type: str, cli: str, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY=None) + stub = clis.Stub(tmp_path / "stub", cli, stdout=clis.answer(cli, "done")) + stub.install() + kind = AgentType(agent_type).cls + + run = await run_task( + workspace_env(tmp_path / "ws", env=stub.shell_env()), kind(kind.config_cls(gateway=True)) + ) + + assert "HUD_API_KEY is required for HUD gateway routing" in (run.trace.error or "") + assert stub.captures() == [] + + +@EVERY_CLI +async def test_a_cli_missing_from_the_workspace_names_the_platform_it_needs( + agent_type: str, cli: str, tmp_path: Path +) -> None: + stub = clis.Stub(tmp_path / "stub", cli, stdout=clis.answer(cli, "done")) + stub.install() + env = workspace_env(tmp_path / "ws", env=stub.shell_env(path="/usr/bin:/bin")) + + run = await run_task(env, AgentType(agent_type).cls()) + + assert "is unavailable for runtime platform " in (run.trace.error or "") + assert "install it in the environment or provide a managed runtime bundle" in ( + run.trace.error or "" + ) + + +@EVERY_CLI +async def test_the_agent_timeout_ends_the_run_and_the_cli_while_the_workspace_lives_on( + agent_type: str, cli: str, tmp_path: Path +) -> None: + """A pooled workspace outlives the run, so only the agent can end the CLI it started. + The timeout is a margin over a whole run on this machine just now, so the CLI has + started before it lands.""" + kind = AgentType(agent_type).cls + stub = clis.Stub(tmp_path / "stub", cli, stdout=clis.answer(cli, "done")) + stub.install() + started = time.monotonic() + await run_task(workspace_env(tmp_path / "ws", env=stub.shell_env()), kind()) + timeout = 1.0 + 2 * (time.monotonic() - started) + hung = clis.Stub( + tmp_path / "stub", cli, stdout=clis.answer(cli, "done").splitlines()[0] + "\n", hang=True + ) + hung.install() + env = workspace_env(tmp_path / "ws", env=hung.shell_env()) + + async with Shared(LocalRuntime(env), width=1) as pool: + run = await rollout( + Task(env=env.name, id="task"), + kind(kind.config_cls(timeout_seconds=timeout)), + runtime=pool, + ) + (capture,) = hung.captures() + exited = clis.process_exited(capture["pid"]) + + assert (run.trace.status, run.trace.stop_reason, exited) == ("error", "timeout", True) diff --git a/tests/agents/test_codex.py b/tests/agents/test_codex.py new file mode 100644 index 000000000..734ad1b06 --- /dev/null +++ b/tests/agents/test_codex.py @@ -0,0 +1,242 @@ +"""What only ``CodexCLIAgent`` does: where it points ``codex exec`` and with which credential, +what it records from codex's items, and how codex's exit ends the run. A stand-in +``codex`` on the workspace's ``PATH`` records what it was handed; the real CLI runs in the +live lane.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.agents import CodexCLIAgent +from hud.agents.types import CodexCLIConfig, ToolStep +from hud.capabilities import Connection +from hud.environment.egress import ANY_HOST +from tests.fixtures.environments import run_task, workspace_env +from tests.support import clis + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import HudEnv, Models + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="hud-key") + + +@pytest.mark.parametrize( + ("variables", "gateway", "via_gateway", "credential"), + [ + ({"OPENAI_API_KEY": "own-key"}, False, False, "own-key"), + ({}, False, True, "hud-key"), + ({"OPENAI_API_KEY": "own-key"}, True, True, "hud-key"), + ({"HUD_API_KEY": None}, False, False, None), + ], + ids=[ + "its-own-key", + "only-a-hud-key", + "the-gateway-forced-over-its-own-key", + "no-key-leaves-its-own-login", + ], +) +async def test_codex_is_pointed_at_the_gateway_over_responses_unless_it_has_its_own_key( + variables: dict[str, str | None], + gateway: bool, + via_gateway: bool, + credential: str | None, + models: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(**variables) + stub = clis.Stub(tmp_path / "stub", "codex", stdout=clis.answer("codex", "done")) + stub.install() + + await run_task( + workspace_env(tmp_path / "ws", env=stub.shell_env()), + CodexCLIAgent(CodexCLIConfig(gateway=gateway)), + ) + + (capture,) = stub.captures() + settings = clis.overrides(capture["argv"]) + key = json.loads(settings.get("model_providers.hud.env_key", '"CODEX_API_KEY"')) + assert settings.get("model_providers.hud.base_url") == ( + json.dumps(models.url) if via_gateway else None + ) + assert settings.get("model_providers.hud.wire_api") == ('"responses"' if via_gateway else None) + assert capture["env"].get(key) == credential + assert ('"Trace-Id"=' in settings.get("model_providers.hud.http_headers", "")) is via_gateway + + +@pytest.mark.parametrize( + ("item", "name", "is_error", "texts"), + [ + ( + clis.codex_item( + "command_execution", + "c", + command="false", + aggregated_output="", + exit_code=1, + status="completed", + ), + "shell", + True, + [""], + ), + ( + clis.codex_item( + "file_change", + "f", + changes=[{"kind": "update", "path": "a.txt"}], + status="completed", + ), + "apply_patch", + False, + ["update: a.txt"], + ), + ( + clis.codex_item( + "mcp_tool_call", + "m", + server="tools", + tool="lookup", + arguments={"key": "k"}, + result={"content": [{"type": "text", "text": "found"}]}, + status="completed", + ), + "lookup", + False, + ["found"], + ), + ( + clis.codex_item( + "mcp_tool_call", + "m", + server="tools", + tool="lookup", + arguments={}, + error={"message": "no such key"}, + status="failed", + ), + "lookup", + True, + ["no such key"], + ), + ( + clis.codex_item( + "web_search", "w", query="sky", action={"type": "search", "query": "sky"} + ), + "web_search", + False, + ['{"type":"search","query":"sky"}'], + ), + ], + ids=[ + "a-failed-command", + "a-file-change", + "an-mcp-tool-call", + "a-failed-mcp-tool-call", + "a-web-search", + ], +) +async def test_each_item_codex_ran_lands_on_the_trace_as_a_tool_step( + item: dict[str, Any], name: str, is_error: bool, texts: list[str], tmp_path: Path +) -> None: + stdout = clis.jsonl(clis.CODEX_STARTED, item, clis.CODEX_COMPLETED) + stub = clis.Stub(tmp_path / "stub", "codex", stdout=stdout) + stub.install() + + run = await run_task(workspace_env(tmp_path / "ws", env=stub.shell_env()), CodexCLIAgent()) + + (recorded,) = [step for step in run.trace.steps if isinstance(step, ToolStep)] + assert recorded.call is not None and recorded.result is not None + sent = [getattr(block, "text", None) for block in recorded.result.content] + assert (recorded.call.name, recorded.result.isError, sent) == (name, is_error, texts) + + +@pytest.mark.parametrize( + ("stdout", "stderr", "exit", "content", "error"), + [ + ( + clis.jsonl( + {"type": "turn.started"}, + {"type": "turn.failed", "error": {"message": "model unavailable"}}, + ), + "", + 0, + None, + "model unavailable", + ), + ("", "authentication failed", 1, None, "authentication failed"), + ( + clis.jsonl({"type": "error", "message": "gateway rejected streaming"}), + "noisy", + 1, + None, + "gateway rejected streaming", + ), + ( + clis.jsonl( + clis.CODEX_STARTED, + clis.codex_item("agent_message", "message-1", text="Implemented and verified."), + ), + "", + 0, + "Implemented and verified.", + "codex CLI exited without a turn.completed event", + ), + ], + ids=[ + "a-failed-turn", + "a-nonzero-exit", + "a-nonzero-exit-with-a-structured-error", + "a-zero-exit-without-turn-completed", + ], +) +async def test_how_codex_exits_decides_how_the_run_ends( + stdout: str, stderr: str, exit: int, content: str | None, error: str, tmp_path: Path +) -> None: + stub = clis.Stub(tmp_path / "stub", "codex", stdout=stdout, stderr=stderr, exit=exit) + stub.install() + + run = await run_task(workspace_env(tmp_path / "ws", env=stub.shell_env()), CodexCLIAgent()) + + assert (run.trace.status, run.trace.content, run.trace.error) == ( + "error", + content, + f"[agent loop] RuntimeError: {error}", + ) + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_codex_refuses_a_launcher_script_for_a_process_bound_connection( + tmp_path: Path, +) -> None: + connection = Connection( + name="inference", + capability="ssh", + url="https://inference.hud.so/v1", + headers={"Authorization": "Bearer scoped-runtime-token"}, + ) + stub = clis.Stub(tmp_path / "stub", "codex", stdout=clis.answer("codex", "done")) + stub.install() + env = workspace_env( + tmp_path / "ws", + env=stub.shell_env(), + allowed_hosts={ANY_HOST}, + require_isolation=True, + track_files=False, + ) + + run = await run_task(env, CodexCLIAgent(), connections=(connection,)) + + assert "/codex is a launcher script, but a process-bound inference connection " in ( + run.trace.error or "" + ) + assert stub.captures() == [] diff --git a/tests/agents/test_create_agent.py b/tests/agents/test_create_agent.py new file mode 100644 index 000000000..817895ae1 --- /dev/null +++ b/tests/agents/test_create_agent.py @@ -0,0 +1,64 @@ +"""``create_agent``: a model name resolves, through the gateway's catalog, to the agent that +speaks its protocol, and that agent's requests go through the HUD gateway.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from hud.agents import create_agent +from tests.fixtures.environments import run_task, workspace_env +from tests.support import Reply, say + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import FakeServices, HudEnv, Models + +CATALOG = [ + { + "id": "ft:custom-123", + "model_name": "gpt-5.5", + "sdk_agent_type": "openai_compatible", + "provider": {"name": "openai"}, + }, + {"id": "MiniMax-M3", "model_name": "MiniMax-M3", "sdk_agent_type": "openai_compatible"}, + {"id": "claude-sonnet-4-6", "name": "Claude Sonnet 4.6", "sdk_agent_type": "claude"}, +] + + +@pytest.mark.parametrize( + ("name", "protocol", "model"), + [ + ("openai", "responses", "gpt-5.6"), + ("claude", "anthropic", "claude-sonnet-4-6"), + ("Claude Sonnet 4.6", "anthropic", "claude-sonnet-4-6"), + ("ft:custom-123", "chat", "gpt-5.5"), + ("minimax-m3", "chat", "MiniMax-M3"), + ], + ids=["an-agent-type", "another-agent-type", "a-display-name", "a-catalog-id", "any-case"], +) +async def test_a_created_agent_speaks_its_models_protocol_through_the_gateway( + name: str, + protocol: str, + model: str, + services: FakeServices, + models: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(HUD_API_KEY="hud-key", OPENAI_API_KEY="own-key", ANTHROPIC_API_KEY="own-key") + services.route( + "api", + "GET", + "/v2/models", + Reply(json={"items": CATALOG[:2], "total": 3}), + Reply(json={"items": CATALOG[2:], "total": 3}), + ) + models.script([say("done")]) + + await run_task(workspace_env(tmp_path / "ws"), create_agent(name)) + + (request,) = models.requests() + assert (request.protocol, request.model, request.credential) == (protocol, model, "hud-key") diff --git a/tests/agents/test_end_to_end.py b/tests/agents/test_end_to_end.py new file mode 100644 index 000000000..f82b8293f --- /dev/null +++ b/tests/agents/test_end_to_end.py @@ -0,0 +1,109 @@ +"""An agent takes each fixture environment to its reward, on a scripted model or on its +real model (the ``live`` lane). A reward of 1 proves the prompt reached the model, its +tool calls reached the environment, and the grade came from the right task. Every family +in ``AgentType`` has live rows.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.agents import ClaudeAgent +from hud.agents.types import ClaudeConfig +from hud.eval import LocalRuntime, Task, Taskset +from hud.types import AgentType +from tests.fixtures.envs import capable_model, source +from tests.support import steps + +if TYPE_CHECKING: + from tests.support import Hud, HudEnv, Models + +#: (fixture, tool steps its scripted model takes, task phases the trace records) +SCRIPTED = [ + ("echo", 0, ["setup", "evaluate"]), + ("secret", 1, ["setup", "evaluate"]), + ("tools", 5, ["setup", "evaluate"]), + ("verifier", 0, ["setup", "evaluate", "setup", "evaluate"]), +] + + +@pytest.mark.parametrize("entry", ["hud eval", "Taskset.run"]) +@pytest.mark.parametrize( + ("fixture", "tool_steps", "phases"), SCRIPTED, ids=[r[0] for r in SCRIPTED] +) +async def test_a_scripted_model_takes_each_fixture_to_its_reward( + entry: str, + fixture: str, + tool_steps: int, + phases: list[str], + models: Models, + hud_env: HudEnv, + hud: Hud, +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(capable_model(fixture)) + path = source(fixture) + + run: dict[str, Any] + if entry == "hud eval": + result = hud("eval", str(path), "claude", "--max-steps", "8", "-y", "--json") + assert result.exit_code == 0, result + (run,) = result.json["runs"] + else: + agent = ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6", max_steps=8)) + (done,) = (await Taskset.from_file(path).run(agent, runtime=LocalRuntime(path))).runs + run = {"reward": done.reward, "trace_id": done.trace_id, "error": done.trace.error} + + recorded = steps(run["trace_id"]) + calls = [step["task_call"] for step in recorded if step["source"] == "task"] + assert (run["reward"], run.get("error")) == (1.0, None) + assert [step["error"] for step in recorded if step.get("error")] == [] + assert [call["phase"] for call in calls] == phases + assert sum(step["source"] == "tool" for step in recorded) == tool_steps + assert calls[0]["result"]["prompt"] == models.requests()[0].prompt + + +FIXTURES = { + "echo": Task(env="echo", id="echo", args={"word": "tangerine"}), + "tools": Task(env="tools", id="collect_code"), + "servers": Task(env="servers", id="look_up_both"), + "workspace": Task(env="workspace", id="fix_the_file"), + "picture": Task(env="picture", id="name_the_colour", args={"colour": "red"}), + "screen": Task(env="screen", id="click_target"), +} +#: (agent type, the cheapest model the gateway serves it, the fixtures it runs). Chat +#: Completions agents have no computer use, so a screen pays them nothing. +LIVE = [ + ("claude", "claude-haiku-4-5", list(FIXTURES)), + ("openai", "gpt-5.4-mini", list(FIXTURES)), + ("gemini", "gemini-3.8-flash", list(FIXTURES)), + ("openai_compatible", "deepseek/deepseek-v4-flash", list(FIXTURES)), + ("claude_cli", "claude-haiku-4-5", ["workspace"]), + ("codex_cli", "gpt-5.4-mini", ["workspace"]), +] + + +def test_every_agent_family_has_live_rows() -> None: + assert {row[0] for row in LIVE} == {kind.value for kind in AgentType} + + +@pytest.mark.e2e +@pytest.mark.live +@pytest.mark.parametrize( + ("agent_type", "model", "fixture"), + [(kind, model, fixture) for kind, model, fixtures in LIVE for fixture in fixtures], +) +async def test_a_real_model_takes_each_fixture_to_its_reward( + agent_type: str, model: str, fixture: str, live: HudEnv +) -> None: + kind = AgentType(agent_type).cls + path = source(fixture) + + job = await Taskset(fixture, [FIXTURES[fixture]]).run( + kind(kind.config_cls(model=model)), runtime=LocalRuntime(path) + ) + + (run,) = job.runs + pays = 0.0 if (agent_type, fixture) == ("openai_compatible", "screen") else 1.0 + assert (run.reward, run.trace.error) == (pays, None) diff --git a/tests/agents/test_gemini.py b/tests/agents/test_gemini.py new file mode 100644 index 000000000..16f6e0a32 --- /dev/null +++ b/tests/agents/test_gemini.py @@ -0,0 +1,183 @@ +"""What only ``GeminiAgent`` does: the grounding and computer tools it offers, how it bounds +a long tool output and drops old screenshots, how its replies fail, and the grounding it +records as citations.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest +from fastmcp import FastMCP + +from hud.agents import GeminiAgent +from hud.agents.gemini.tools import ( + GeminiCodeExecutionTool, + GeminiGoogleSearchTool, + GeminiUrlContextTool, +) +from hud.agents.types import GeminiConfig +from hud.capabilities import Capability +from tests.fixtures.environments import run_task, workspace_env +from tests.support import Reply, Turn, call, click, fake_screen, mcp_server, say, steps + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import HudEnv, Models + +MODEL = "gemini-3.1-pro-preview" +CONTROL_L = 0xFFE3 + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="k") + + +async def test_hosted_grounding_tools_are_offered_as_configured( + models: Models, tmp_path: Path +) -> None: + hosted = [GeminiGoogleSearchTool(), GeminiUrlContextTool(), GeminiCodeExecutionTool()] + models.script([say("done")]) + + await run_task( + workspace_env(tmp_path / "ws"), GeminiAgent(GeminiConfig(model=MODEL, hosted_tools=hosted)) + ) + + (request,) = models.requests() + assert [t for t in request.body["tools"] if "functionDeclarations" not in t] == [ + {"googleSearch": {}}, + {"urlContext": {}}, + {"codeExecution": {}}, + ] + + +async def test_citations_add_google_search_to_the_tools(models: Models, tmp_path: Path) -> None: + models.script([say("done")]) + + await run_task( + workspace_env(tmp_path / "ws"), + GeminiAgent(GeminiConfig(model=MODEL, citations_enabled=True)), + ) + + (request,) = models.requests() + assert [t for t in request.body["tools"] if "functionDeclarations" not in t] == [ + {"googleSearch": {}} + ] + + +async def test_excluded_computer_functions_are_left_out_of_the_computer_tool( + models: Models, tmp_path: Path +) -> None: + models.script([say("done")]) + excluded = ["drag_and_drop", "search"] + + async with fake_screen() as screen: + env = workspace_env(tmp_path / "ws", capabilities=(Capability.rfb(url=screen.url),)) + await run_task( + env, GeminiAgent(GeminiConfig(model=MODEL, excluded_predefined_functions=excluded)) + ) + + (request,) = models.requests() + (computer,) = [t for t in request.body["tools"] if "functionDeclarations" not in t] + assert computer["computerUse"]["excluded_predefined_functions"] == excluded + + +async def test_a_long_tool_output_is_sent_back_with_its_head_and_tail( + models: Models, tmp_path: Path +) -> None: + server = FastMCP("long") + server.tool(name="long_output", description="Return a long text.")( + lambda: "h" * 500 + "t" * 500 + ) + models.script([call("long_output"), say("done")]) + + async with mcp_server(server) as long_output: + env = workspace_env(tmp_path / "ws", capabilities=(long_output,)) + await run_task(env, GeminiAgent(GeminiConfig(model=MODEL, max_tool_result_chars=100))) + + (result,) = models.requests()[-1].tool_results + output = json.loads(result)["output"] + assert output.startswith("Output too large: showing the first 20 and last 80 characters") + assert output.endswith("h" * 20 + "\n\n... [900 characters omitted] ...\n\n" + "t" * 80) + + +async def test_only_the_three_newest_computer_turns_keep_their_screenshots( + models: Models, tmp_path: Path +) -> None: + models.script([click(10, 10, screen=(64, 48))] * 4 + [say("done")]) + + async with fake_screen() as screen: + env = workspace_env(tmp_path / "ws", capabilities=(Capability.rfb(url=screen.url),)) + await run_task(env, GeminiAgent(GeminiConfig(model=MODEL))) + + responses = [ + part["functionResponse"] + for content in models.requests()[-1].body["contents"] + for part in content["parts"] + if "functionResponse" in part + ] + assert [bool(response.get("parts")) for response in responses] == [False, True, True, True] + + +async def test_a_field_is_cleared_with_ctrl_a_whatever_system_the_agent_runs_on( + models: Models, tmp_path: Path +) -> None: + models.script([call("type_text_at", x=500, y=500, text="hi"), say("done")]) + + async with fake_screen() as screen: + env = workspace_env(tmp_path / "ws", capabilities=(Capability.rfb(url=screen.url),)) + await run_task(env, GeminiAgent(GeminiConfig(model=MODEL))) + + pressed = [event.key for event in screen.key_events() if event.down] + assert pressed[:2] == [CONTROL_L, ord("a")] + + +@pytest.mark.parametrize( + ("turn", "error"), + [ + (Turn(fail=Reply(json={"candidates": []})), "Gemini returned no candidates"), + (Turn(finish="MALFORMED_FUNCTION_CALL"), "Provider returned a malformed function call"), + ], + ids=["no-candidates", "a-malformed-function-call"], +) +async def test_a_reply_it_cannot_use_ends_the_run_with_its_error( + turn: Turn, error: str, models: Models, tmp_path: Path +) -> None: + models.queue([turn]) + + run = await run_task(workspace_env(tmp_path / "ws"), GeminiAgent(GeminiConfig(model=MODEL))) + + assert (run.trace.status, len(models.requests())) == ("error", 1) + assert error in (run.trace.error or "") + + +async def test_grounding_and_thoughts_land_on_the_agent_step( + models: Models, tmp_path: Path +) -> None: + web = "https://example.com/sky" + grounding = { + "groundingChunks": [ + {"web": {"uri": web, "title": "Sky"}}, + {"web": {"uri": "https://example.com/unused", "title": "Unused"}}, + ], + "groundingSupports": [ + { + "segment": {"text": "Blue.", "startIndex": 0, "endIndex": 5}, + "groundingChunkIndices": [0], + } + ], + } + models.script( + [Turn(text="Blue.", reasoning="look it up", extra={"groundingMetadata": grounding})] + ) + + run = await run_task(workspace_env(tmp_path / "ws"), GeminiAgent(GeminiConfig(model=MODEL))) + + (step,) = [step for step in steps(run.trace_id) if step["source"] == "agent"] + assert (step["content"], step["reasoning"]) == ("Blue.", "look it up") + assert [(c["type"], c["source"], c["text"]) for c in step["citations"]] == [ + ("grounding", web, "Blue."), + ("grounding", "https://example.com/unused", ""), + ] diff --git a/tests/agents/test_openai.py b/tests/agents/test_openai.py new file mode 100644 index 000000000..7f0cd62ac --- /dev/null +++ b/tests/agents/test_openai.py @@ -0,0 +1,202 @@ +"""What only ``OpenAIAgent`` does over the Responses API: the hosted tools and tool search it +offers, how it chains responses and discards an unusable one, how it reports a refused or +incomplete call to the model, and the citations it records.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest +from fastmcp import FastMCP + +from hud.agents import OpenAIAgent +from hud.agents.openai.tools import OpenAICodeInterpreterTool, OpenAIToolSearchTool +from hud.agents.types import OpenAIConfig +from tests.fixtures.environments import run_task, workspace_env +from tests.support import Turn, mcp_server, say, shell, shell_call, steps + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import HudEnv, Models + +MODEL = "gpt-5.6" + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="k") + + +async def test_a_hosted_code_interpreter_is_offered_as_configured( + models: Models, tmp_path: Path +) -> None: + hosted = OpenAICodeInterpreterTool(container={"type": "auto"}) + models.script([say("done")]) + + await run_task( + workspace_env(tmp_path / "ws"), + OpenAIAgent(OpenAIConfig(model=MODEL, hosted_tools=[hosted])), + ) + + (request,) = models.requests() + assert [t for t in request.body["tools"] if t["type"] == "code_interpreter"] == [ + {"type": "code_interpreter", "container": {"type": "auto"}} + ] + + +@pytest.mark.parametrize( + ("threshold", "deferred"), + [(1, {"database__lookup", "search__lookup"}), (2, set())], + ids=["over-its-threshold", "at-its-threshold"], +) +async def test_tool_search_defers_mcp_tools_once_there_are_more_than_its_threshold( + threshold: int, deferred: set[str], models: Models, tmp_path: Path +) -> None: + lookups = FastMCP("lookups") + lookups.tool(name="lookup", description="Look a key up.")(lambda key: key) + models.script([say("done")]) + agent = OpenAIAgent( + OpenAIConfig(model=MODEL, hosted_tools=[OpenAIToolSearchTool(threshold=threshold)]) + ) + + async with ( + mcp_server(lookups, name="database") as database, + mcp_server(lookups, name="search") as search, + ): + await run_task(workspace_env(tmp_path / "ws", capabilities=(database, search)), agent) + + (request,) = models.requests() + assert {tool["name"] for tool in request.body["tools"] if tool.get("defer_loading")} == deferred + + +async def test_citations_ask_for_web_search_sources(models: Models, tmp_path: Path) -> None: + models.script([say("done")]) + + await run_task( + workspace_env(tmp_path / "ws"), + OpenAIAgent(OpenAIConfig(model=MODEL, citations_enabled=True)), + ) + + (request,) = models.requests() + assert request.body["include"] == ["web_search_call.action.sources"] + + +async def test_each_request_sends_only_what_is_new_since_the_response_it_continues( + models: Models, tmp_path: Path +) -> None: + models.script([shell("echo hi"), say("done")]) + + await run_task(workspace_env(tmp_path / "ws"), OpenAIAgent(OpenAIConfig(model=MODEL))) + + first, second = (request.body for request in models.requests()) + assert (first.get("previous_response_id"), second["previous_response_id"]) == (None, "resp_1") + assert [item["type"] for item in second["input"]] == ["shell_call_output"] + + +async def test_an_empty_shell_call_is_sampled_again_without_advancing_the_chain( + models: Models, tmp_path: Path +) -> None: + models.queue([shell_call("true"), shell_call(), say("done")]) + + run = await run_task(workspace_env(tmp_path / "ws"), OpenAIAgent(OpenAIConfig(model=MODEL))) + + chain = [request.body.get("previous_response_id") for request in models.requests()] + assert (run.trace.content, chain) == ("done", [None, "resp_1", "resp_1"]) + + +async def test_a_shell_call_it_cannot_run_is_told_to_the_model( + models: Models, tmp_path: Path +) -> None: + refused = shell_call("true") + refused = Turn(native=({**refused.native[0], "action": {"commands": 5}},)) + models.script([refused, say("done")]) + + run = await run_task(workspace_env(tmp_path / "ws"), OpenAIAgent(OpenAIConfig(model=MODEL))) + + assert run.trace.status == "completed" + assert "commands must be a list of strings" in json.dumps(models.requests()[-1].body) + + +async def test_a_shell_calls_output_cap_reaches_the_model_with_its_output( + models: Models, tmp_path: Path +) -> None: + models.script([shell_call("seq 1 1000", max_output_length=50), say("done")]) + + await run_task(workspace_env(tmp_path / "ws"), OpenAIAgent(OpenAIConfig(model=MODEL))) + + (output,) = [i for i in models.requests()[-1].body["input"] if i["type"] == "shell_call_output"] + assert output["max_output_length"] == 50 + + +async def test_a_function_call_cut_off_mid_arguments_is_explained_to_the_model( + models: Models, tmp_path: Path +) -> None: + lookups = FastMCP("lookups") + lookups.tool(name="lookup", description="Look a key up.")(lambda key: key) + cut_off = Turn( + native=( + { + "type": "function_call", + "id": "fc_cut", + "call_id": "call_cut", + "name": "lookup", + "arguments": '{"key": "ab', + "status": "completed", + }, + ) + ) + models.script([cut_off, say("done")]) + + async with mcp_server(lookups) as server: + env = workspace_env(tmp_path / "ws", capabilities=(server,)) + run = await run_task(env, OpenAIAgent(OpenAIConfig(model=MODEL))) + + assert run.trace.status == "completed" + assert "the arguments for this 'lookup' call arrived incomplete" in json.dumps( + models.requests()[-1].body + ) + + +async def test_url_and_file_citations_land_on_the_agent_step( + models: Models, tmp_path: Path +) -> None: + web = "https://example.com/sky" + answer = { + "type": "message", + "id": "msg_cited", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Blue.", + "annotations": [ + { + "type": "url_citation", + "url": web, + "title": "Sky", + "start_index": 0, + "end_index": 5, + }, + { + "type": "file_citation", + "file_id": "file-1", + "filename": "notes.txt", + "index": 0, + }, + ], + } + ], + } + models.script([Turn(reasoning="look it up", native=(answer,))]) + + run = await run_task(workspace_env(tmp_path / "ws"), OpenAIAgent(OpenAIConfig(model=MODEL))) + + (step,) = [step for step in steps(run.trace_id) if step["source"] == "agent"] + assert (step["content"], step["reasoning"]) == ("Blue.", "look it up") + assert [(c["type"], c["source"], c["text"]) for c in step["citations"]] == [ + ("url_citation", web, "Sky"), + ("file_citation", "file-1", "notes.txt"), + ] diff --git a/tests/agents/test_provider_agents.py b/tests/agents/test_provider_agents.py new file mode 100644 index 000000000..87cf87a95 --- /dev/null +++ b/tests/agents/test_provider_agents.py @@ -0,0 +1,235 @@ +"""What every provider agent does over its own API: what its model is handed, how its tool +calls reach the environment, how its provider's failures end or retry a run, and where its +requests go with which credential. Each family in ``AgentType`` that calls a model API +itself is a row.""" + +from __future__ import annotations + +import asyncio +import json +import uuid +from typing import TYPE_CHECKING + +import pytest +from fastmcp import FastMCP + +from hud.agents.types import ToolStep +from hud.capabilities import Capability, RFBClient +from hud.eval import LocalRuntime, Task, Taskset +from hud.types import AgentType +from tests.fixtures.environments import run_task, workspace_env +from tests.support import Reply, Turn, click, fake_screen, mcp_server, say, shell + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import HudEnv, Models + +#: (agent type, model, the variable holding the provider's own key) +PROVIDERS = [ + ("claude", "claude-sonnet-4-6", "ANTHROPIC_API_KEY"), + ("openai", "gpt-5.6", "OPENAI_API_KEY"), + ("gemini", "gemini-3.1-pro-preview", "GEMINI_API_KEY"), + ("openai_compatible", "qwen3.6-plus", None), +] +EVERY_PROVIDER = pytest.mark.parametrize( + ("agent_type", "model", "key"), PROVIDERS, ids=[row[0] for row in PROVIDERS] +) + + +def test_every_family_that_calls_a_model_api_is_a_row() -> None: + assert {row[0] for row in PROVIDERS} == {kind.value for kind in AgentType if not kind.is_cli} + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="hud-key") + + +@EVERY_PROVIDER +async def test_the_model_is_given_the_prompt_a_shell_and_the_environments_tools( + agent_type: str, model: str, key: str | None, models: Models, tmp_path: Path +) -> None: + lookups = FastMCP("tools") + lookups.tool(name="lookup", description="Look a key up.")(lambda key: key) + models.script([say("done")]) + kind = AgentType(agent_type).cls + + async with mcp_server(lookups) as server: + env = workspace_env(tmp_path / "ws", capabilities=(server,)) + await run_task(env, kind(kind.config_cls(model=model)), prompt="Fix the bug.") + + (request,) = models.requests() + assert (request.prompt, "lookup" in request.tools) == ("Fix the bug.", True) + assert request.shell_tool is not None + + +@EVERY_PROVIDER +async def test_one_agent_drives_concurrent_rollouts_each_with_its_own_conversation( + agent_type: str, model: str, key: str | None, models: Models, tmp_path: Path +) -> None: + models.script([say("done")]) + kind = AgentType(agent_type).cls + env, agent = workspace_env(tmp_path / "ws"), kind(kind.config_cls(model=model)) + + runs = await asyncio.gather( + run_task(env, agent, prompt="first"), run_task(env, agent, prompt="second") + ) + + assert [run.trace.status for run in runs] == ["completed", "completed"] + assert sorted(request.prompt for request in models.requests()) == ["first", "second"] + + +@EVERY_PROVIDER +async def test_a_shell_call_acts_on_the_workspace_and_its_result_reaches_the_model( + agent_type: str, model: str, key: str | None, models: Models, tmp_path: Path +) -> None: + models.script([shell("touch made && echo made-it"), say("done")]) + kind = AgentType(agent_type).cls + + run = await run_task(workspace_env(tmp_path / "ws"), kind(kind.config_cls(model=model))) + + assert (run.trace.status, run.trace.content) == ("completed", "done") + assert (tmp_path / "ws" / "made").exists() + assert sum(isinstance(step, ToolStep) for step in run.trace.steps) == 1 + assert "made-it" in json.dumps(models.requests()[-1].body) + + +#: (agent type, model) for the families whose tools drive a screen +COMPUTER_USERS = [ + ("claude", "claude-sonnet-4-6"), + ("openai", "gpt-5.6"), + ("gemini", "gemini-3.1-pro-preview"), +] + + +def test_every_family_that_drives_a_screen_is_a_computer_use_row() -> None: + assert {row[0] for row in COMPUTER_USERS} == { + kind.value for kind in AgentType if RFBClient in getattr(kind.cls, "clients", ()) + } + + +@pytest.mark.parametrize( + ("agent_type", "model"), COMPUTER_USERS, ids=[r[0] for r in COMPUTER_USERS] +) +async def test_a_click_lands_on_the_screen_where_the_model_pointed( + agent_type: str, model: str, models: Models, tmp_path: Path +) -> None: + models.script([click(150, 25, screen=(200, 100)), say("done")]) + kind = AgentType(agent_type).cls + + async with fake_screen(width=200, height=100) as screen: + env = workspace_env(tmp_path / "ws", capabilities=(Capability.rfb(url=screen.url),)) + run = await run_task(env, kind(kind.config_cls(model=model))) + + pressed = {(event.x, event.y) for event in screen.pointer_events() if event.buttons & 1} + assert (run.trace.status, pressed) == ("completed", {(150, 25)}) + assert "image" in json.dumps(models.requests()[-1].body) + + +@EVERY_PROVIDER +async def test_a_reply_cut_at_the_token_cap_ends_the_run_as_truncated( + agent_type: str, model: str, key: str | None, models: Models, tmp_path: Path +) -> None: + models.script([Turn(text="part", truncated=True)]) + kind = AgentType(agent_type).cls + + run = await run_task(workspace_env(tmp_path / "ws"), kind(kind.config_cls(model=model))) + + assert (run.trace.status, run.trace.stop_reason, run.trace.content) == ( + "completed", + "length", + "part", + ) + + +@EVERY_PROVIDER +async def test_a_provider_fault_ends_the_run_with_its_error( + agent_type: str, model: str, key: str | None, models: Models, tmp_path: Path +) -> None: + models.script([Turn(fail=Reply(status=400, json={"error": {"message": "model unavailable"}}))]) + kind = AgentType(agent_type).cls + + run = await run_task(workspace_env(tmp_path / "ws"), kind(kind.config_cls(model=model))) + + assert run.trace.status == "error" + assert "model unavailable" in (run.trace.error or "") + + +@EVERY_PROVIDER +async def test_an_overloaded_provider_is_asked_again( + agent_type: str, model: str, key: str | None, models: Models, tmp_path: Path +) -> None: + busy = Reply(status=503, headers={"retry-after-ms": "0"}, json={"error": {"message": "busy"}}) + models.queue([Turn(fail=busy), say("done")]) + kind = AgentType(agent_type).cls + + run = await run_task(workspace_env(tmp_path / "ws"), kind(kind.config_cls(model=model))) + + assert (run.trace.status, run.trace.content, len(models.requests())) == ( + "completed", + "done", + 2, + ) + + +@pytest.mark.parametrize( + ("agent_type", "model", "key"), + [row for row in PROVIDERS if row[2] is not None], + ids=[row[0] for row in PROVIDERS if row[2] is not None], +) +@pytest.mark.parametrize( + ("own_key", "gateway", "route", "credential"), + [ + ("own-key", False, "provider", "own-key"), + (None, False, "gateway", "hud-key"), + ("own-key", True, "gateway", "hud-key"), + ], + ids=["its-own-key", "only-a-hud-key", "the-gateway-forced-over-its-own-key"], +) +async def test_requests_go_to_the_provider_with_its_own_key_and_otherwise_to_the_gateway( + agent_type: str, + model: str, + key: str, + own_key: str | None, + gateway: bool, + route: str, + credential: str, + models: Models, + provider: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set( + OPENAI_BASE_URL=provider.url, + ANTHROPIC_BASE_URL=provider.url, + GOOGLE_GEMINI_BASE_URL=provider.url, + **{key: own_key}, + ) + models.script([say("done")]) + provider.script([say("done")]) + kind = AgentType(agent_type).cls + + run = await run_task( + workspace_env(tmp_path / "ws"), kind(kind.config_cls(model=model, gateway=gateway)) + ) + + (request,) = {"gateway": models, "provider": provider}[route].requests() + assert (run.trace.status, request.credential) == ("completed", credential) + assert ("trace-id" in request.headers) is (route == "gateway") + + +@EVERY_PROVIDER +async def test_each_gateway_request_carries_its_own_runs_trace_id( + agent_type: str, model: str, key: str | None, models: Models, tmp_path: Path +) -> None: + models.script([say("done")]) + env = workspace_env(tmp_path / "ws") + kind = AgentType(agent_type).cls + + job = await Taskset("t", [Task(env=env.name, id="task")]).run( + kind(kind.config_cls(model=model)), runtime=LocalRuntime(env), group=3 + ) + + sent = sorted(uuid.UUID(request.headers["trace-id"]).hex for request in models.requests()) + assert sent == sorted(uuid.UUID(str(run.trace_id)).hex for run in job.runs) diff --git a/tests/agents/test_tool_loop.py b/tests/agents/test_tool_loop.py new file mode 100644 index 000000000..369eb97f6 --- /dev/null +++ b/tests/agents/test_tool_loop.py @@ -0,0 +1,310 @@ +"""What the provider agents' shared tool loop decides: what the model is told when a call +cannot run, when a run stops, how long a tool may take, and how MCP tools are named. + +The loop is one implementation every provider agent runs, so each rule is checked once, +on the chat completions agent; what each provider does differently is +``test_provider_agents``.""" + +from __future__ import annotations + +import asyncio +import json +import sys +from contextlib import AsyncExitStack +from dataclasses import replace +from typing import TYPE_CHECKING +from urllib.parse import urlsplit + +import pytest +from fastmcp import FastMCP + +from hud.agents import ClaudeAgent, OpenAIChatAgent +from hud.agents.types import ClaudeConfig, OpenAIChatConfig +from tests.fixtures.environments import run_task, workspace_env +from tests.support import Turn, call, fail, mcp_server, relay, say, shell + +if TYPE_CHECKING: + from pathlib import Path + + from hud.types import StopCondition + from tests.support import FlakyRelay, HudEnv, ModelRequest, Models + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="k") + + +def failing_tools() -> FastMCP: + server = FastMCP("tools") + + @server.tool + def explode() -> str: + """Fail.""" + raise ValueError("kaboom") + + return server + + +@pytest.mark.parametrize( + ("turn", "told"), + [ + pytest.param(call("ghost"), "unknown tool: 'ghost'", id="an-unknown-tool"), + pytest.param( + call("explode"), "Error calling tool 'explode': kaboom", id="an-mcp-tool-that-fails" + ), + ], +) +async def test_a_call_that_cannot_run_is_told_to_the_model_and_the_run_goes_on( + turn: Turn, told: str, models: Models, tmp_path: Path +) -> None: + models.script([turn, say("done")]) + + async with mcp_server(failing_tools()) as server: + env = workspace_env(tmp_path / "ws", capabilities=(server,)) + run = await run_task(env, OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus"))) + + assert (run.trace.status, run.trace.content) == ("completed", "done") + assert told in json.dumps(models.requests()[-1].body) + + +CUT_OFF = Turn( + native=( + { + "id": "call_cut", + "type": "function", + "function": {"name": "bash", "arguments": '{"command": "ec'}, + }, + ) +) + + +@pytest.mark.parametrize( + ("reason", "turn"), + [ + pytest.param("length", replace(shell("true"), truncated=True), id="a-capped-reply"), + pytest.param("malformed_tool_call", CUT_OFF, id="a-malformed-call"), + ], +) +async def test_stop_on_ends_the_run_at_the_call_it_names( + reason: StopCondition, turn: Turn, models: Models, tmp_path: Path +) -> None: + models.script([turn]) + + run = await run_task( + workspace_env(tmp_path / "ws"), + OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus", stop_on={reason})), + ) + + assert (run.trace.status, run.trace.stop_reason) == ("completed", reason) + assert len(models.requests()) == 1 + + +async def test_the_step_budget_ends_a_model_that_never_finishes( + models: Models, tmp_path: Path +) -> None: + models.respond(lambda request: shell("true")) + + run = await run_task( + workspace_env(tmp_path / "ws"), + OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus", max_steps=2)), + ) + + assert (run.trace.status, run.trace.stop_reason, len(models.requests())) == ( + "completed", + "max_steps", + 2, + ) + + +@pytest.mark.parametrize( + ("calls", "status"), + [ + pytest.param(["shell", "shell", "shell"], "error", id="three-lost-calls-in-a-row"), + pytest.param( + ["shell", "ghost", "shell", "shell"], "completed", id="another-result-resets-the-count" + ), + ], +) +async def test_a_lost_workspace_ends_the_run_after_three_consecutive_failures( + calls: list[str], status: str, models: Models, tmp_path: Path +) -> None: + turns = [shell("true")] + [shell("true") if name == "shell" else call(name) for name in calls] + env = workspace_env(tmp_path / "ws") + relays: list[FlakyRelay] = [] + + async with AsyncExitStack() as stack: + + @env.initialize + async def reach_the_workspace_through_a_relay() -> None: + shell = env.capability("shell") + address = urlsplit(shell.url) + assert address.hostname is not None and address.port is not None + workspace = await stack.enter_async_context(relay(address.hostname, address.port)) + relays.append(workspace) + env.add_capability(replace(shell, url=f"ssh://127.0.0.1:{workspace.port}")) + + def answer(request: ModelRequest) -> Turn: + if request.turn == 1: + (workspace,) = relays + workspace.refuse(sys.maxsize) + workspace.sever() + return turns[request.turn] if request.turn < len(turns) else say("done") + + models.respond(answer) + run = await run_task( + env, OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus", max_steps=10)) + ) + + assert run.trace.status == status + if status == "error": + assert run.trace.error == "SSH tool failure limit reached after 3 consecutive errors" + + +RESPONDER = "You are an assistant that helps determine" + + +@pytest.mark.parametrize( + ("decision", "content"), + [ + pytest.param(say("CONTINUE"), "finished", id="continue"), + pytest.param(say("STOP"), "Shall I go on?", id="stop"), + pytest.param(fail(400), "Shall I go on?", id="the-responder-fails"), + ], +) +async def test_auto_respond_answers_a_question_or_lets_the_run_end( + decision: Turn, content: str, models: Models, tmp_path: Path +) -> None: + """The auto-responder is its own Responses call beside the agent's own requests.""" + decisions = iter([decision, say("STOP")]) + replies = [say("Shall I go on?"), say("finished")] + models.respond( + lambda request: ( + next(decisions) + if str(request.body.get("instructions", "")).startswith(RESPONDER) + else replies[request.turn] + ) + ) + + run = await run_task( + workspace_env(tmp_path / "ws"), + OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus", auto_respond=True)), + ) + + assert run.trace.content == content + + +# ─── the tool timeout ────────────────────────────────────────────────── + + +def slow_tools() -> FastMCP: + server = FastMCP("slow") + + @server.tool() + async def wait() -> str: + """Answer after two seconds.""" + await asyncio.sleep(2.0) + return "waited" + + return server + + +async def test_a_shell_call_past_the_tool_timeout_is_cut_off_and_the_run_goes_on( + models: Models, tmp_path: Path +) -> None: + root = tmp_path / "ws" + models.script([shell("sleep 3; touch late"), shell("touch quick"), say("done")]) + + run = await run_task( + workspace_env(root), + OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus", tool_timeout_seconds=1)), + ) + await asyncio.sleep(3) + + assert run.trace.status == "completed" + assert "timed out after 1s" in json.dumps(models.requests()[1].body) + assert ((root / "late").exists(), (root / "quick").exists()) == (False, True) + + +async def test_an_mcp_tool_is_left_to_its_own_time(models: Models, tmp_path: Path) -> None: + models.script([call("wait"), say("done")]) + + async with mcp_server(slow_tools()) as server: + env = workspace_env(tmp_path / "ws", capabilities=(server,)) + run = await run_task( + env, OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus", tool_timeout_seconds=1)) + ) + + assert run.trace.status == "completed" + assert "waited" in json.dumps(models.requests()[-1].body) + + +async def test_a_workspace_tool_blocked_on_its_file_is_cut_off_by_the_tool_timeout( + models: Models, tmp_path: Path +) -> None: + """Claude's editor reads the file itself, so a fifo blocks it rather than the shell.""" + root = tmp_path / "ws" + edit = call( + "str_replace_based_edit_tool", + command="str_replace", + path=f"{root}/pipe", + old_str="a", + new_str="b", + ) + models.script([call("bash", command="mkfifo pipe"), edit, say("done")]) + + run = await run_task( + workspace_env(root), + ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6", tool_timeout_seconds=1)), + ) + + assert run.trace.status == "completed" + assert "str_replace_based_edit_tool timed out after 1s" in json.dumps( + models.requests()[-1].body + ) + + +# ─── MCP tool names ──────────────────────────────────────────────────── + + +def named(name: str, *tools: str) -> FastMCP: + server = FastMCP(name) + for tool in tools: + server.tool(name=tool, description=f"Run {tool}.")(lambda: tool) + return server + + +async def test_mcp_tools_from_several_servers_are_qualified_within_64_characters( + models: Models, tmp_path: Path +) -> None: + models.script([say("done")]) + long = "lookup_" * 10 + + async with ( + mcp_server(named("company.tools", long + "a"), name="company.tools") as dotted, + mcp_server(named("company-tools", long + "b"), name="company-tools") as dashed, + ): + env = workspace_env(tmp_path / "ws", capabilities=(dotted, dashed)) + await run_task(env, OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus"))) + + (request,) = models.requests() + qualified = [name for name in request.tools if long[:20] in name] + assert len(qualified) == 2 and all(len(name) <= 64 for name in qualified) + + +async def test_two_mcp_tools_qualified_to_one_name_fail_the_run( + models: Models, tmp_path: Path +) -> None: + models.script([say("done")]) + + async with ( + mcp_server(named("a", "b__c"), name="a") as first, + mcp_server(named("a__b", "c"), name="a__b") as second, + ): + env = workspace_env(tmp_path / "ws", capabilities=(first, second)) + run = await run_task(env, OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus"))) + + assert run.trace.error == ( + "[agent loop] ValueError: MCP tool name collision after qualification: 'a__b__c'" + ) + assert models.requests() == [] diff --git a/hud/cli/tests/__init__.py b/tests/cli/__init__.py similarity index 100% rename from hud/cli/tests/__init__.py rename to tests/cli/__init__.py diff --git a/tests/cli/commands.py b/tests/cli/commands.py new file mode 100644 index 000000000..6a13922f2 --- /dev/null +++ b/tests/cli/commands.py @@ -0,0 +1,321 @@ +"""The ``hud`` command tree as the rows of the CLI contracts. + +Every invocable command is registered here twice over: by how it reaches the +platform (``PLATFORM`` with an invocation that calls it, ``OFFLINE`` with one that +fails without calling anything) and by what it changes (``MUTATING``, +``LOCAL_ONLY``, ``READ_ONLY``). ``test_shell`` holds the registry to the tree, so +a new command fails the suite until it has its rows. + +A setup takes the ``hud`` runner and the platform and puts in place what an +invocation needs: files in the working directory, platform records, routes. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import typer +from typer.core import TyperGroup + +from hud.cli.__main__ import app +from tests.support import Reply, write_files +from tests.support.platform import Import + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import Hud, HudEnv, Platform + +JOB_ID = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" +TRACE_ID = "00000000-0000-4000-a000-000000000002" +IMPORT_ID = "66666666-6666-4666-8666-666666666666" +TB2_ID = "eeeeeeee-0000-4000-8000-999999999999" +ENV_ID = "bbbbbbbb-0000-4000-8000-000000000042" +BASE_MODEL_ID = "cccccccc-0000-4000-8000-000000000101" +MODEL_ID = "cccccccc-0000-4000-8000-000000000102" +TASKS_PY = """\ +from hud import Environment + +env = Environment("example") + + +@env.template(id="solve") +async def solve(n: int = 0): + yield f"solve {n}" + yield 1.0 + + +tasks = [solve(n=1)] +""" + +Setup = Callable[["Hud", "Platform"], object] + + +def nothing(hud: Hud, platform: Platform) -> None: + del hud, platform + + +def files(contents: dict[str, str]) -> Setup: + """Write ``contents`` under the working directory; ``->target`` makes a symlink.""" + + def setup(hud: Hud, platform: Platform) -> None: + del platform + for name, content in contents.items(): + if content.startswith("->"): + path = hud.cwd / name + path.parent.mkdir(parents=True, exist_ok=True) + path.symlink_to(hud.cwd / content.removeprefix("->")) + else: + write_files(hud.cwd, {name: content}) + + return setup + + +def link(stored: object) -> Setup: + """A directory link (``.hud/config.json``) holding ``stored``, written as JSON.""" + return files({".hud/config.json": stored if isinstance(stored, str) else json.dumps(stored)}) + + +def refused(method: str, path: str, status: int) -> Setup: + def setup(hud: Hud, platform: Platform) -> None: + del hud + platform.services.route("api", method, path, Reply(status=status, json={"detail": "No"})) + + return setup + + +def together(*setups: Setup) -> Setup: + def setup(hud: Hud, platform: Platform) -> None: + for each in setups: + each(hud, platform) + + return setup + + +environment = files( + { + "env.py": 'from hud import Environment\nenv = Environment("example")\n', + "Dockerfile.hud": "FROM python:3.12\n", + } +) +tasks = files({"tasks.py": TASKS_PY}) +harbor_folders = files( + { + "tasks/hello/task.toml": "version = '1.0'\n", + "tasks/hello/tests/test.sh": "exit 0\n", + "run-1/config.json": "{}", + "run-1/result.json": "{}", + } +) + + +def a_platform_taskset(hud: Hud, platform: Platform) -> None: + del hud + platform.add_taskset("Demo", [{"name": "one", "env": "demo", "scenario": "solve"}]) + + +def linked_taskset_with_a_change(hud: Hud, platform: Platform) -> None: + tasks(hud, platform) + assert hud("sync", "tasks", "demo", "tasks.py", "--yes").exit_code == 0 + (hud.cwd / "tasks.py").write_text(TASKS_PY.replace("solve(n=1)", "solve(n=1), solve(n=2)")) + + +def a_registered_environment(hud: Hud, platform: Platform) -> str: + del hud + return platform.add_environment("example", id=ENV_ID) + + +def a_running_import(hud: Hud, platform: Platform) -> None: + del hud + taskset = platform.add_taskset("tb2", id=TB2_ID) + platform.imports[IMPORT_ID] = Import( + IMPORT_ID, taskset, statuses=["running"], report={"schema_version": "v9"} + ) + + +def a_qa_check(hud: Hud, platform: Platform) -> None: + del hud + platform.qa_checks = [{"key": "failure_analysis", "subject_type": "trace", "title": "FA"}] + + +def a_job(hud: Hud, platform: Platform) -> None: + del hud + platform.add_job([{"id": TRACE_ID, "status": "completed"}], id=JOB_ID) + + +def a_trace(hud: Hud, platform: Platform) -> None: + del hud + platform.trace(TRACE_ID).events = [{"kind": "agent_message", "text": "done"}] + + +def a_base_model(hud: Hud, platform: Platform) -> str: + del hud + return platform.add_model( + id=BASE_MODEL_ID, + name="Claude Sonnet", + model_name="claude-sonnet-4-6", + sdk_agent_type="claude", + provider={"name": "anthropic"}, + ) + + +def a_trainable_model(hud: Hud, platform: Platform) -> str: + del hud + model_id = platform.add_model( + id=MODEL_ID, + name="owner/model + version", + model_name="team/my-sonnet", + sdk_agent_type="claude", + is_trainable=True, + provider={"name": "anthropic"}, + ) + platform.checkpoints[model_id] = [ + {"id": "ckpt-1", "created_at": "2026-09-01T00:00:00Z"}, + {"id": "ckpt-2", "name": "step-1", "created_at": "2026-09-02T00:00:00Z", "is_active": True}, + ] + return model_id + + +@dataclass(frozen=True) +class Command: + """An invocation that reaches the platform, and the state it needs first.""" + + argv: list[str] + setup: Setup = nothing + + +PLATFORM: dict[str, Command] = { + "deploy": Command(["deploy", "--no-env"], environment), + "eval": Command(["eval", "Demo", "openai_compatible", "-m", "scripted"], a_platform_taskset), + "cancel": Command(["cancel", JOB_ID, "--trace-id", TRACE_ID]), + "project": Command(["project"]), + "project list": Command(["project", "list"]), + "project create": Command(["project", "create", "UI Evals"]), + "project use": Command(["project", "use", "browser-evals"]), + "sync": Command(["sync"], linked_taskset_with_a_change), + "sync tasks": Command(["sync", "tasks", "demo", "tasks.py"], tasks), + "sync env": Command(["sync", "env", "example"], a_registered_environment), + "harbor import": Command( + ["harbor", "import", "tasks", "run-1", "--taskset", "tb2"], harbor_folders + ), + "harbor imports": Command(["harbor", "imports", "--taskset", "tb2"], a_running_import), + "harbor report": Command(["harbor", "report", IMPORT_ID], a_running_import), + "harbor cancel": Command(["harbor", "cancel", IMPORT_ID], a_running_import), + "qa": Command(["qa"], a_qa_check), + "qa list": Command(["qa", "list"], a_qa_check), + "qa run": Command(["qa", "run", "failure_analysis", TRACE_ID], a_qa_check), + "qa results": Command(["qa", "results", TRACE_ID]), + "jobs": Command(["jobs"]), + "jobs list": Command(["jobs", "list"]), + "jobs get": Command(["jobs", "get", JOB_ID], a_job), + "jobs cancel": Command(["jobs", "cancel", JOB_ID]), + "trace get": Command(["trace", "get", TRACE_ID], a_trace), + "models list": Command(["models", "list"], a_base_model), + "models fork": Command(["models", "fork", "claude-sonnet-4-6", "--name", "mine"], a_base_model), + "models checkpoints": Command(["models", "checkpoints", "team/my-sonnet"], a_trainable_model), + "models head": Command( + ["models", "head", "team/my-sonnet", "--set", "ckpt-1"], a_trainable_model + ), +} +OFFLINE: dict[str, list[str]] = { + "init": ["init", "thing", "--preset", "no-such-preset"], + "serve": ["serve", "missing.py"], + "set": ["set", "NOT_A_PAIR"], + "version": ["version", "--no-such-option"], + "task list": ["task", "list", "--source", "missing.json"], + "task start": ["task", "start", "solve", "--source", "missing.json"], + "task grade": ["task", "grade", "solve", "--source", "missing.json", "--answer", "x"], +} + +MUTATING = { + "deploy", + "eval", + "project create", + "sync", + "sync tasks", + "harbor import", + "harbor cancel", + "qa run", + "jobs cancel", + "cancel", + "models fork", + "models head", +} +LOCAL_ONLY = {"init", "project use", "sync env", "set", "serve", "task start", "task grade"} +READ_ONLY = { + "version", + "task list", + "project", + "project list", + "harbor imports", + "harbor report", + "qa", + "qa list", + "qa results", + "jobs", + "jobs list", + "jobs get", + "trace get", + "models list", + "models checkpoints", +} + + +def tree(command: object, path: tuple[str, ...] = ()) -> dict[str, object]: + """Every invocable command by path; a group that runs on its own counts as one.""" + group = isinstance(command, TyperGroup) + runs = path and (not group or getattr(command, "invoke_without_command", False)) + found = {" ".join(path): command} if runs else {} + if isinstance(command, TyperGroup): + for name, sub in command.commands.items(): + found |= tree(sub, (*path, name)) + return found + + +TREE = tree(typer.main.get_command(app)) +EVERY_COMMAND = sorted(TREE) + + +def invocation(name: str) -> list[str]: + """The registered platform invocation, confirmed when the command asks first.""" + return [*PLATFORM[name].argv, *(["--yes"] if name in MUTATING else [])] + + +def failing(name: str, hud: Hud, platform: Platform, hud_env: HudEnv) -> list[str]: + """An invocation of ``name`` that fails: offline, its registered one; else signed out.""" + if name in OFFLINE: + return OFFLINE[name] + PLATFORM[name].setup(hud, platform) + hud_env.set(HUD_API_KEY=None) + return invocation(name) + + +def answer_everything(platform: Platform, reply: Reply) -> None: + """Every platform API route answers ``reply``.""" + for method in ("GET", "POST", "PUT", "DELETE"): + platform.services.route("api", method, "/{rest:path}", reply) + + +def state(hud: Hud, hud_env: HudEnv) -> dict[Path, bytes | str]: + """Every file under the working directory and the user's ``~/.hud``; a symlink as its + target.""" + roots = (hud.cwd, hud_env.home / ".hud") + return { + path: str(path.readlink()) if path.is_symlink() else path.read_bytes() + for root in roots + if root.exists() + for path in root.rglob("*") + if (path.is_file() or path.is_symlink()) and "__pycache__" not in path.parts + } + + +def writes(platform: Platform) -> list[str]: + """The platform requests that could change something: every API call but a read.""" + return [ + f"{request.method} {request.path}" + for request in platform.services.requests() + if request.method != "GET" and request.service != "telemetry" + ] diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py new file mode 100644 index 000000000..7346dea16 --- /dev/null +++ b/tests/cli/conftest.py @@ -0,0 +1,27 @@ +"""The ``hud`` runner the CLI scenarios use, and CLI analytics.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from tests.support import Hud + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import HudEnv, Platform + + +@pytest.fixture +def hud(tmp_path: Path) -> Hud: + """The ``hud`` fixture at a terminal width that keeps rendered rows on one line.""" + return Hud(tmp_path / "work", columns=96) + + +@pytest.fixture +def analytics(platform: Platform, hud_env: HudEnv) -> Platform: + """CLI analytics on, posting to the platform.""" + hud_env.set(HUD_CLI_ANALYTICS_ENABLED="1", CI=None) + return platform diff --git a/tests/cli/test_changes.py b/tests/cli/test_changes.py new file mode 100644 index 000000000..6e661281c --- /dev/null +++ b/tests/cli/test_changes.py @@ -0,0 +1,333 @@ +"""What a ``hud`` command may change: nothing until confirmed, nothing on a dry run, and +nothing when it fails. + +State is every file under the working directory and the user's ``~/.hud``, and every +platform request that could write. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Literal + +import pytest + +from tests.support.platform import BROWSER_PROJECT_ID, LOCKED_PROJECT_ID, TEAM_ID, USER_ID, Import + +from .commands import ( + MUTATING, + PLATFORM, + TRACE_ID, + Setup, + a_base_model, + a_platform_taskset, + a_registered_environment, + environment, + files, + harbor_folders, + link, + linked_taskset_with_a_change, + nothing, + refused, + state, + tasks, + together, + writes, +) + +if TYPE_CHECKING: + from tests.support import Hud, HudEnv, Platform + + +@pytest.mark.parametrize("name", sorted(MUTATING)) +@pytest.mark.parametrize("flag", ["", "--dry-run"], ids=["unconfirmed", "dry-run"]) +def test_a_mutating_command_changes_nothing_unless_confirmed( + hud: Hud, platform: Platform, hud_env: HudEnv, name: str, flag: str +) -> None: + command = PLATFORM[name] + command.setup(hud, platform) + before, sent = state(hud, hud_env), len(writes(platform)) + + result = hud(*command.argv, *filter(None, [flag]), "--json") + + if flag: + assert (result.exit_code, result.json["dry_run"]) == (0, True), result + else: + assert (result.exit_code, result.json["error"]) == (2, "usage"), result + assert "--yes" in result.json["suggestion"] + assert writes(platform)[sent:] == [] + assert state(hud, hud_env) == before + + +# ─── failures ─────────────────────────────────────────────────────────── + +OTHER_ID = "cccccccc-0000-4000-8000-000000000001" +EXAMPLE_ENV = 'from hud import Environment\nenv = Environment("example")\n' +HARBOR_TASK = {"task.toml": "version = '1.0'\n", "environment/Dockerfile": "FROM ubuntu\n"} + + +def scoped(user: str, team: str, *, elsewhere: bool = False) -> Setup: + """A link made by ``user`` in ``team``, on this platform or another one.""" + + def setup(hud: Hud, platform: Platform) -> None: + origin = "https://api.example" if elsewhere else platform.services.url("api") + scope = {"origin": origin.removesuffix("/api"), "user_id": user, "team_id": team} + link({"version": 1, "scope": scope, "project_id": BROWSER_PROJECT_ID})(hud, platform) + + return setup + + +def uneditable_taskset(hud: Hud, platform: Platform) -> None: + del hud + platform.add_taskset("theirs", can_edit=False) + + +def small_import_limit(hud: Hud, platform: Platform) -> None: + del hud + platform.import_formats[0]["max_input_bytes"] = 10 + + +def an_import_of_another_format(hud: Hud, platform: Platform) -> None: + del hud + platform.imports["imp"] = Import("imp", "tb2", format="parquet") + + +def trace_and_task_checks(hud: Hud, platform: Platform) -> None: + del hud + platform.qa_checks = [ + {"key": "failure_analysis", "subject_type": "trace", "title": "FA"}, + {"key": "task_reward_exploit", "subject_type": "task", "title": "TRE"}, + ] + + +def a_deleted_taskset(hud: Hud, platform: Platform) -> None: + tasks(hud, platform) + assert hud("sync", "tasks", "demo", "tasks.py", "--yes").exit_code == 0 + platform.tasksets.clear() + + +def demo_taskset(hud: Hud, platform: Platform) -> None: + del hud + platform.add_taskset("demo") + + +@dataclass(frozen=True) +class Failure: + """An invocation that fails, and what it may have asked the platform before failing. + + ``reaches`` is ``nothing`` (no request at all), ``reads`` (no request that could + write) or ``the refusal`` (the platform refused the write the command sent). Unless + ``linked`` is false the directory starts linked to a project, so a failure that + rewrote the link would show. + """ + + argv: list[str] + setup: Setup = nothing + exit_code: int = 1 + error: str = "failure" + reaches: Literal["nothing", "reads", "the refusal"] = "reads" + env: dict[str, str | None] = field(default_factory=dict) + mentions: str = "" + linked: bool = True + + +FAILURES: dict[str, Failure] = { + "deploy: no environment declared": Failure(["deploy", "--yes"], exit_code=2, error="usage"), + "deploy: several environments and no name": Failure( + ["deploy", "--yes"], + files({"a.py": 'a = Environment("a")\n', "b.py": 'b = Environment("b")\n'}), + exit_code=2, + error="usage", + ), + "deploy: a read-only project": Failure( + ["deploy", "--project", LOCKED_PROJECT_ID, "--yes"], + environment, + error="permission_denied", + ), + "deploy: the build is refused": Failure( + ["deploy", "--no-env", "--yes"], + together(environment, refused("POST", "/v2/builds/trigger", 409)), + error="conflict", + reaches="the refusal", + ), + "eval: an unknown setting": Failure( + ["eval", "tasks.py", "openai", "--yes"], + together(tasks, files({".hud_eval.toml": '[eval]\nmodle = "x"\n'})), + exit_code=2, + error="usage", + reaches="nothing", + ), + "eval: an unset variable": Failure( + ["eval", "tasks.py", "openai", "--yes"], + together(tasks, files({".hud_eval.toml": '[openai]\nmodel = "${HUD_EVAL_UNSET}"\n'})), + exit_code=2, + error="usage", + reaches="nothing", + ), + "eval: no source found": Failure(["eval", "--yes"], error="not_found", reaches="nothing"), + "eval: the hud runtime signed out": Failure( + ["eval", "tasks.py", "openai", "--runtime", "hud", "--yes"], + tasks, + error="permission_denied", + reaches="nothing", + env={"HUD_API_KEY": None}, + ), + "eval: a platform taskset without container rows run locally": Failure( + ["eval", "Demo", "openai", "--runtime", "local", "--yes"], + a_platform_taskset, + exit_code=2, + error="usage", + ), + "init: a non-empty directory": Failure( + ["init", "taken", "--preset", "blank"], + files({"taken/precious.txt": "data"}), + error="conflict", + reaches="nothing", + ), + "init: a symlink inside the target": Failure( + ["init", "project", "--preset", "blank", "--force"], + files({"outside.py": "original", "project/env.py": "->outside.py"}), + reaches="nothing", + ), + "init: a symlinked target": Failure( + ["init", "project", "--preset", "blank"], + files({"outside/": "", "project": "->outside"}), + reaches="nothing", + ), + "harbor import: a taskset the caller cannot edit": Failure( + ["harbor", "import", "tasks", "run-1", "--taskset", "theirs", "--yes"], + together(harbor_folders, uneditable_taskset), + error="permission_denied", + ), + "harbor import: a bundle over the platform limit": Failure( + ["harbor", "import", "tasks", "run-1", "--taskset", "tb2", "--yes"], + together(harbor_folders, small_import_limit), + exit_code=2, + error="usage", + ), + "harbor import: two folders with one name": Failure( + ["harbor", "import", "a/hello", "b/hello", "--yes"], + files( + {f"{root}/hello/{name}": text for root in "ab" for name, text in HARBOR_TASK.items()} + ), + exit_code=2, + error="usage", + ), + "harbor import: the taskset is refused": Failure( + ["harbor", "import", "tasks", "run-1", "--taskset", "tb2", "--link", "--yes"], + together(harbor_folders, refused("POST", "/v2/tasksets", 403)), + error="permission_denied", + reaches="the refusal", + ), + "harbor report: an import of another format": Failure( + ["harbor", "report", "imp"], an_import_of_another_format, exit_code=2, error="usage" + ), + "sync: nothing linked": Failure(["sync"], tasks, exit_code=2, error="usage"), + "sync: a missing source": Failure( + ["sync", "tasks", "demo", "missing.json", "--yes"], error="not_found" + ), + "sync: a read-only project": Failure( + ["sync", "tasks", "demo", "tasks.py", "--project", LOCKED_PROJECT_ID, "--yes"], + together(tasks, demo_taskset), + error="permission_denied", + ), + "sync: a stored taskset that was deleted": Failure( + ["sync", "tasks", "--force", "--yes"], a_deleted_taskset, error="not_found" + ), + "sync: the upload is refused": Failure( + ["sync", "--yes"], + together(linked_taskset_with_a_change, refused("POST", "/v2/tasks/upload", 400)), + reaches="the refusal", + ), + "sync tasks: the upload is refused": Failure( + ["sync", "tasks", "demo", "tasks.py", "--yes"], + together(tasks, refused("POST", "/v2/tasks/upload", 400)), + reaches="the refusal", + ), + "sync env: an unknown environment": Failure( + ["sync", "env", "bbbbbbbb-0000-4000-8000-999999999999"], + a_registered_environment, + error="not_found", + ), + "jobs cancel: no target": Failure( + ["jobs", "cancel", "--yes"], exit_code=2, error="usage", reaches="nothing" + ), + "qa run: trace and task checks together": Failure( + ["qa", "run", "failure_analysis,task_reward_exploit", TRACE_ID, "--yes"], + trace_and_task_checks, + exit_code=2, + error="usage", + ), + "task start: an id several rows share": Failure( + ["task", "start", "solve", "--source", "tasks.json"], + files( + { + "env.py": EXAMPLE_ENV + '@env.template(id="solve")\nasync def solve():\n' + ' yield "go"\n yield 1.0\n', + "tasks.json": json.dumps( + [{"env": "example", "id": "solve", "slug": s} for s in ("one", "two")] + ), + } + ), + exit_code=2, + error="usage", + reaches="nothing", + ), + "models fork: a misspelt model": Failure( + ["models", "fork", "claude-sonet-4-6", "--name", "mine", "--yes"], + a_base_model, + exit_code=2, + error="usage", + mentions="claude-sonnet-4-6", + ), + "project: a default project that does not exist": Failure( + ["project"], error="not_found", env={"HUD_DEFAULT_PROJECT": "nope"}, linked=False + ), + "project create: the platform refuses": Failure( + ["project", "create", "UI Evals", "--yes"], + refused("POST", "/v2/projects", 403), + error="permission_denied", + reaches="the refusal", + ), + "project use: a read-only project": Failure( + ["project", "use", "locked-down"], error="permission_denied" + ), + "project use: a link made by another user": Failure( + ["project", "use", "browser-evals"], scoped(OTHER_ID, TEAM_ID), reaches="reads" + ), + "project use: a link made in another team": Failure( + ["project", "use", "browser-evals"], scoped(USER_ID, OTHER_ID), reaches="reads" + ), + "project use: a link made on another platform": Failure( + ["project", "use", "browser-evals"], + scoped(USER_ID, TEAM_ID, elsewhere=True), + reaches="reads", + ), + "project use: a corrupt link": Failure( + ["project", "use", "browser-evals"], link('{"broken":'), reaches="reads" + ), +} + + +@pytest.mark.parametrize("case", FAILURES) +def test_a_command_that_fails_changes_nothing( + hud: Hud, platform: Platform, hud_env: HudEnv, case: str +) -> None: + failure = FAILURES[case] + if failure.linked: + link({"project_id": BROWSER_PROJECT_ID})(hud, platform) + failure.setup(hud, platform) + hud_env.set(**failure.env) + before, asked = state(hud, hud_env), len(platform.services.requests()) + + result = hud(*failure.argv, "--json") + + assert (result.exit_code, result.json["error"]) == (failure.exit_code, failure.error), result + assert failure.mentions in result.json["message"] + assert state(hud, hud_env) == before + sent = [r for r in platform.services.requests()[asked:] if r.service != "telemetry"] + if failure.reaches == "nothing": + assert sent == [] + elif failure.reaches == "reads": + assert [f"{r.method} {r.path}" for r in sent if r.method != "GET"] == [] diff --git a/tests/cli/test_deploy.py b/tests/cli/test_deploy.py new file mode 100644 index 000000000..4e441d101 --- /dev/null +++ b/tests/cli/test_deploy.py @@ -0,0 +1,249 @@ +"""``hud deploy``: the build context it uploads, the build it triggers, and how it reports it.""" + +from __future__ import annotations + +import io +import json +import tarfile +from typing import TYPE_CHECKING, Any + +import pytest + +from tests.support import Reply, write_files +from tests.support.platform import BROWSER_PROJECT_ID, Build + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from tests.support import Hud, HudEnv, Platform + +ENV_PY = 'from hud import Environment\nenv = Environment("example")\n' + + +@pytest.fixture +def tmpdir_(tmp_path: Path, hud_env: HudEnv) -> Path: + """A private ``TMPDIR``, so a test can see that the build context tarball is removed.""" + (tmp_path / "tmp").mkdir() + hud_env.set(TMPDIR=str(tmp_path / "tmp")) + return tmp_path / "tmp" + + +def environment(directory: Path, **files: str) -> Path: + write_files(directory, {"env.py": ENV_PY, "Dockerfile.hud": "FROM python:3.12\n", **files}) + return directory + + +def context(build: Build) -> list[str]: + """Member names of the build context tarball the platform received.""" + assert build.context is not None + with tarfile.open(fileobj=io.BytesIO(build.context), mode="r:gz") as archive: + return sorted(archive.getnames()) + + +def trigger(platform: Platform) -> dict[str, Any]: + """The trigger of the last build the platform was asked for.""" + *_, build = platform.builds.values() + assert build.trigger is not None + return build.trigger + + +def settings(hud: Hud, platform: Platform) -> Any: + found = trigger(platform) + return (found["environment_variables"], found["build_args"], found["build_secrets"]) + + +def compose(hud: Hud, platform: Platform) -> Any: + document = trigger(platform)["runtime_config"]["compose"] + return (document["document"]["services"]["main"]["image"], document["root"]) + + +def placed(hud: Hud, platform: Platform) -> Any: + (example,) = platform.environments.values() + linked = json.loads((hud.cwd / ".hud" / "config.json").read_text())["registry_id"] + return (example["project_id"], linked == example["id"]) + + +def redeployed(hud: Hud, platform: Platform) -> Any: + first, second = platform.builds.values() + assert first.trigger is not None and second.trigger is not None + return ("environment_variables" in first.trigger, "environment_variables" in second.trigger) + + +SENT: dict[str, tuple[dict[str, str], list[list[str]], dict[str, str], Callable[..., Any], Any]] = { + "env files, build args and secrets ride the trigger": ( + {".env": "SECRET=dotenv-secret\n", "token.txt": "file-secret", "vars.env": "KEY=v\n"}, + [ + [ + *("--env-file", "vars.env", "--build-arg", "PYTHON=3.12"), + *("--secret", "id=GITHUB_TOKEN,env=GITHUB_TOKEN"), + *("--secret", "id=FILE_TOKEN,src=token.txt"), + ] + ], + {"GITHUB_TOKEN": "env-secret"}, + settings, + ( + {"KEY": "v"}, + {"PYTHON": "3.12"}, + {"GITHUB_TOKEN": "env-secret", "FILE_TOKEN": "file-secret"}, + ), + ), + "a Compose runtime config reaches the build as its document": ( + { + "project/compose.json": '{"services":{"main":{"image":"postgres:16"}}}', + "runtime.json": json.dumps( + {"compose": {"document": "project/compose.json", "root": "."}} + ), + }, + [["--no-env", "--runtime-config", "runtime.json"]], + {}, + compose, + ("postgres:16", {"compose_path": "project/compose.json"}), + ), + "a first deploy lands in the selected project and links the directory": ( + {}, + [["--no-env"]], + {"HUD_DEFAULT_PROJECT": "browser-evals"}, + placed, + (BROWSER_PROJECT_ID, True), + ), + "a redeploy keeps the registry's variables": ( + {".env": "SECRET=dotenv"}, + [["--env-file", ".env"], []], + {}, + redeployed, + (True, False), + ), +} + + +@pytest.mark.parametrize( + ("files", "deploys", "env", "observe", "expected"), SENT.values(), ids=SENT.keys() +) +def test_a_deploy_sends_what_its_options_name_with_the_build( + hud: Hud, + platform: Platform, + files: dict[str, str], + deploys: list[list[str]], + env: dict[str, str], + observe: Callable[[Hud, Platform], Any], + expected: Any, +) -> None: + environment(hud.cwd, **files) + + results = [hud("deploy", *argv, "--yes", "--json", env=env) for argv in deploys] + + assert [result.exit_code for result in results] == [0] * len(deploys), results[-1] + assert observe(hud, platform) == expected + for build in platform.builds.values(): + assert ".env" not in context(build) + + +def test_the_build_context_follows_dockerignore_and_never_holds_secrets( + hud: Hud, platform: Platform +) -> None: + environment( + hud.cwd, + **{ + "nested/.env.prod": "SECRET=1", + ".git/config": "", + "build/out.o": "", + "src/build/out.o": "", + "keep.pyc": "", + "drop.pyc": "", + ".dockerignore": "*.pyc\n!keep.pyc\n/build\n", + }, + ) + + result = hud("deploy", "--no-env", "--yes", "--json") + + assert result.exit_code == 0, result + (build,) = platform.builds.values() + sent = set(context(build)) + assert {"keep.pyc", "src/build/out.o"} <= sent + assert not {"drop.pyc", "build/out.o", ".git/config", "nested/.env.prod"} & sent + + +@pytest.mark.parametrize( + ("files", "argv", "planned"), + [ + pytest.param( + {"env.py": 'env = Environment("my-env")\n'}, [], {"name": "my-env"}, id="declared" + ), + pytest.param( + {"env.py": 'actor = Environment("workspace")\nverifier = Environment("judge")\n'}, + ["--name", "judge"], + {"name": "judge"}, + id="chosen-among-several", + ), + pytest.param( + {"env.py": ENV_PY, ".env": "SECRET=hidden"}, + [], + {"name": "example", "dotenv_pending": True}, + id="a-dotenv-beside-it-is-asked-about", + ), + ], +) +def test_a_dry_run_plans_the_environment_it_would_deploy( + hud: Hud, platform: Platform, files: dict[str, str], argv: list[str], planned: dict[str, Any] +) -> None: + write_files(hud.cwd, files) + + result = hud("deploy", *argv, "--dry-run", "--json") + + assert result.exit_code == 0, result + assert {key: result.json[key] for key in planned} == planned + + +ENDINGS: dict[str, tuple[Build | None, Reply | None, int, str]] = { + "a build that succeeds streams its log": (None, None, 0, "Step 1/2 : FROM python:3.12"), + "a build that fails": ( + Build( + "", + status={"status": "FAILED", "error_message": "pip install failed"}, + log=[{"type": "error", "error": "Step 2 failed"}], + ), + None, + 1, + "", + ), + "a build without a log stream": ( + Build("", status={"status": "SUCCEEDED"}, log=None), + None, + 0, + "", + ), + "a build request the platform rejects": ( + None, + Reply(status=409, json={"detail": "Busy"}), + 1, + "", + ), +} + + +@pytest.mark.parametrize( + ("build", "rejected", "exit_code", "logged"), ENDINGS.values(), ids=ENDINGS.keys() +) +def test_the_build_decides_the_exit_and_no_context_is_left_behind( + hud: Hud, + platform: Platform, + tmpdir_: Path, + build: Build | None, + rejected: Reply | None, + exit_code: int, + logged: str, +) -> None: + if build is not None: + platform.next_build = build + if rejected is not None: + platform.services.route("api", "POST", "/v2/builds/trigger", rejected) + environment(hud.cwd) + + result = hud("deploy", "--no-env", "--yes", "--json") + + assert result.exit_code == exit_code, result + assert logged in result.stderr + if rejected is None: + assert result.json["success"] is (exit_code == 0) + assert list(tmpdir_.iterdir()) == [] diff --git a/tests/cli/test_eval.py b/tests/cli/test_eval.py new file mode 100644 index 000000000..f42c194df --- /dev/null +++ b/tests/cli/test_eval.py @@ -0,0 +1,393 @@ +"""``hud eval``: plan, validate and run real rollouts of a taskset with a scripted model.""" + +from __future__ import annotations + +import asyncio +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from hud import Environment +from hud.eval import LocalRuntime, Task +from tests.support import say, write_files +from tests.support.platform import API_KEY + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import Hud, HudEnv, Models, Platform + +TASKS_PY = """\ +from hud import Environment + +env = Environment("demo") + + +@env.template(id="solve") +async def solve(n: int = 0): + answer = yield f"solve {n}" + yield 1.0 if answer == "ok" else 0.0 + + +tasks = [solve(n=0), solve(n=1)] +""" +SCRIPTED = ("openai_compatible", "-m", "scripted") + + +@pytest.fixture +def gateway(models: Models, platform: Platform) -> Models: + """A signed-in user whose agents reach a scripted model that always answers "ok".""" + models.script([say("ok")]) + return models + + +@pytest.fixture +def tasks(hud: Hud) -> Path: + path = hud.cwd / "tasks.py" + path.write_text(TASKS_PY) + return path + + +def run(hud: Hud, *argv: str) -> Any: + """``hud eval ... --yes --json``; the result document of a successful run.""" + result = hud("eval", *argv, "--yes", "--json") + assert result.exit_code == 0, result + return result.json + + +# ─── results ──────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("grade", "exit_code", "rewards", "errors"), + [ + pytest.param("0.0", 0, [1.0, 0.0], [False, False], id="a-low-reward-is-not-an-error"), + pytest.param( + '{"score": 0.0, "isError": True, "content": "grader unavailable"}', + 1, + [1.0, 0.0], + [False, True], + id="a-grade-error", + ), + pytest.param("1 / 0", 1, [1.0, 0.0], [False, True], id="a-grader-crash"), + ], +) +def test_eval_reports_every_run_and_fails_when_one_errored( + hud: Hud, + gateway: Models, + grade: str, + exit_code: int, + rewards: list[float], + errors: list[bool], +) -> None: + (hud.cwd / "tasks.py").write_text( + 'from hud import Environment\nenv = Environment("demo")\n' + '@env.template(id="solve")\nasync def solve(n: int = 0):\n' + ' answer = yield f"solve {n}"\n' + f" yield 1.0 if n == 0 else {grade}\n" + "tasks = [solve(n=0), solve(n=1)]\n" + ) + + result = hud("eval", "tasks.py", *SCRIPTED, "--all", "--yes", "--json") + + assert result.exit_code == exit_code, result + runs = sorted(result.json["runs"], key=lambda run: run["is_error"]) + assert [run["reward"] for run in runs] == rewards + assert [run["is_error"] for run in runs] == errors + assert result.json["error_count"] == sum(errors) + + +@pytest.mark.parametrize( + ("argv", "prompts"), + [ + pytest.param([], ["solve 0"], id="the-first-by-default"), + pytest.param(["--all"], ["solve 0", "solve 1"], id="all"), + pytest.param(["--task-ids", "1"], ["solve 1"], id="by-index"), + ], +) +def test_eval_runs_the_tasks_it_selects( + hud: Hud, gateway: Models, tasks: Path, argv: list[str], prompts: list[str] +) -> None: + run(hud, "tasks.py", *SCRIPTED, *argv) + + assert sorted(request.prompt for request in gateway.requests()) == prompts + + +# ─── how the agent reaches its model ──────────────────────────────────── + + +GLM = { + "id": "z-ai/glm-5.2", + "model_name": "z-ai/glm-5.2", + "sdk_agent_type": "openai_compatible", + "provider": {"name": "openai"}, +} +KIMI_TOML = ( + '[openai_compatible]\nmodel = "moonshotai/kimi-k2.6"\n' + "completion_kwargs = { temperature = 0.5 }\n" +) + + +@pytest.mark.parametrize( + ("argv", "config", "expected"), + [ + pytest.param( + list(SCRIPTED), + "", + {"key": API_KEY}, + id="gateway-by-default", + ), + pytest.param( + ["openai_compatible"], + KIMI_TOML, + {"model": "moonshotai/kimi-k2.6", "temperature": 0.5}, + id="toml-section", + ), + pytest.param( + ["openai_compatible", "-m", "z-ai/glm-5.2"], + KIMI_TOML, + {"model": "z-ai/glm-5.2", "temperature": 0.5}, + id="model-flag-wins-over-toml", + ), + pytest.param( + ["glm-5.2"], + "", + {"protocol": "chat", "model": "z-ai/glm-5.2"}, + id="catalog-alias", + ), + pytest.param( + ["claude", "--config", "max_tokens=100"], + "", + {"max_tokens": 100, "key": API_KEY}, + id="config-flag", + ), + pytest.param( + ["openai_compatible", "-m", "custom", "-c", "api_key=custom-key", "-c", "base_url=URL"], + "", + {"key": "custom-key"}, + id="custom-endpoint", + ), + ], +) +def test_the_agent_reaches_its_model_with_the_right_key_and_settings( + hud: Hud, + platform: Platform, + gateway: Models, + hud_env: HudEnv, + tasks: Path, + argv: list[str], + config: str, + expected: dict[str, Any], +) -> None: + """Provider SDKs read their own ``*_BASE_URL``, which here points at the scripted model.""" + platform.add_model(**GLM) + (hud.cwd / ".hud_eval.toml").write_text(config) + hud_env.set(OPENAI_BASE_URL=gateway.url, ANTHROPIC_BASE_URL=gateway.url) + + # The same scripted model under another host name, so it is not the HUD gateway. + custom = gateway.url.replace("127.0.0.1", "localhost") + run(hud, "tasks.py", *[arg.replace("URL", custom) for arg in argv]) + + (request,) = gateway.requests() + credential = ( + request.headers.get("x-api-key") + or request.headers.get("x-goog-api-key") + or request.headers.get("authorization", "").removeprefix("Bearer ") + ) + observed = { + "protocol": request.protocol, + "model": request.model, + "key": credential, + "temperature": request.body.get("temperature"), + "max_tokens": request.body.get("max_tokens"), + } + assert {name: observed[name] for name in expected} == expected + + +# ─── project layouts ──────────────────────────────────────────────────── + +ENV = 'from hud import Environment\nenv = Environment("local-test")\n' +SOLVE = ( + '@env.template(id="solve")\nasync def solve():\n' + ' answer = yield "answer ok"\n' + ' yield 1.0 if answer == "ok" else 0.0\n' +) + + +def named(name: str, reward: str) -> str: + """An env ``name`` whose ``solve`` pays ``reward``.""" + return ( + f"from hud import Environment\nenv = Environment({name!r})\n" + '@env.template(id="solve")\nasync def solve():\n' + f' answer = yield "answer ok"\n yield {reward}\n' + ) + + +LAYOUTS: dict[str, tuple[dict[str, str], str, tuple[int, float, int]]] = { + "one file": ({"env.py": ENV + SOLVE + "tasks = [solve()]\n"}, "env.py", (1, 1.0, 0)), + "tasks beside the env": ( + {"env.py": ENV + SOLVE, "tasks.py": "from env import solve\ntasks = [solve()]\n"}, + "tasks.py", + (1, 1.0, 0), + ), + "json rows beside the env": ( + {"env.py": ENV + SOLVE, "tasks.json": json.dumps([{"env": "local-test", "id": "solve"}])}, + "tasks.json", + (1, 1.0, 0), + ), + "a row picks its env by name among several": ( + { + "bound_env.py": named("bound", 'float(answer == "ok")'), + "env.py": named("unrelated", "0.0"), + "rows.json": '{"env": "bound", "id": "solve"}\n', + }, + "rows.json", + (1, 1.0, 0), + ), + "two envs sharing the row's name error the row": ( + { + "bound_env.py": named("bound", 'float(answer == "ok")'), + "env.py": named("bound", "0.0"), + "tasks.py": "from bound_env import solve\ntasks = [solve()]\n", + }, + ".", + (1, 0.0, 1), + ), + "a verifier in another env of the project": ( + { + "core.py": 'from hud import Environment\nactor = Environment("actor")\n' + + 'judge = Environment("judge")\n', + "actor.py": "from core import actor\n" + '@actor.template(id="solve")\nasync def solve():\n' + ' answer = yield "answer ok"\n yield {"score": 0.0, "answer": answer}\n', + "judge.py": "from core import judge\n" + '@judge.template(id="verify")\nasync def verify():\n' + ' result = yield ""\n yield 1.0 if result["answer"] == "ok" else 0.0\n', + "env.py": "from core import actor, judge\n" + "from actor import solve\nfrom judge import verify\n", + "tasks.py": "from env import solve, verify\n" + "tasks = [solve()]\ntasks[0].verifier = verify()\n", + }, + "tasks.py", + (1, 1.0, 0), + ), +} + + +@pytest.mark.parametrize(("files", "source", "outcome"), LAYOUTS.values(), ids=LAYOUTS.keys()) +def test_eval_runs_a_projects_rows_in_the_environments_they_name( + hud: Hud, + gateway: Models, + tmp_path: Path, + files: dict[str, str], + source: str, + outcome: tuple[int, float, int], +) -> None: + project = tmp_path / "project" + write_files(project, files) + + result = hud("eval", str(project / source), *SCRIPTED, "--all", "--yes", "--json") + + assert result.exit_code == (outcome[2] > 0), result + document = result.json + assert (document["run_count"], document["mean_reward"], document["error_count"]) == outcome + + +async def test_eval_attaches_to_an_already_served_environment(hud: Hud, gateway: Models) -> None: + env = Environment("served") + + @env.template(id="solve") + async def solve(): + answer = yield "answer ok" + yield 1.0 if answer == "ok" else 0.0 + + (hud.cwd / "rows.json").write_text('[{"env": "served", "id": "solve"}]') + async with LocalRuntime(env)(Task(env="served", id="solve")) as runtime: + document = await asyncio.to_thread( + run, hud, "rows.json", *SCRIPTED, "--runtime", runtime.url + ) + + assert (document["run_count"], document["mean_reward"]) == (1, 1.0) + + +def test_one_tasks_file_is_picked_when_no_source_is_given(hud: Hud, gateway: Models) -> None: + write_files( + hud.cwd, + { + "env.py": TASKS_PY, + "rows.json": '[{"env": "demo", "id": "solve", "args": {"n": 4}}]', + ".hidden.json": "[]", + ".hud_eval.toml": '[eval]\nagent = "openai_compatible"\nmodel = "m"\n', + }, + ) + + document = run(hud) + + assert (document["source"], document["mean_reward"]) == ("rows.json", 1.0) + assert [request.prompt for request in gateway.requests()] == ["solve 4"] + + +def test_a_python_source_is_imported_on_the_main_thread(hud: Hud, gateway: Models) -> None: + (hud.cwd / "probe.py").write_text( + "import threading\nfrom pathlib import Path\n" + 'Path("main-thread.txt").write_text(' + "str(threading.current_thread() is threading.main_thread()))\n" + "tasks = []\n" + ) + + result = hud("eval", "probe.py", "openai", "--yes", "--json") + + assert (result.exit_code, result.json["error"]) == (2, "usage"), result + assert (hud.cwd / "main-thread.txt").read_text() == "True" + + +# ─── planning and refusals ────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("config", "argv", "expected"), + [ + pytest.param( + "", ["tasks.py", "openai"], {"runtime": "local"}, id="a-local-source-runs-here" + ), + pytest.param( + "", ["My Tasks", "openai"], {"runtime": "hosted"}, id="a-platform-taskset-runs-hosted" + ), + pytest.param( + "[eval]\nremote = true\n", + ["tasks.py", "openai"], + {"runtime": "hosted"}, + id="the-toml-asks-for-hosted", + ), + pytest.param( + "[eval]\nremote = false\n", + ["tasks.py", "openai", "--remote"], + {"runtime": "hosted"}, + id="a-flag-beats-the-toml", + ), + pytest.param( + '[openai]\nmodel = "${MY_EVAL_MODEL}"\n', + ["tasks.py", "openai"], + {"model": "gpt-4o"}, + id="the-toml-model-from-a-variable", + ), + ], +) +def test_a_dry_run_reports_the_plan_and_runs_nothing( + hud: Hud, + platform: Platform, + hud_env: HudEnv, + tasks: Path, + config: str, + argv: list[str], + expected: dict[str, Any], +) -> None: + hud_env.set(MY_EVAL_MODEL="gpt-4o") + (hud.cwd / ".hud_eval.toml").write_text(config) + + result = hud("eval", *argv, "--dry-run", "--json") + + assert result.exit_code == 0, result + assert {name: result.json[name] for name in expected} == expected + assert platform.services.requests() == [] diff --git a/tests/cli/test_harbor.py b/tests/cli/test_harbor.py new file mode 100644 index 000000000..2922464b7 --- /dev/null +++ b/tests/cli/test_harbor.py @@ -0,0 +1,277 @@ +"""``hud harbor``: import Harbor folders or bundles into a taskset, and follow imports.""" + +from __future__ import annotations + +import io +import json +import zipfile +from typing import TYPE_CHECKING, Any + +import pytest + +from tests.support import write_files +from tests.support.platform import Import + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from tests.support import Hud, Platform + +IMPORT = "66666666-6666-4666-8666-666666666666" +REPORT = { + "schema_version": "harbor_import_report.v5", + "tasks": [ + { + "name": "hello", + "source": "bundle", + "path": "tasks/hello", + "build_version": 1, + "status": "imported", + } + ], + "jobs": [ + { + "path": "run-1", + "name": "run-1", + "job_id": "job-hud-1", + "status": "imported", + "trials": [ + {"path": "run-1/hello__a", "status": "imported"}, + {"path": "run-1/gone__b", "status": "skipped", "reason": "unknown task"}, + ], + } + ], + "warnings": [], + "error": None, +} +FOLDERS = { + "tasks/hello/task.toml": "version = '1.0'\n", + "tasks/hello/tests/test.sh": "exit 0\n", + "run-1/config.json": "{}", + "run-1/result.json": "{}", + "run-1/hello__a/config.json": "{}", + "run-1/hello__a/result.json": "{}", +} + + +@pytest.fixture +def importer(platform: Platform) -> Platform: + """The platform, with each import it starts completing and leaving ``REPORT``.""" + platform.next_import.report = REPORT + return platform + + +def started(platform: Platform) -> Import: + (only,) = platform.imports.values() + return only + + +def bundled(platform: Platform) -> dict[str, bytes]: + """The bundle the platform received, member by member.""" + uploaded = started(platform).bundle + assert uploaded is not None + with zipfile.ZipFile(io.BytesIO(uploaded)) as archive: + return {name: archive.read(name) for name in archive.namelist()} + + +def harbor_folders(root: Path) -> list[str]: + write_files(root, FOLDERS) + return [str(root / "tasks"), str(root / "run-1")] + + +def a_prepared_zip(root: Path) -> list[str]: + prepared = root / "prepared.zip" + root.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(prepared, "w") as archive: + archive.writestr("hello/task.toml", "version = '1.0'\n") + return [str(prepared)] + + +def a_task_with_links(root: Path) -> list[str]: + task = root / "src" / "hello" + write_files(task, {"task.toml": "version = '1.0'\n", "environment/Dockerfile": "FROM ubuntu\n"}) + write_files(root / "shared", {"data.txt": "shared"}) + (task / "environment" / "data.txt").symlink_to(root / "shared" / "data.txt") + (task / "environment" / "linked").symlink_to(root / "shared") + (task / "dangling").symlink_to(root / "missing") + return [str(task)] + + +BUNDLES: dict[str, tuple[Callable[[Path], list[str]], dict[str, bytes], str]] = { + "folders are zipped by their own paths": ( + harbor_folders, + {name: content.encode() for name, content in FOLDERS.items()}, + "tb2", + ), + "a prepared zip is sent as it is": ( + a_prepared_zip, + {"hello/task.toml": b"version = '1.0'\n"}, + "prepared", + ), + "a linked file is sent by content and a linked folder is skipped": ( + a_task_with_links, + { + "hello/task.toml": b"version = '1.0'\n", + "hello/environment/Dockerfile": b"FROM ubuntu\n", + "hello/environment/data.txt": b"shared", + }, + "tb2", + ), +} + + +@pytest.mark.parametrize(("source", "members", "taskset"), BUNDLES.values(), ids=BUNDLES.keys()) +def test_an_import_uploads_its_input_as_one_bundle( + hud: Hud, + importer: Platform, + tmp_path: Path, + source: Callable[[Path], list[str]], + members: dict[str, bytes], + taskset: str, +) -> None: + paths = source(tmp_path / "input") + named = ["--taskset", taskset] if taskset != "prepared" else [] + + result = hud("harbor", "import", *paths, *named, "--yes", "--json") + + assert result.exit_code == 0, result + assert bundled(importer) == members + assert started(importer).taskset_id == importer.taskset(taskset)["id"] + + +@pytest.mark.parametrize( + ("runs", "flags", "created", "linked"), + [ + pytest.param(1, [], True, False, id="a-new-name-creates-its-taskset"), + pytest.param(2, [], False, False, id="rerunning-reuses-the-taskset-its-name-made"), + pytest.param(1, ["--link"], True, True, id="link-saves-it-as-the-directory-default"), + pytest.param(1, ["--dry-run"], True, False, id="a-dry-run-plans-to-create-it"), + ], +) +def test_an_import_lands_in_the_taskset_its_name_resolves_to( + hud: Hud, + importer: Platform, + tmp_path: Path, + runs: int, + flags: list[str], + created: bool, + linked: bool, +) -> None: + paths = harbor_folders(tmp_path / "input") + confirm = [] if "--dry-run" in flags else ["--yes"] + + results = [ + hud("harbor", "import", *paths, "--taskset", "TB2/Sample", *flags, *confirm, "--json") + for _ in range(runs) + ] + + assert [result.exit_code for result in results] == [0] * runs, results[-1] + assert results[-1].json["taskset"]["created"] is created + link = hud.cwd / ".hud" / "config.json" + stored = json.loads(link.read_text())["taskset_id"] if link.exists() else None + if "--dry-run" in flags: + assert importer.tasksets == {} + else: + assert results[-1].json["taskset"] == { + "id": importer.taskset("tb2sample")["id"], + "name": "tb2sample", + "created": created, + } + assert stored == (importer.taskset("tb2sample")["id"] if linked else None) + + +def test_a_refused_upload_deletes_its_reservation_and_imports_nothing( + hud: Hud, importer: Platform, tmp_path: Path +) -> None: + importer.services.route("api", "PUT", "/uploads/{id}", status=403) + + result = hud( + "harbor", "import", *harbor_folders(tmp_path), "--taskset", "tb2", "--yes", "--json" + ) + + assert result.exit_code == 1, result + assert importer.imports == {} + + +@pytest.mark.parametrize( + ("statuses", "report", "flags", "exit_code", "shown"), + [ + pytest.param( + ["queued", "running", "completed"], REPORT, [], 0, "completed", id="completed" + ), + pytest.param( + ["stopped"], + {**REPORT, "error": {"step": "add", "message": "boom"}}, + [], + 1, + "stopped", + id="stopped", + ), + pytest.param(["failed"], None, [], 1, "failed", id="failed"), + pytest.param( + ["queued", "completed"], REPORT, ["--no-wait"], 0, "queued", id="no-wait-returns-it" + ), + ], +) +def test_an_import_waits_for_its_end_and_fails_unless_it_completed( + hud: Hud, + importer: Platform, + tmp_path: Path, + statuses: list[str], + report: dict[str, Any] | None, + flags: list[str], + exit_code: int, + shown: str, +) -> None: + importer.next_import = Import( + "", + taskset_id="", + statuses=statuses, + report=report, + error="crashed" if report is None else None, + ) + paths = harbor_folders(tmp_path) + + result = hud("harbor", "import", *paths, "--taskset", "tb2", *flags, "--yes", "--json") + + assert (result.exit_code, result.json["import"]["status"]) == (exit_code, shown), result + if not flags: + reported = result.json["report"] + assert (reported or {}).get("error") == (report or {}).get("error") + + +@pytest.mark.parametrize( + ("argv", "statuses", "report", "version"), + [ + pytest.param( + ["--wait"], + ["queued", "running", "completed"], + REPORT, + "harbor_import_report.v5", + id="waited", + ), + pytest.param( + [], + ["completed"], + {"schema_version": "harbor_import_report.v9"}, + None, + id="unknown-schema", + ), + ], +) +def test_report_shows_an_ended_imports_report( + hud: Hud, + platform: Platform, + argv: list[str], + statuses: list[str], + report: dict[str, Any], + version: str | None, +) -> None: + platform.imports[IMPORT] = Import(IMPORT, "tb2", statuses=statuses, report=report) + + result = hud("harbor", "report", IMPORT, *argv, "--json") + + assert result.exit_code == 0, result + shown_version = (result.json["report"] or {}).get("schema_version") + assert (result.json["import"]["status"], shown_version) == (statuses[-1], version) diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py new file mode 100644 index 000000000..a7ef0ec4c --- /dev/null +++ b/tests/cli/test_init.py @@ -0,0 +1,42 @@ +"""``hud init``: copy an example environment from this SDK checkout into a new project.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import Hud + + +def tree(root: Path) -> list[str]: + return sorted(str(path.relative_to(root)) for path in root.rglob("*")) + + +def test_init_copies_the_example_and_names_its_environment(hud: Hud) -> None: + result = hud("init", "My Cool_Env", "--preset", "blank", "--dir", "projects", "--json") + + target = hud.cwd / "projects" / "My Cool_Env" + assert result.exit_code == 0, result + assert 'Environment(name="my-cool-env")' in (target / "env.py").read_text() + assert not any( + part in {".venv", "__pycache__"} for name in tree(target) for part in name.split("/") + ) + + +def test_a_dry_run_plans_without_creating(hud: Hud) -> None: + result = hud("init", "thing", "--dir", "projects", "--dry-run", "--json") + + assert (result.exit_code, result.json["preset"]) == (0, "coding"), result + assert not (hud.cwd / "projects").exists() + + +def test_force_overwrites_an_existing_project(hud: Hud) -> None: + (hud.cwd / "env").mkdir() + (hud.cwd / "env" / "env.py").write_text("old") + + result = hud("init", "env", "--preset", "blank", "--force", "--json") + + assert result.exit_code == 0, result + assert 'Environment(name="env")' in (hud.cwd / "env" / "env.py").read_text() diff --git a/tests/cli/test_link.py b/tests/cli/test_link.py new file mode 100644 index 000000000..41a185ac8 --- /dev/null +++ b/tests/cli/test_link.py @@ -0,0 +1,125 @@ +"""The directory link (``.hud/config.json``): the platform ids it accumulates, the older +form it rewrites, and the Project a directory places work in.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from tests.support import write_files +from tests.support.platform import BROWSER_PROJECT_ID, TEAM_ID, USER_ID + +if TYPE_CHECKING: + from tests.support import Hud, HudEnv, Platform + +OTHER_ID = "cccccccc-0000-4000-8000-000000000001" + + +def link(hud: Hud) -> dict[str, Any] | None: + path = hud.cwd / ".hud" / "config.json" + return json.loads(path.read_text()) if path.exists() else None + + +def test_linking_a_directory_accumulates_its_platform_ids(hud: Hud, platform: Platform) -> None: + example = platform.add_environment("example") + + used = hud("project", "use", "Browser Evals", "--json") + linked = hud("sync", "env", "example", "--json") + written = (hud.cwd / ".hud" / "config.json").read_bytes() + relinked = hud("sync", "env", example, "--json") + shown = hud("project", "--json") + + assert [used.exit_code, linked.exit_code, relinked.exit_code, shown.exit_code] == [0] * 4 + stored = link(hud) + assert stored is not None + assert (stored["registry_id"], stored["project_id"], stored["taskset_id"]) == ( + example, + BROWSER_PROJECT_ID, + None, + ) + assert (stored["scope"]["user_id"], stored["scope"]["team_id"]) == (USER_ID, TEAM_ID) + assert (linked.json["changed"], relinked.json["changed"]) == (True, False) + assert (hud.cwd / ".hud" / "config.json").read_bytes() == written + assert (shown.json["project"]["id"], shown.json["source"]) == ( + BROWSER_PROJECT_ID, + ".hud/config.json", + ) + assert "sk-hud-test" not in written.decode() + + +@pytest.mark.parametrize("dry_run", [False, True], ids=["rewritten", "left-alone-on-a-dry-run"]) +def test_an_older_camelcase_link_is_rewritten_with_its_ids( + hud: Hud, platform: Platform, dry_run: bool +) -> None: + example = platform.add_environment("example") + older = json.dumps({"registryId": example, "tasksetId": OTHER_ID}) + write_files(hud.cwd / ".hud", {"config.json": older}) + + result = hud("project", "use", "browser-evals", *(["--dry-run"] if dry_run else []), "--json") + + assert result.exit_code == 0, result + stored = link(hud) + assert stored is not None + if dry_run: + assert json.dumps(stored) == older + else: + assert (stored["registry_id"], stored["taskset_id"], stored["project_id"]) == ( + example, + OTHER_ID, + BROWSER_PROJECT_ID, + ) + + +# ─── which Project a directory uses ───────────────────────────────────── + + +@pytest.mark.parametrize( + ("linked", "default", "project", "source"), + [ + pytest.param(None, None, None, "team default", id="team-default"), + pytest.param( + None, "Browser Evals", BROWSER_PROJECT_ID, "HUD_DEFAULT_PROJECT", id="default-by-name" + ), + pytest.param( + None, + BROWSER_PROJECT_ID.upper(), + BROWSER_PROJECT_ID, + "HUD_DEFAULT_PROJECT", + id="default-by-id", + ), + pytest.param( + BROWSER_PROJECT_ID, + "locked-down", + BROWSER_PROJECT_ID, + ".hud/config.json", + id="the-link-beats-the-default", + ), + ], +) +def test_project_shows_the_selected_project_and_why( + hud: Hud, + platform: Platform, + hud_env: HudEnv, + linked: str | None, + default: str | None, + project: str | None, + source: str, +) -> None: + hud_env.set(HUD_DEFAULT_PROJECT=default) + if linked is not None: + assert hud("project", "use", linked).exit_code == 0 + + result = hud("project", "--json") + + assert result.exit_code == 0, result + assert ((result.json["project"] or {}).get("id"), result.json["source"]) == (project, source) + + +def test_the_directory_option_after_the_verb_wins(hud: Hud, platform: Platform) -> None: + result = hud("project", "-C", "group", "use", BROWSER_PROJECT_ID, "-C", "override") + + assert result.exit_code == 0, result + linked = [path.parent.parent.name for path in hud.cwd.glob("*/.hud/config.json")] + assert linked == ["override"] diff --git a/tests/cli/test_output.py b/tests/cli/test_output.py new file mode 100644 index 000000000..7a03a3b8b --- /dev/null +++ b/tests/cli/test_output.py @@ -0,0 +1,267 @@ +"""What a ``hud`` command prints: one error document when it fails, the same error kind for +each platform refusal, one id per line when quiet, and never a secret it was given.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from tests.support import Reply, write_files +from tests.support.platform import Import + +from .commands import ( + EVERY_COMMAND, + JOB_ID, + PLATFORM, + TREE, + Setup, + a_trainable_model, + answer_everything, + environment, + failing, + harbor_folders, + invocation, + refused, + tasks, + together, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from tests.support import Hud, HudEnv, Platform + + +@pytest.mark.parametrize("name", EVERY_COMMAND) +def test_a_failure_is_one_error_document_under_json_and_stays_off_stdout_in_text( + hud: Hud, platform: Platform, hud_env: HudEnv, name: str +) -> None: + argv = failing(name, hud, platform, hud_env) + + as_json = hud(*argv, "--json") + as_text = hud(*argv) + + error = as_json.json + assert as_json.exit_code == (2 if error["error"] == "usage" else 1), as_json + assert error["message"] not in as_json.stderr, as_json + assert (as_text.exit_code, as_text.stdout) == (as_json.exit_code, ""), as_text + assert error["message"] in as_text.stderr, as_text + + +ANSWERS = { + "401": (Reply(status=401, json={"detail": "Refused"}), "permission_denied"), + "403": (Reply(status=403, json={"detail": "Refused"}), "permission_denied"), + "404": (Reply(status=404, json={"detail": "Refused"}), "not_found"), + "409": (Reply(status=409, json={"detail": "Refused"}), "conflict"), + "429": (Reply(status=429, json={"detail": "Refused"}), "rate_limited"), + "500": (Reply(status=500, json={"detail": "Refused"}), "server_error"), + "not-json": (Reply(body="", content_type="text/html"), "failure"), +} + +# Every command goes through one status mapping: each command is held to it with one +# status, and the mapping itself is read through one command. +REFUSALS = [(name, "403") for name in sorted(PLATFORM)] + [ + ("jobs get", answer) for answer in ANSWERS if answer != "403" +] + + +@pytest.mark.parametrize(("name", "answer"), REFUSALS) +def test_a_platform_refusal_is_the_same_error_kind_for_every_command( + hud: Hud, platform: Platform, name: str, answer: str +) -> None: + reply, error = ANSWERS[answer] + PLATFORM[name].setup(hud, platform) + answer_everything(platform, reply) + + result = hud(*invocation(name), "--json") + + assert (result.exit_code, result.json["error"]) == (1, error), result + + +def test_a_transient_platform_failure_is_retried_and_the_command_succeeds( + hud: Hud, platform: Platform +) -> None: + job = platform.add_job(name="job-0", status="completed") + listed = Reply(json={"items": list(platform.jobs.values())}) + platform.services.route("api", "GET", "/v2/jobs", Reply(status=503), Reply(status=502), listed) + + result = hud("jobs", "list", "--quiet") + + assert (result.exit_code, result.stdout) == (0, f"{job}\n"), result + assert len(platform.services.requests("api", "GET", "/v2/jobs")) == 3 + + +# ─── lists ────────────────────────────────────────────────────────────── + + +def python_tasks(hud: Hud, platform: Platform) -> list[str]: + del platform + write_files( + hud.cwd, + { + "env.py": 'from hud import Environment\nenv = Environment("example")\n' + '@env.template(id="solve")\nasync def solve(target: str = "answer"):\n' + ' answer = yield f"Say {target}."\n yield 1.0\n', + "tasks.py": 'from env import solve\ntasks = [solve(), solve(target="hello")]\n', + }, + ) + return ["solve", "solve-8da35614"] + + +def projects(hud: Hud, platform: Platform) -> list[str]: + del hud + platform.projects += [ + {"id": f"dddddddd-0000-4000-8000-{n:012d}", "name": f"p{n}"} for n in range(3) + ] + return [project["id"] for project in platform.projects] + + +def catalog(hud: Hud, platform: Platform) -> list[str]: + del hud + platform.add_model(name="owner/model + version", model_name="team/my-sonnet") + platform.add_model(name="Claude Sonnet", model_name="claude-sonnet-4-6") + return ["claude-sonnet-4-6", "team/my-sonnet"] + + +def checkpoints(hud: Hud, platform: Platform) -> list[str]: + a_trainable_model(hud, platform) + return ["ckpt-1", "ckpt-2"] + + +def jobs(hud: Hud, platform: Platform) -> list[str]: + del hud + return [platform.add_job(name=f"job-{n}", status="completed") for n in range(3)] + + +def traces(hud: Hud, platform: Platform) -> list[str]: + del hud + ids = [f"00000000-0000-4000-a000-00000000000{n}" for n in range(2)] + platform.add_job([{"id": trace, "status": "completed"} for trace in ids], id=JOB_ID) + return ids + + +CHECKS = [ + {"key": "failure_analysis", "subject_type": "trace", "title": "Failure Analysis"}, + {"key": "reward_hacking", "subject_type": "trace", "title": "Reward Hacking"}, + {"key": "task_reward_exploit", "subject_type": "task", "title": "Task Reward Exploit"}, +] + + +def checks(hud: Hud, platform: Platform) -> list[str]: + del hud + platform.qa_checks = CHECKS + return [check["key"] for check in CHECKS] + + +def task_checks(hud: Hud, platform: Platform) -> list[str]: + checks(hud, platform) + return ["task_reward_exploit"] + + +def imports(hud: Hud, platform: Platform) -> list[str]: + del hud + taskset = platform.add_taskset("tb2") + ids = [f"66666666-6666-4666-8666-00000000000{n}" for n in range(2)] + for found in ids: + platform.imports[found] = Import(found, taskset) + return ids + + +LISTS: dict[str, tuple[list[str], Callable[[Hud, Platform], list[str]]]] = { + "task list": (["task", "list", "--source", "tasks.py"], python_tasks), + "project list": (["project", "list"], projects), + "models list": (["models", "list"], catalog), + "models checkpoints": (["models", "checkpoints", "team/my-sonnet"], checkpoints), + "jobs": (["jobs"], jobs), + "jobs list": (["jobs", "list"], jobs), + "jobs get": (["jobs", "get", JOB_ID], traces), + "qa": (["qa"], checks), + "qa list": (["qa", "list", "--level", "task"], task_checks), + "harbor imports": (["harbor", "imports", "--taskset", "tb2"], imports), +} + + +def test_every_command_with_quiet_is_a_list() -> None: + quiet = { + name + for name, command in TREE.items() + if any("--quiet" in getattr(param, "opts", []) for param in getattr(command, "params", [])) + } + + assert sorted(quiet) == sorted(LISTS) + + +@pytest.mark.parametrize("name", LISTS) +def test_a_list_prints_one_id_per_line_under_quiet_from_every_page( + hud: Hud, platform: Platform, name: str +) -> None: + """The platform pages its project and model catalogs; one item a page makes a list + of several read every page.""" + argv, seed = LISTS[name] + expected = seed(hud, platform) + platform.page_size = 1 + + result = hud(*argv, "--quiet") + + assert (result.exit_code, result.stdout.split("\n")[:-1]) == (0, expected), result + + +# ─── secrets ──────────────────────────────────────────────────────────── + +SECRET = "s3cr3t-c0d3" + + +def secrets_beside_an_environment(hud: Hud, platform: Platform) -> None: + environment(hud, platform) + write_files(hud.cwd, {".env": f"SECRET={SECRET}\n", "token.txt": SECRET}) + + +SECRETS: dict[str, tuple[list[str], Setup | None, dict[str, str], str]] = { + "set": (["set", f"HUD_API_KEY={SECRET}"], None, {}, SECRET), + "deploy": ( + [ + *("deploy", "--yes"), + *("--secret", "id=ENV_TOKEN,env=ENV_TOKEN"), + *("--secret", "id=FILE_TOKEN,src=token.txt"), + ], + secrets_beside_an_environment, + {"ENV_TOKEN": SECRET}, + SECRET, + ), + "eval": ( + [ + *("eval", "tasks.py", "openai_compatible", "-m", "custom"), + *("-c", f"api_key={SECRET}", "-c", "base_url=http://127.0.0.1:9", "--dry-run"), + ], + tasks, + {}, + SECRET, + ), + "harbor import": ( + ["harbor", "import", "tasks", "run-1", "--taskset", "tb2", "--yes"], + harbor_folders, + {}, + "signature", + ), + "harbor import, upload refused": ( + ["harbor", "import", "tasks", "run-1", "--taskset", "tb2", "--yes"], + together(harbor_folders, refused("PUT", "/uploads/{id}", 403)), + {}, + "signature", + ), +} + + +@pytest.mark.parametrize("name", SECRETS) +@pytest.mark.parametrize("mode", [[], ["--json"]], ids=["text", "json"]) +def test_a_secret_given_to_a_command_never_appears_in_its_output( + hud: Hud, platform: Platform, name: str, mode: list[str] +) -> None: + argv, setup, env, secret = SECRETS[name] + if setup is not None: + setup(hud, platform) + + result = hud(*argv, *mode, env=env) + + assert secret not in result.stdout + result.stderr, result diff --git a/tests/cli/test_platform_effects.py b/tests/cli/test_platform_effects.py new file mode 100644 index 000000000..64f0c8117 --- /dev/null +++ b/tests/cli/test_platform_effects.py @@ -0,0 +1,259 @@ +"""What a platform command asks the platform for, on the object its reference names. + +A reference is a name, a display name, an id, or an id in another spelling (upper +case, no dashes); each row runs one command and reads back what it acted on: a +platform record, the request it sent, the directory link, or the answer it printed. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import pytest + +from tests.support import Reply +from tests.support.platform import BROWSER_PROJECT_ID + +from .commands import ( + BASE_MODEL_ID, + ENV_ID, + IMPORT_ID, + JOB_ID, + MODEL_ID, + TB2_ID, + TRACE_ID, + Setup, + a_base_model, + a_job, + a_registered_environment, + a_running_import, + a_trainable_model, + nothing, + tasks, + together, +) + +if TYPE_CHECKING: + from tests.support import Hud, Platform + from tests.support.machine import Result + +TASK_ID = "00000000-0000-4000-a000-000000000005" + +Observe = Callable[["Hud", "Platform", "Result"], Any] + + +def printed(*path: str) -> Observe: + def observe(hud: Hud, platform: Platform, result: Result) -> Any: + found = result.json + for key in path: + found = (found or {}).get(key) + return found + + return observe + + +def linked(key: str) -> Observe: + def observe(hud: Hud, platform: Platform, result: Result) -> Any: + return json.loads((hud.cwd / ".hud" / "config.json").read_text()).get(key) + + return observe + + +def asked(path: str, *keys: str) -> Observe: + """The ``keys`` of the query the last GET of ``path`` carried.""" + + def observe(hud: Hud, platform: Platform, result: Result) -> Any: + query = platform.services.requests("api", "GET", path)[-1].query + return {key: query.get(key) for key in keys} + + return observe + + +def asked_for(path: str) -> Observe: + """The path parameters of the last GET of ``path``.""" + + def observe(hud: Hud, platform: Platform, result: Result) -> Any: + return platform.services.requests("api", "GET", path)[-1].params + + return observe + + +def cancellations(hud: Hud, platform: Platform, result: Result) -> Any: + return platform.cancellations + + +def heads(hud: Hud, platform: Platform, result: Result) -> Any: + return platform.heads + + +def forked_from(hud: Hud, platform: Platform, result: Result) -> Any: + return [model["forked_from"] for model in platform.models if "forked_from" in model] + + +def import_cancelled(hud: Hud, platform: Platform, result: Result) -> Any: + return platform.imports[IMPORT_ID].cancelled + + +def created_project(hud: Hud, platform: Platform, result: Result) -> Any: + (created,) = [project for project in platform.projects if project["name"] == "UI Evals"] + link = hud.cwd / ".hud" / "config.json" + linked_to = json.loads(link.read_text())["project_id"] if link.exists() else None + return (result.json["id"] == created["id"], created.get("description"), linked_to is not None) + + +def a_finished_import(hud: Hud, platform: Platform) -> None: + a_running_import(hud, platform) + platform.imports[IMPORT_ID].statuses = ["completed"] + + +def environments_sharing_a_prefix(hud: Hud, platform: Platform) -> None: + del hud + platform.add_environment("browser") + platform.add_environment("browser-anchor", id=ENV_ID) + + +def a_demo_taskset(hud: Hud, platform: Platform) -> None: + del hud + platform.add_taskset("demo") + + +def the_fork_name_is_taken(hud: Hud, platform: Platform) -> None: + a_base_model(hud, platform) + a_trainable_model(hud, platform) + platform.services.route( + "api", "POST", "/v2/models/fork", Reply(status=409, json={"detail": "Model exists"}) + ) + + +@dataclass(frozen=True) +class Effect: + argv: list[str] + observe: Observe + expected: Any + setup: Setup = nothing + + +EFFECTS: dict[str, Effect] = { + "jobs get: a job id without dashes": Effect( + ["jobs", JOB_ID.replace("-", "")], + asked_for("/v2/jobs/{id}/traces"), + {"id": JOB_ID}, + a_job, + ), + "jobs cancel: a job": Effect( + ["jobs", "cancel", JOB_ID, "--yes"], cancellations, [("job", {"job_id": JOB_ID})] + ), + "jobs cancel: one trace": Effect( + ["jobs", "cancel", JOB_ID, "--trace-id", TRACE_ID, "--yes"], + cancellations, + [("trace", {"job_id": JOB_ID, "trace_id": TRACE_ID})], + ), + "jobs cancel: every job": Effect( + ["jobs", "cancel", "--all", "--yes"], cancellations, [("all", {})] + ), + "models head: by display name": Effect( + ["models", "head", "owner/model + version"], + lambda hud, platform, result: (result.json["model_id"], result.json["head"]["id"]), + (MODEL_ID, "ckpt-2"), + a_trainable_model, + ), + "models head: by id": Effect( + ["models", "head", MODEL_ID], + lambda hud, platform, result: (result.json["model_id"], result.json["head"]["id"]), + (MODEL_ID, "ckpt-2"), + a_trainable_model, + ), + "models head: a model without checkpoints": Effect( + ["models", "head", "claude-sonnet-4-6"], printed("head"), None, a_base_model + ), + "models head --set: by short name": Effect( + ["models", "head", "my-sonnet", "--set", "ckpt-1", "--yes"], + heads, + {MODEL_ID: "ckpt-1"}, + a_trainable_model, + ), + "models fork: by display name": Effect( + ["models", "fork", "Claude Sonnet", "--name", "fresh", "--yes"], + forked_from, + [BASE_MODEL_ID], + a_base_model, + ), + "models fork --if-not-exists: onto a taken name": Effect( + ["models", "fork", "Claude Sonnet", "--name", "my-sonnet", "--if-not-exists", "--yes"], + lambda hud, platform, result: (result.json["id"], result.json["existed"]), + (MODEL_ID, True), + the_fork_name_is_taken, + ), + "project create: links the directory": Effect( + ["project", "create", "UI Evals", "--description", "UI tasks", "--yes"], + created_project, + (True, "UI tasks", True), + ), + "project create --no-use": Effect( + ["project", "create", "UI Evals", "--no-use", "--yes"], created_project, (True, None, False) + ), + "project use: by display name": Effect( + ["project", "use", "Browser Evals"], linked("project_id"), BROWSER_PROJECT_ID + ), + "project use: by an upper-case id": Effect( + ["project", "use", BROWSER_PROJECT_ID.upper()], linked("project_id"), BROWSER_PROJECT_ID + ), + "project list: marks the default and read-only projects": Effect( + ["project", "list"], + lambda hud, platform, result: [ + (p["name"], p["is_default"], p["can_create"]) for p in result.json + ], + [("default", True, True), ("browser-evals", False, True), ("locked-down", False, False)], + ), + "sync env: by name across separators": Effect( + ["sync", "env", "browser_anchor"], + linked("registry_id"), + ENV_ID, + environments_sharing_a_prefix, + ), + "sync env: by id": Effect( + ["sync", "env", ENV_ID], linked("registry_id"), ENV_ID, a_registered_environment + ), + "harbor imports: a taskset by name": Effect( + ["harbor", "imports", "--taskset", "tb2"], + asked("/v2/imports", "taskset_id", "format"), + {"taskset_id": [TB2_ID], "format": ["harbor"]}, + a_running_import, + ), + "harbor cancel: a live import": Effect( + ["harbor", "cancel", IMPORT_ID, "--yes"], import_cancelled, True, a_running_import + ), + "harbor cancel: an ended import": Effect( + ["harbor", "cancel", IMPORT_ID, "--yes"], import_cancelled, False, a_finished_import + ), + "sync tasks --project: places the upload without linking it": Effect( + ["sync", "tasks", "demo", "tasks.py", "--project", BROWSER_PROJECT_ID, "--yes"], + lambda hud, platform, result: ( + [upload["project_id"] for upload in platform.uploads], + (hud.cwd / ".hud" / "config.json").exists(), + ), + ([BROWSER_PROJECT_ID], False), + together(tasks, a_demo_taskset), + ), + "qa results: by task": Effect( + ["qa", "results", "--level", "task", TASK_ID], + asked("/v2/qa/results", "task_ids"), + {"task_ids": [TASK_ID]}, + ), +} + + +@pytest.mark.parametrize("case", EFFECTS) +def test_a_command_acts_on_the_object_its_reference_names( + hud: Hud, platform: Platform, case: str +) -> None: + effect = EFFECTS[case] + effect.setup(hud, platform) + + result = hud(*effect.argv, "--json") + + assert result.exit_code == 0, result + assert effect.observe(hud, platform, result) == effect.expected diff --git a/tests/cli/test_qa.py b/tests/cli/test_qa.py new file mode 100644 index 000000000..d2703f38b --- /dev/null +++ b/tests/cli/test_qa.py @@ -0,0 +1,170 @@ +"""``hud qa``: list QA checks, run them on traces or tasks, and read their verdicts.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest + +if TYPE_CHECKING: + from tests.support import Hud, Platform + +TRACE_ID = "00000000-0000-4000-a000-000000000002" +TASK_ID = "00000000-0000-4000-a000-000000000005" +CHECK = "failure_analysis" +OTHER_CHECK = "reward_hacking" +TASK_CHECK = "task_reward_exploit" +CHECKS = [ + { + "key": CHECK, + "subject_type": "trace", + "title": "Failure Analysis", + "question": "Why did the agent fail?", + }, + { + "key": OTHER_CHECK, + "subject_type": "trace", + "title": "Reward Hacking", + "question": "Did the agent game the grader?", + }, + { + "key": TASK_CHECK, + "subject_type": "task", + "title": "Task Reward Exploit", + "question": "Can the reward be earned without solving the task?", + }, +] + + +@pytest.fixture(autouse=True) +def catalog(platform: Platform) -> None: + """The platform's QA check catalog, which ``hud qa`` reads to place checks by level.""" + platform.qa_checks = CHECKS + + +def row(status: str = "completed", **fields: Any) -> dict[str, Any]: + """One ``/v2/qa/results`` row as the platform returns it.""" + return { + "id": "00000000-0000-4000-a000-000000000003", + "check_key": CHECK, + "subject_type": "trace", + "subject_trace_id": TRACE_ID, + "source": "analysis", + "status": status, + "verdict": None, + "result": None, + "legacy_result": None, + "note": None, + "error": None, + **fields, + } + + +def verdict(tag: str, *, check_key: str = CHECK) -> dict[str, Any]: + """A completed ``qa_agent_result.v1`` row.""" + result = {"schema_version": "qa_agent_result.v1", "verdict": tag, "summary": "", "findings": []} + return row(check_key=check_key, verdict=tag, result=result) + + +def started(platform: Platform) -> list[tuple[str, str]]: + """The (check, subject) pairs QA runs were started for.""" + return [ + (r["check_key"], r.get("subject_trace_id") or r["subject_task_id"]) + for r in platform.qa_results + ] + + +@pytest.mark.parametrize( + ("argv", "runs"), + [ + ( + [f"{CHECK}, {OTHER_CHECK},{CHECK}", TRACE_ID.upper()], + [(CHECK, TRACE_ID), (OTHER_CHECK, TRACE_ID)], + ), + ([TASK_CHECK, TASK_ID], [(TASK_CHECK, TASK_ID)]), + ], + ids=["deduplicated-and-normalised", "task-level"], +) +def test_qa_run_without_waiting_starts_each_check_once( + hud: Hud, platform: Platform, argv: list[str], runs: list[tuple[str, str]] +) -> None: + result = hud("qa", "run", *argv, "--no-wait", "--yes", "--json") + + assert result.exit_code == 0, result + assert started(platform) == runs + + +@pytest.mark.parametrize( + ("outcome", "exit_code", "verdicts"), + [ + pytest.param(verdict("passed"), 0, ["passed"], id="passed"), + pytest.param(verdict("failed"), 1, ["failed"], id="failed"), + pytest.param(row("error", error="invalid result"), 1, [None], id="errored"), + ], +) +def test_qa_run_waits_for_every_check_and_fails_unless_all_pass( + hud: Hud, + platform: Platform, + outcome: dict[str, Any], + exit_code: int, + verdicts: list[str | None], +) -> None: + another_check = verdict("failed", check_key=OTHER_CHECK) + platform.qa_polls = [[row("running"), another_check], [outcome, another_check]] + + result = hud("qa", "run", CHECK, TRACE_ID, "--yes", "--json") + + assert result.exit_code == exit_code, result + assert [(r["check_key"], r["verdict"]) for r in result.json] == [ + (CHECK, tag) for tag in verdicts + ] + + +def test_qa_run_times_out_while_checks_are_running(hud: Hud, platform: Platform) -> None: + platform.qa_polls = [[row("running")]] + + result = hud("qa", "run", CHECK, TRACE_ID, "--timeout", "1", "--yes", "--json") + + assert (result.exit_code, result.json["error"]) == (1, "timeout"), result + + +def legacy(check_key: str = CHECK, **output: Any) -> dict[str, Any]: + """A result stored before ``qa_agent_result.v1``, as its check's own JSON.""" + return row(check_key=check_key, legacy_result={"content": json.dumps(output)}) + + +@pytest.mark.parametrize( + ("results", "verdicts"), + [ + pytest.param( + [{**verdict("failed"), "source": "manual", "verdict": "passed"}], + ["passed"], + id="a-manual-verdict-wins", + ), + pytest.param( + [legacy(problems=[{"problem": "No output", "fault": "agent"}])], + ["failed"], + id="legacy-agent-fault", + ), + pytest.param( + [ + legacy("false_negative", is_false_negative=False, reasoning="Matches."), + legacy("false_negative", is_false_negative=True, reasoning="Grader missed it."), + ], + ["passed", "failed"], + id="legacy-false-negative", + ), + ], +) +def test_qa_results_read_a_verdict_from_each_result_shape( + hud: Hud, platform: Platform, results: list[dict[str, Any]], verdicts: list[str] +) -> None: + platform.qa_results += results + + text = hud("qa", "results", TRACE_ID) + + assert text.exit_code == 0, text + assert [ + line.removeprefix("Verdict: ") for line in text.lines if line.startswith("Verdict:") + ] == verdicts diff --git a/tests/cli/test_settings.py b/tests/cli/test_settings.py new file mode 100644 index 000000000..21cb5ecb6 --- /dev/null +++ b/tests/cli/test_settings.py @@ -0,0 +1,38 @@ +"""Where the SDK's configuration comes from, and what a few settings switch.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from tests.support import Hud, HudEnv, Platform + + +@pytest.mark.parametrize( + ("process", "project", "user", "sent"), + [ + pytest.param(None, None, "from-hud-set", "from-hud-set", id="hud-set-only"), + pytest.param(None, "from-project", "from-hud-set", "from-project", id="project-wins"), + pytest.param("from-env", "from-project", "from-hud-set", "from-env", id="env-wins"), + ], +) +def test_the_api_key_comes_from_env_then_project_env_then_hud_set( + platform: Platform, + hud_env: HudEnv, + hud: Hud, + process: str | None, + project: str | None, + user: str, + sent: str, +) -> None: + platform.keys = {sent} + assert hud("set", f"HUD_API_KEY={user}").exit_code == 0 + if project is not None: + (hud.cwd / ".env").write_text(f"HUD_API_KEY={project}\n") + hud_env.set(HUD_API_KEY=process) + + result = hud("jobs", "list", "--json") + + assert result.exit_code == 0, result diff --git a/tests/cli/test_shell.py b/tests/cli/test_shell.py new file mode 100644 index 000000000..07d5dff51 --- /dev/null +++ b/tests/cli/test_shell.py @@ -0,0 +1,190 @@ +"""What ``hud`` does around every command: the registry of commands, help and unknown +commands, the upgrade notice, anonymous analytics, and an interrupt.""" + +from __future__ import annotations + +import json +import signal +import sys +import time +from typing import TYPE_CHECKING, Any + +import pytest +from dirty_equals import IsInt, IsStr, IsUUID + +from hud.version import __version__ +from tests.support import Reply + +from .commands import ( + EVERY_COMMAND, + LOCAL_ONLY, + MUTATING, + OFFLINE, + PLATFORM, + READ_ONLY, + TREE, + answer_everything, + failing, + invocation, +) + +if TYPE_CHECKING: + from tests.support import Hud, HudEnv, Platform + + +def test_every_command_is_registered_and_classified() -> None: + by_reach = [set(PLATFORM), set(OFFLINE)] + by_change = [MUTATING, LOCAL_ONLY, READ_ONLY] + + for kinds in (by_reach, by_change): + assert sorted(set(TREE) - set().union(*kinds)) == [] + assert sorted(set().union(*kinds) - set(TREE)) == [] + assert sum(len(kind) for kind in kinds) == len(TREE) + + +def test_bare_hud_prints_help_and_exits_2(hud: Hud) -> None: + result = hud() + + assert (result.exit_code, result.stderr) == (2, "") + assert "Usage:" in result.stdout and "COMMAND" in result.stdout + + +def test_an_unknown_command_is_a_usage_error_on_stderr(hud: Hud) -> None: + result = hud("nope") + + assert (result.exit_code, result.stdout) == (2, ""), result + assert "nope" in result.stderr + + +@pytest.mark.parametrize( + ("latest", "environment", "notified"), + [ + ("999.0.0", {}, True), + ("999.0.0", {"CI": "1"}, False), + (__version__, {}, False), + ], +) +def test_an_outdated_install_prints_an_upgrade_notice( + hud: Hud, + hud_env: HudEnv, + latest: str, + environment: dict[str, str], + notified: bool, +) -> None: + """The notice reads PyPI's latest version from a fresh cache, so no request is made.""" + hud_env.set(HUD_SKIP_VERSION_CHECK=None, CI=None) + cache = hud_env.home / ".hud" / ".cache" / "version_check.json" + cache.parent.mkdir(parents=True) + cache.write_text(json.dumps({"latest": latest, "checked_at": time.time()})) + + result = hud("version", env=environment) + + assert result.exit_code == 0, result + assert ("999.0.0" in result.stderr) is notified + + +# ─── analytics ────────────────────────────────────────────────────────── + + +def event(name: str, exit_code: int, error: Any) -> dict[str, Any]: + """The anonymous event an invocation of command ``name`` posts.""" + command, _, subcommand = name.partition(" ") + return { + "command": command, + "subcommand": subcommand or None, + "exit_code": exit_code, + "error_class": error, + "duration_ms": IsInt(ge=0), + "cli_version": __version__, + "python_version": IsStr(regex=r"3\.\d+\.\d+"), + "os": sys.platform, + "is_ci": False, + "install_id": IsUUID, + } + + +@pytest.mark.parametrize("name", EVERY_COMMAND) +def test_every_invocation_posts_one_anonymous_event_naming_its_command( + hud: Hud, analytics: Platform, hud_env: HudEnv, name: str +) -> None: + argv = failing(name, hud, analytics, hud_env) + before = len(analytics.cli_events) + + result = hud(*argv) + + assert result.exit_code != 0 + assert analytics.cli_events[before:] == [event(name, result.exit_code, IsStr)] + + +@pytest.mark.parametrize( + ("argv", "expected"), + [ + (["version"], [event("version", 0, None)]), + (["secret-name", "arg"], [event("other", 2, "UsageError")]), + (["--version"], []), + ], + ids=["a-command", "an-unknown-command-is-not-named", "the-version-flag"], +) +def test_what_is_not_a_command_posts_nothing_that_names_it( + hud: Hud, analytics: Platform, argv: list[str], expected: list[dict[str, Any]] +) -> None: + hud(*argv) + + assert analytics.cli_events == expected + assert "secret-name" not in json.dumps(analytics.cli_events) + + +def test_the_install_id_is_created_once_and_announced_once(hud: Hud, analytics: Platform) -> None: + first = hud("version") + second = hud("version") + + assert len({event["install_id"] for event in analytics.cli_events}) == 1 + assert "HUD_CLI_ANALYTICS_ENABLED=0" in first.stderr + assert second.stderr == "" + + +def test_analytics_opt_out_posts_nothing(hud: Hud, analytics: Platform, hud_env: HudEnv) -> None: + hud_env.set(HUD_CLI_ANALYTICS_ENABLED="0") + + hud("version") + + assert analytics.services.requests("telemetry") == [] + + +def test_a_stalled_analytics_endpoint_does_not_hold_up_the_command( + hud: Hud, analytics: Platform +) -> None: + analytics.services.route("telemetry", "POST", "/sdk-events/cli", Reply(json={}, delay=20)) + + started = time.monotonic() + result = hud("version") + elapsed = time.monotonic() - started + + assert result.exit_code == 0 + assert len(analytics.services.requests("telemetry")) == 1 + # Without the bound the command would outlast the 20 s stall; with it, a few seconds. + assert elapsed < 15, elapsed + + +# ─── interrupt ────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("name", sorted(set(PLATFORM) - {"deploy"})) +def test_an_interrupted_command_exits_130( + hud: Hud, analytics: Platform, hud_env: HudEnv, name: str +) -> None: + PLATFORM[name].setup(hud, analytics) + answer_everything(analytics, Reply(json={}, delay=60)) + asked, posted = len(analytics.services.requests("api")), len(analytics.cli_events) + process = hud.start(*invocation(name)) + + deadline = time.monotonic() + 30 + while len(analytics.services.requests("api")) == asked: + assert process.poll() is None, process.communicate() + assert time.monotonic() < deadline, f"hud {name} never reached the platform" + time.sleep(0.05) + process.send_signal(signal.SIGINT) + process.communicate(timeout=30) + + assert process.returncode == 130 + assert analytics.cli_events[posted:] == [event(name, 130, "KeyboardInterrupt")] diff --git a/tests/cli/test_sync.py b/tests/cli/test_sync.py new file mode 100644 index 000000000..6c6e6b433 --- /dev/null +++ b/tests/cli/test_sync.py @@ -0,0 +1,150 @@ +"""``hud sync``: push local tasks to a platform taskset, export it, and link an environment.""" + +from __future__ import annotations + +import csv +import json +from typing import TYPE_CHECKING, Any + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import Hud, Platform + +TASKS_PY = """\ +from hud import Environment + +env = Environment("example") + + +@env.template(id="solve") +async def solve(n: int = 0): + yield f"solve {n}" + yield 1.0 + + +tasks = [solve(n=1), solve(n=2)] +""" + + +@pytest.fixture +def source(hud: Hud) -> Path: + path = hud.cwd / "tasks.py" + path.write_text(TASKS_PY) + return path + + +def link(hud: Hud, key: str = "taskset_id") -> Any: + path = hud.cwd / ".hud" / "config.json" + return json.loads(path.read_text()).get(key) if path.exists() else None + + +def stored(platform: Platform, taskset_id: str) -> list[tuple[str, str, dict[str, Any]]]: + """A taskset's tasks as (env, template, args), sorted.""" + tasks = platform.tasksets[taskset_id]["tasks"].values() + return sorted(((t["env"], t["scenario"], t["args"]) for t in tasks), key=repr) + + +def test_a_taskset_round_trip(hud: Hud, platform: Platform, source: Path) -> None: + plan = hud("sync", "tasks", "demo", "tasks.py", "--dry-run", "--json") + assert (plan.exit_code, plan.json["create_count"], platform.tasksets, link(hud)) == ( + 0, + 2, + {}, + None, + ) + + first = hud("sync", "tasks", "demo", "tasks.py", "--yes", "--json") + assert first.exit_code == 0, first + taskset_id = first.json["taskset_id"] + assert stored(platform, taskset_id) == [ + ("example", "solve", {"n": 1}), + ("example", "solve", {"n": 2}), + ] + assert link(hud) == taskset_id + + again = hud("sync", "tasks", "demo", "tasks.py", "--yes", "--json") + assert (again.json["status"], len(platform.uploads)) == ("up_to_date", 1) + + source.write_text(TASKS_PY.replace("solve(n=2)]", "solve(n=2), solve(n=3)]")) + edited = hud("sync", "tasks", "--yes", "--json") + assert edited.exit_code == 0, edited + assert [task["args"] for task in platform.uploads[-1]["tasks"]] == [{"n": 3}] + assert len(stored(platform, taskset_id)) == 3 + + platform.tasksets[taskset_id]["name"] = "renamed" + bare = hud("sync", "--json") + assert (bare.exit_code, bare.json["status"], bare.json["taskset"]) == ( + 0, + "up_to_date", + "renamed", + ) + forced = hud("sync", "tasks", "--force", "--yes", "--json") + assert forced.exit_code == 0, forced + last = platform.uploads[-1] + assert (last["taskset_id"], last["taskset_name"], len(last["tasks"])) == ( + taskset_id, + "renamed", + 3, + ) + + slug = platform.uploads[0]["tasks"][0]["name"] + one = hud("sync", "tasks", "--task", slug, "--force", "--yes", "--json") + assert one.exit_code == 0, one + assert [task["name"] for task in platform.uploads[-1]["tasks"]] == [slug] + + other = platform.add_taskset("other") + elsewhere = hud("sync", "tasks", other, "tasks.py", "--force", "--yes", "--json") + planned = hud("sync", "tasks", other, "tasks.py", "--link", "--dry-run", "--json") + assert (elsewhere.exit_code, planned.exit_code, link(hud)) == (0, 0, taskset_id) + assert len(stored(platform, other)) == 3 + relinked = hud("sync", "tasks", other, "tasks.py", "--link", "--yes", "--json") + assert (relinked.exit_code, link(hud)) == (0, other) + + +REMOTE: list[dict[str, Any]] = [ + {"name": "one", "env": "e", "scenario": "solve", "args": {"n": 1}}, + {"name": "two", "env": "e", "scenario": "solve", "args": {"n": {"x": 2}}}, +] + + +@pytest.mark.parametrize("target", ["tasks.json", "out/tasks.jsonl"]) +def test_an_exported_taskset_reads_back_as_the_same_tasks( + hud: Hud, platform: Platform, target: str +) -> None: + platform.add_taskset("demo", REMOTE) + + result = hud("sync", "tasks", "demo", "--export", target, "--json") + rows = hud("task", "list", "--source", target, "--json") + + assert result.exit_code == 0, result + assert [(r["slug"], r["id"], r["args"]) for r in rows.json] == [ + (t["name"], t["scenario"], t["args"]) for t in REMOTE + ] + + +def test_a_csv_export_has_a_column_per_argument(hud: Hud, platform: Platform) -> None: + platform.add_taskset("demo", REMOTE) + + result = hud("sync", "tasks", "demo", "--export", "tasks.csv", "--json") + + assert result.exit_code == 0, result + with (hud.cwd / "tasks.csv").open(newline="") as exported: + rows = list(csv.DictReader(exported)) + assert [(r["slug"], r["env"], r["id"], json.loads(r["arg:n"])) for r in rows] == [ + (t["name"], t["env"], t["scenario"], t["args"]["n"]) for t in REMOTE + ] + + +def test_tasks_for_another_environment_than_the_linked_one_are_flagged( + hud: Hud, platform: Platform, source: Path +) -> None: + platform.add_environment("browser") + assert hud("sync", "env", "browser").exit_code == 0 + + result = hud("sync", "tasks", "demo", "tasks.py", "--dry-run") + + assert result.exit_code == 0, result + assert "'browser': example" in " ".join(result.stderr.split()) diff --git a/tests/cli/test_task.py b/tests/cli/test_task.py new file mode 100644 index 000000000..2456f485b --- /dev/null +++ b/tests/cli/test_task.py @@ -0,0 +1,144 @@ +"""``hud task``: start a source's task for its prompt and grade an answer, locally or on an +environment ``hud serve`` or a runtime serves.""" + +from __future__ import annotations + +import asyncio +import json +import signal +import socket +from typing import TYPE_CHECKING, Any + +import pytest + +from hud import Environment +from hud.clients import connect +from hud.eval import LocalRuntime, Task + +if TYPE_CHECKING: + from tests.support import Hud + +ENV_PY = """\ +from hud import Environment + +env = Environment("example") + + +@env.template(id="solve") +async def solve(target: str = "answer"): + answer = yield f"Say {target}." + yield 1.0 if answer == target else 0.0 +""" +ROWS = [ + {"env": "example", "id": "solve", "slug": "say-answer"}, + {"env": "example", "id": "solve", "slug": "say-hello", "args": {"target": "hello"}}, +] + + +@pytest.fixture +def project(hud: Hud) -> Any: + """An env with a Python task source (``tasks.py``) and authored rows (``tasks.json``).""" + (hud.cwd / "env.py").write_text(ENV_PY) + (hud.cwd / "tasks.py").write_text( + 'from env import solve\ntasks = [solve(), solve(target="hello")]\n' + ) + (hud.cwd / "tasks.json").write_text(json.dumps(ROWS)) + return hud.cwd + + +@pytest.mark.parametrize( + ("argv", "stdout"), + [ + (["start", "say-hello", "--source", "tasks.json"], "Say hello.\n"), + (["start", "1", "--source", "tasks.py"], "Say hello.\n"), + (["grade", "say-answer", "-s", "tasks.json", "--answer-file", "-"], "1.0\n"), + ], +) +def test_a_task_spawned_from_its_source_starts_and_grades( + hud: Hud, project: Any, argv: list[str], stdout: str +) -> None: + result = hud("task", *argv, input="answer") + + assert (result.exit_code, result.stdout) == (0, stdout), result + + +# ─── attached to a served environment ─────────────────────────────────── + + +def counting_env(starts: list[str], *, failing: bool = False) -> Environment: + env = Environment("example") + + @env.template(id="solve") + async def solve(target: str = "answer"): + starts.append(target) + answer = yield f"Say {target}." + if failing: + raise ValueError("grader failed") + yield 1.0 if answer == target else 0.0 + + return env + + +@pytest.mark.parametrize( + ("parked", "failing", "starts", "outcome"), + [ + pytest.param(0, False, 1, {"score": 1.0}, id="starts-when-nothing-is-parked"), + pytest.param(1, False, 1, {"score": 1.0}, id="resumes-a-parked-session"), + pytest.param(1, True, 1, {"error": "failure"}, id="grader-error"), + pytest.param(2, False, 2, {"error": "failure"}, id="ambiguous-parked-sessions"), + ], +) +async def test_grade_resumes_a_started_task_on_a_served_env( + hud: Hud, parked: int, failing: bool, starts: int, outcome: dict[str, Any] +) -> None: + started: list[str] = [] + async with LocalRuntime(counting_env(started, failing=failing))( + Task(env="example", id="solve") + ) as runtime: + for _ in range(parked): + async with connect(runtime) as client: + await client.start_task("solve", {}) + result = await asyncio.to_thread( + hud, "task", "grade", "solve", "--url", runtime.url, "--answer", "answer", "--json" + ) + + assert result.exit_code == ("error" in outcome), result + assert len(started) == starts + assert {key: result.json[key] for key in outcome} == outcome + + +def test_a_task_starts_on_the_env_hud_serve_serves_until_interrupted( + hud: Hud, project: Any +) -> None: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + server = hud.start("serve", "env.py", "--port", str(port)) + try: + url = f"tcp://127.0.0.1:{port}" + result = hud("task", "start", "say-hello", "--source", "tasks.json", "--url", url) + finally: + server.send_signal(signal.SIGINT) + server.communicate(timeout=30) + + assert (result.exit_code, result.stdout) == (0, "Say hello.\n"), result + assert server.returncode == 0 + + +@pytest.mark.parametrize( + ("argv", "target"), + [ + (["--source", "tasks.json"], "hello"), + (["--args", '{"target": "hello"}'], "hello"), + ], +) +async def test_a_url_runs_the_task_a_source_names_or_the_raw_id( + hud: Hud, project: Any, argv: list[str], target: str +) -> None: + started: list[str] = [] + task = "say-hello" if "--source" in argv else "solve" + async with LocalRuntime(counting_env(started))(Task(env="example", id="solve")) as runtime: + result = await asyncio.to_thread(hud, "task", "start", task, "--url", runtime.url, *argv) + + assert (result.exit_code, result.stdout) == (0, f"Say {target}.\n"), result + assert started == [target] diff --git a/tests/cli/test_trace.py b/tests/cli/test_trace.py new file mode 100644 index 000000000..1bfcc523f --- /dev/null +++ b/tests/cli/test_trace.py @@ -0,0 +1,137 @@ +"""``hud trace``: a rollout's turns, from local spans or the platform.""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Any + +import pytest + +from hud import Environment +from hud.agents.openai_compatible import OpenAIChatAgent +from hud.agents.types import OpenAIChatConfig +from hud.eval import LocalRuntime, Task, rollout +from tests.support import call, say + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import Hud, HudEnv, Models, Platform + +TRACE_ID = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" +COMPACT_TRACE_ID = "03dd2a73d3df4d10a54ae3d87c2d530d" +EVENTS: list[dict[str, Any]] = [ + { + "kind": "agent_message", + "text": "Splitting at [len(s) // 2] and [bold]this[/bold]", + "reasoning": "Think about the middle.", + "tool_calls": [{"name": "bash", "arguments": {"command": "ls"}}], + }, + {"kind": "tool_call", "tool_name": "bash", "result_text": "a.txt\nb.txt"}, + {"kind": "tool_call", "tool_name": "bash", "error": "exit status 1"}, + {"kind": "agent_message", "text": "done", "error": "max steps reached"}, +] + + +@pytest.fixture +def events(platform: Platform) -> None: + platform.trace(TRACE_ID).events = EVENTS + + +def test_a_platform_trace_prints_the_models_text_verbatim(hud: Hud, events: None) -> None: + result = hud("trace", COMPACT_TRACE_ID) + + assert result.exit_code == 0, result + for shown in (EVENTS[0]["text"], EVENTS[0]["reasoning"], "exit status 1", "max steps reached"): + assert shown in result.stdout + + +# ─── local spans ──────────────────────────────────────────────────────── + + +async def write_a_report( + models: Models, hud_env: HudEnv, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> str: + """Run a rollout whose scripted model writes REPORT.md through a tool; return its trace id. + + Spans land in ``spans/`` under ``tmp_path``, the working directory of the + rollout and of ``hud``, so rendered paths stay short. + """ + monkeypatch.chdir(tmp_path) + hud_env.set(HUD_TELEMETRY_LOCAL_DIR="spans") + workspace = tmp_path / "workspace" + report = workspace / "REPORT.md" + env = Environment("report") + env.workspace(workspace, guest_path=str(workspace)) + + @env.initialize + async def seed() -> None: + workspace.mkdir(parents=True, exist_ok=True) + + @env.template() + async def write_report(): + yield "Write PASS to REPORT.md." + yield 1.0 if report.read_text().strip() == "PASS" else 0.0 + + models.script([call("write", filePath="REPORT.md", content="PASS"), say("Wrote it.")]) + agent = OpenAIChatAgent(OpenAIChatConfig(model="scripted", max_steps=4)) + run = await rollout(Task(env="report", id="write_report"), agent, runtime=LocalRuntime(env)) + assert run.reward == 1.0 + assert run.trace_id is not None + return run.trace_id + + +async def test_a_local_trace_lists_tool_calls_with_their_results( + hud: Hud, + platform: Platform, + models: Models, + hud_env: HudEnv, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + trace_id = await write_a_report(models, hud_env, tmp_path, monkeypatch) + + result = hud("trace", "get", trace_id, "--json", cwd=tmp_path) + + assert result.exit_code == 0, result + assert [(event["kind"], event.get("tool_name")) for event in result.json] == [ + ("agent_message", None), + ("tool_call", "write"), + ("agent_message", None), + ] + assert result.json[1]["arguments"] == {"filePath": "REPORT.md", "content": "PASS"} + assert "4 bytes" in result.json[1]["result_text"] + assert result.json[2]["text"] == "Wrote it." + + +async def test_a_local_trace_renders_and_skips_a_record_cut_short( + hud: Hud, + platform: Platform, + models: Models, + hud_env: HudEnv, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + trace_id = await write_a_report(models, hud_env, tmp_path, monkeypatch) + spans = tmp_path / "spans" / f"{uuid.UUID(trace_id).hex}.jsonl" + complete = spans.read_text() + spans.write_text(complete + complete.splitlines()[-1][:40]) + + result = hud("trace", trace_id, cwd=tmp_path) + + assert result.exit_code == 0, result + assert "Wrote it." in result.stdout + assert "Skipped 1 incomplete span record" in result.stderr + + +def test_a_trace_missing_from_the_local_spans_is_read_from_the_platform( + hud: Hud, events: None, hud_env: HudEnv, tmp_path: Path +) -> None: + spans = tmp_path / "spans" + spans.mkdir() + (spans / f"{uuid.uuid4().hex}.jsonl").write_text("") + hud_env.set(HUD_TELEMETRY_LOCAL_DIR=str(spans)) + + result = hud("trace", "get", TRACE_ID, "--json") + + assert (result.exit_code, result.json) == (0, EVENTS), result diff --git a/hud/agents/gemini/tools/tests/__init__.py b/tests/clients/__init__.py similarity index 100% rename from hud/agents/gemini/tools/tests/__init__.py rename to tests/clients/__init__.py diff --git a/tests/clients/test_capability_clients.py b/tests/clients/test_capability_clients.py new file mode 100644 index 000000000..c4a75d98d --- /dev/null +++ b/tests/clients/test_capability_clients.py @@ -0,0 +1,366 @@ +"""The capability client contract, held by every client ``hud.capabilities`` exports. + +Each client is a row, keyed by its exported name, with a backend: a served +workspace's SSH daemon, the fake DevTools endpoint, the fake VNC screen, or a +FastMCP server. An Environment publishes the backend, it is served on loopback, +and the client reaches it the way an agent does: at the binding the control +channel hands out, here through a relay (:func:`tests.support.relay`) that can +drop the connection. A client that cannot take part in a scenario is listed with +the reason beside that scenario's table, and a client the package exports with +neither fails :func:`test_every_exported_client_is_a_row_of_every_table`. +""" + +from __future__ import annotations + +import asyncio +import io +import socket +from contextlib import asynccontextmanager +from dataclasses import dataclass, replace +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit, urlunsplit + +import httpx +import pytest +from fastmcp import FastMCP +from mcp.shared.exceptions import McpError +from PIL import Image + +import hud.capabilities +from hud.capabilities import Capability, CapabilityClient +from hud.environment import Environment +from tests.fixtures.environments import served +from tests.support import eventually, fake_browser, fake_screen, mcp_server, relay + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine + from contextlib import AbstractAsyncContextManager + + from tests.support import FlakyRelay, HudEnv + +COLOR = (10, 200, 30) + + +@dataclass(frozen=True) +class Backend: + """What a row serves, and how a scenario drives and observes it.""" + + env: Environment + #: The capability's name in the manifest. + name: str + #: A call whose answer proves the client reached the backend, and that answer. + probe: Callable[[Any], Awaitable[object]] + answer: object + #: A call the backend never finishes, under a timeout (``None``: no timeout). + held: Callable[[Any, float | None], Coroutine[Any, Any, object]] | None = None + #: Whether the held call reached the backend. + reached: Callable[[], bool] = lambda: True + #: Whether the backend ended the held call; ``None`` where it cannot tell. + ended: Callable[[], bool] | None = None + #: Whether the client has seen its connection drop. + dropped: Callable[[Any], bool] = lambda client: True + + +def running(pid: int) -> bool: + """Whether ``pid`` is a live process (an unreaped zombie counts as gone).""" + stat = Path(f"/proc/{pid}/stat") + try: + return stat.read_text().split(") ", 1)[1][0] != "Z" + except FileNotFoundError: + return False + + +@asynccontextmanager +async def ssh(tmp_path: Path) -> AsyncIterator[Backend]: + root = tmp_path / "workspace" + root.mkdir() + env = Environment("ssh") + env.workspace(root, track_files=False) + pid = root / "pid" + + async def probe(client: Any) -> object: + return (await client.run("echo ok", timeout=10)).stdout + + yield Backend( + env, + "shell", + probe, + "ok\n", + held=lambda client, timeout: client.run("echo $$ > pid; exec sleep 30", timeout=timeout), + reached=lambda: pid.is_file() and bool(pid.read_text().strip()), + ended=lambda: not running(int(pid.read_text())), + dropped=lambda client: client.conn.is_closed(), + ) + + +@asynccontextmanager +async def cdp(tmp_path: Path) -> AsyncIterator[Backend]: + replies = {"Page.navigate": {"result": {"frameId": "f-1"}}, "Runtime.evaluate": {"hold": True}} + async with fake_browser(replies=replies) as browser: + page = Capability.cdp(name="browser", url=f"{browser.url}/devtools/page/page-1") + yield Backend( + Environment("cdp", capabilities=[page]), + "browser", + lambda client: client.send("Page.navigate", {"url": "https://example.com"}), + {"frameId": "f-1"}, + held=lambda client, timeout: asyncio.wait_for( + client.send("Runtime.evaluate", {"expression": "1"}), timeout + ), + reached=lambda: any(c.method == "Runtime.evaluate" for c in browser.commands), + ) + + +@asynccontextmanager +async def rfb(tmp_path: Path) -> AsyncIterator[Backend]: + async def probe(client: Any) -> object: + data, _ = await client.screenshot_png() + return Image.open(io.BytesIO(data)).convert("RGB").getpixel((0, 0)) + + async with fake_screen(width=4, height=2, color=COLOR) as screen: + yield Backend( + Environment("rfb", capabilities=[Capability.rfb(name="screen", url=screen.url)]), + "screen", + probe, + COLOR, + ) + + +@asynccontextmanager +async def mcp(tmp_path: Path) -> AsyncIterator[Backend]: + started: list[str] = [] + tools = FastMCP("tools") + + @tools.tool + def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + @tools.tool + async def wait() -> str: + """Never answer.""" + started.append("wait") + await asyncio.Event().wait() + return "unreachable" + + async def probe(client: Any) -> object: + return (await client.call_tool("add", {"a": 2, "b": 3})).structuredContent + + async with mcp_server(tools) as capability: + yield Backend( + Environment("mcp", capabilities=[capability]), + "tools", + probe, + {"result": 5}, + held=lambda client, timeout: asyncio.wait_for(client.call_tool("wait", {}), timeout), + reached=lambda: started == ["wait"], + ) + + +BACKENDS: dict[str, Callable[[Path], AbstractAsyncContextManager[Backend]]] = { + "SSHClient": ssh, + "CDPClient": cdp, + "RFBClient": rfb, + "MCPClient": mcp, +} +#: Clients held to the contract by another area. +ELSEWHERE = {"RobotClient": "needs the robot extra; tests/robot/test_slots.py holds its contract"} + + +def client_class(name: str) -> type[CapabilityClient]: + return getattr(hud.capabilities, name) + + +@asynccontextmanager +async def connected(name: str, tmp_path: Path) -> AsyncIterator[tuple[Any, Backend, FlakyRelay]]: + """The row's client, connected to its served backend's binding through a relay.""" + async with BACKENDS[name](tmp_path) as backend, served(backend.env) as control: + binding = urlsplit(control.binding(backend.name).url) + assert binding.hostname is not None and binding.port is not None + async with relay(binding.hostname, binding.port) as flaky: + user = f"{binding.username}@" if binding.username else "" + netloc = f"{user}127.0.0.1:{flaky.port}" + url = urlunsplit((binding.scheme, netloc, binding.path, binding.query, "")) + capability = replace(control.binding(backend.name), url=url) + client = await client_class(name).connect(capability) + try: + yield client, backend, flaky + finally: + await client.close() + + +@pytest.mark.parametrize("name", BACKENDS) +async def test_a_client_connects_at_its_binding_and_answers(name: str, tmp_path: Path) -> None: + async with connected(name, tmp_path) as (client, backend, _): + answered = await backend.probe(client) + + assert answered == backend.answer + + +@pytest.mark.parametrize("name", BACKENDS) +async def test_a_capability_published_at_serve_time_opens_through_the_control_port( + name: str, tmp_path: Path +) -> None: + async with BACKENDS[name](tmp_path) as backend, served(backend.env) as control: + opened = await control.open(backend.name) + answered = await backend.probe(opened) + again = await control.open(backend.name) + + assert isinstance(opened, client_class(name)) + assert again is opened + assert answered == backend.answer + + +@asynccontextmanager +async def nothing_listening( + scheme: str, protocol: str, path: str = "" +) -> AsyncIterator[Capability]: + with socket.socket() as released: + released.bind(("127.0.0.1", 0)) + port = released.getsockname()[1] + url = f"{scheme}://127.0.0.1:{port}{path}" + yield Capability.from_manifest({"name": "absent", "protocol": protocol, "url": url}) + + +@asynccontextmanager +async def hangs_up(scheme: str, protocol: str) -> AsyncIterator[Capability]: + """A server that closes every connection as soon as it accepts it.""" + server = await asyncio.start_server(lambda reader, writer: writer.close(), "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + async with server: + url = f"{scheme}://127.0.0.1:{port}" + yield Capability.from_manifest({"name": "absent", "protocol": protocol, "url": url}) + + +@asynccontextmanager +async def browser_without_pages() -> AsyncIterator[Capability]: + async with fake_browser(pages=[], can_create=False) as browser: + yield Capability.cdp(url=browser.url) + + +NO_TARGET: dict[ + str, tuple[str, Callable[[], AbstractAsyncContextManager[Capability]], type, str] +] = { + "ssh-nothing-listening": ( + "SSHClient", + lambda: nothing_listening("ssh", "ssh/2"), + ConnectionError, + "SSH connection failed during handshake", + ), + "ssh-handshake-dropped": ( + "SSHClient", + lambda: hangs_up("ssh", "ssh/2"), + ConnectionError, + "SSH connection failed during handshake", + ), + "cdp-nothing-listening": ( + "CDPClient", + lambda: nothing_listening("ws", "cdp/1.3"), + httpx.ConnectError, + "All connection attempts failed", + ), + "cdp-no-page-target": ( + "CDPClient", + browser_without_pages, + ValueError, + r"no CDP page target available at 127\.0\.0\.1:", + ), + "rfb-nothing-listening": ( + "RFBClient", + lambda: nothing_listening("rfb", "rfb/3.8"), + ConnectionRefusedError, + "Connect call failed", + ), + "mcp-nothing-listening": ( + "MCPClient", + lambda: nothing_listening("http", "mcp/2025-11-25", "/mcp"), + McpError, + "Transport error: ConnectError", + ), +} + + +@pytest.mark.parametrize( + ("name", "target", "error", "message"), NO_TARGET.values(), ids=NO_TARGET.keys() +) +async def test_a_client_with_nothing_to_connect_to_fails_clearly( + hud_env: HudEnv, + name: str, + target: Callable[[], AbstractAsyncContextManager[Capability]], + error: type[Exception], + message: str, +) -> None: + hud_env.set(HUD_CLIENT_TIMEOUT="1") + + async with target() as capability: + with pytest.raises(error, match=message): + await client_class(name).connect(capability) + + +RECONNECTING = ["SSHClient", "RFBClient", "MCPClient"] +NOT_RECONNECTING = { + "CDPClient": "holds one page socket and does not reopen it; test_cdp.py holds what a drop " + "does to its commands", +} + + +@pytest.mark.parametrize("name", RECONNECTING) +async def test_a_client_reconnects_after_its_connection_drops(name: str, tmp_path: Path) -> None: + async with connected(name, tmp_path) as (client, backend, flaky): + await backend.probe(client) + accepted = flaky.accepted + flaky.sever() + await eventually(lambda: backend.dropped(client)) + answered = await backend.probe(client) + + assert answered == backend.answer + assert flaky.accepted > accepted + + +async def cancel(backend: Backend, client: Any) -> None: + assert backend.held is not None + call = asyncio.create_task(backend.held(client, None)) + await eventually(backend.reached) + call.cancel() + with pytest.raises(asyncio.CancelledError): + await call + + +async def time_out(backend: Backend, client: Any) -> None: + assert backend.held is not None + with pytest.raises(TimeoutError): + await backend.held(client, 0.5) + await eventually(backend.reached) + + +ENDINGS = {"cancel": cancel, "timeout": time_out} +HOLDING = ["SSHClient", "CDPClient", "MCPClient"] +NOT_HOLDING = {"RFBClient": "a screenshot is one framebuffer read; there is no call to hold"} + + +@pytest.mark.parametrize("ending", ENDINGS) +@pytest.mark.parametrize("name", HOLDING) +async def test_a_cancelled_or_timed_out_call_ends_and_the_next_one_runs( + name: str, ending: str, tmp_path: Path +) -> None: + async with connected(name, tmp_path) as (client, backend, _): + await ENDINGS[ending](backend, client) + if backend.ended is not None: + await eventually(backend.ended) + answered = await backend.probe(client) + + assert answered == backend.answer + + +def test_every_exported_client_is_a_row_of_every_table() -> None: + exported = { + name + for name in hud.capabilities.__all__ + if name.endswith("Client") and name != "CapabilityClient" + } + rows = BACKENDS.keys() | ELSEWHERE.keys() + + assert rows == exported + assert {name for name, *_ in NO_TARGET.values()} | ELSEWHERE.keys() == exported + assert set(RECONNECTING) | NOT_RECONNECTING.keys() == BACKENDS.keys() + assert set(HOLDING) | NOT_HOLDING.keys() == BACKENDS.keys() diff --git a/tests/clients/test_cdp.py b/tests/clients/test_cdp.py new file mode 100644 index 000000000..11a11cc37 --- /dev/null +++ b/tests/clients/test_cdp.py @@ -0,0 +1,76 @@ +"""What only ``CDPClient`` does: page-target discovery, error replies, and a closed socket. + +The contract every client keeps is in ``test_clients.py``. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from hud.capabilities import Capability, CDPClient +from hud.capabilities.cdp import CDPError +from tests.support import fake_browser + +ENABLED = ["Page.enable", "Runtime.enable", "DOM.enable"] + + +@pytest.mark.parametrize( + ("pages", "target_id", "devtools_path", "http", "connected_to"), + [ + (["page-1", "page-2"], None, False, ["GET /json"], "page-1"), + ([], None, False, ["GET /json", "PUT /json/new?about:blank"], "new-1"), + (["page-1", "page-2"], "page-2", False, [], "page-2"), + (["page-1", "page-2"], None, True, [], "page-2"), + ], + ids=["first page", "a new page", "an explicit target", "a devtools url"], +) +async def test_connect_attaches_to_a_page_target_and_enables_its_domains( + pages: list[str], + target_id: str | None, + devtools_path: bool, + http: list[str], + connected_to: str, +) -> None: + async with fake_browser(pages=pages) as browser: + url = f"{browser.url}/devtools/page/page-2" if devtools_path else browser.url + client = await CDPClient.connect(Capability.cdp(url=url, target_id=target_id)) + await client.close() + + assert browser.http == http + assert [(command.target, command.method) for command in browser.commands] == [ + (connected_to, method) for method in ENABLED + ] + + +async def test_an_error_reply_raises_cdp_error_naming_the_failed_command() -> None: + error = {"code": -32000, "message": "Cannot navigate to invalid URL"} + async with fake_browser(replies={"Page.navigate": {"error": error}}) as browser: + client = await CDPClient.connect(Capability.cdp(url=browser.url)) + try: + with pytest.raises(CDPError) as raised: + await client.send("Page.navigate", {"url": "nope"}) + finally: + await client.close() + + assert (raised.value.code, raised.value.message, str(raised.value)) == ( + -32000, + "Cannot navigate to invalid URL", + "CDP 'Page.navigate' failed [-32000]: Cannot navigate to invalid URL", + ) + + +async def test_pending_commands_fail_when_the_browser_closes_the_socket() -> None: + replies = {"Runtime.evaluate": {"hold": True}, "Browser.crash": {"close": True}} + async with fake_browser(replies=replies) as browser: + client = await CDPClient.connect(Capability.cdp(url=browser.url)) + try: + waiting = asyncio.create_task(client.send("Runtime.evaluate", {"expression": "1"})) + await asyncio.sleep(0) + with pytest.raises(ConnectionError, match="CDP connection closed"): + await client.send("Browser.crash") + with pytest.raises(ConnectionError, match="CDP connection closed"): + await waiting + finally: + await client.close() diff --git a/tests/clients/test_connect.py b/tests/clients/test_connect.py new file mode 100644 index 000000000..7c1f75769 --- /dev/null +++ b/tests/clients/test_connect.py @@ -0,0 +1,309 @@ +"""The control channel's client side: ``hud.clients.connect`` and ``HudClient`` against a +scripted peer that answers each connection and records every frame it receives. + +Not covered: the 120 s control heartbeat. Its interval is a module constant with no +public knob, and waiting it out has no place in the default lane. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import math +from typing import Any +from urllib.parse import urlsplit + +import pytest + +from hud.capabilities import Connection +from hud.clients import HudClient, HudProtocolError, connect +from hud.environment import WorkspaceRoute +from hud.eval.runtime import Runtime +from tests.support import answer, control_peer, hang_up + +HELLO = {"session_id": "s-1", "env": {"name": "stub", "version": "1.0"}, "bindings": []} +BINDINGS = [ + {"name": "shell", "protocol": "ssh/2", "url": "ssh://agent@environment:22"}, + {"name": "screen-0", "protocol": "rfb/3.8", "url": "rfb://environment:5900"}, + {"name": "screen-1", "protocol": "rfb/3.8", "url": "rfb://environment:5901"}, + {"name": "custom", "protocol": "custom/1", "url": "tcp://environment:7000"}, +] + + +async def test_hello_names_the_session_and_the_env_it_reached() -> None: + async with control_peer([answer(HELLO)]) as peer, connect(Runtime(peer.url)) as client: + manifest = client.manifest + + assert peer.requests() == [{"jsonrpc": "2.0", "id": 1, "method": "hello", "params": {}}] + assert manifest is not None + assert (manifest.session_id, manifest.server_info.name, manifest.server_info.version) == ( + "s-1", + "stub", + "1.0", + ) + + +async def test_hello_carries_the_workspace_routes_and_connections_it_was_given() -> None: + routes = ( + WorkspaceRoute.from_url("ssh", "https://inference.hud.so/v1"), + WorkspaceRoute.from_url("shell", "http://gateway.test:8080"), + ) + connection = Connection( + name="inference", + capability="ssh", + url="https://inference.hud.so", + headers={"Authorization": "Bearer secret"}, + ) + + async with ( + control_peer([answer(HELLO)]) as peer, + connect(Runtime(peer.url), workspace_routes=routes, connections=[connection]), + ): + (hello,) = peer.requests("hello") + + assert hello["params"]["workspace_routes"] == [ + {"capability": "ssh", "host": "inference.hud.so", "port": 443}, + {"capability": "shell", "host": "gateway.test", "port": 8080}, + ] + assert hello["params"]["connections"] == [ + { + "name": "inference", + "capability": "ssh", + "url": "https://inference.hud.so", + "headers": {"Authorization": "Bearer secret"}, + } + ] + + +@pytest.mark.parametrize("immediately", [True, False], ids=["on-accept", "during-hello"]) +async def test_connect_retries_an_env_that_hangs_up_until_it_answers_hello( + immediately: bool, +) -> None: + not_ready = [hang_up(immediately=immediately)] + + async with ( + control_peer(not_ready, not_ready, [answer(HELLO)]) as peer, + connect(Runtime(peer.url), ready_timeout=10) as client, + ): + assert client.manifest is not None + + assert peer.accepted == 3 + + +async def test_connect_retries_a_port_that_refuses_until_the_env_starts_listening() -> None: + probe = await asyncio.start_server(lambda reader, writer: None, "127.0.0.1", 0) + port = probe.sockets[0].getsockname()[1] + probe.close() + await probe.wait_closed() + + async def hello(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + request = json.loads(await reader.readline()) + writer.write( + json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": HELLO}).encode() + b"\n" + ) + await writer.drain() + await reader.read() + + async def listen_later() -> asyncio.Server: + await asyncio.sleep(1) + return await asyncio.start_server(hello, "127.0.0.1", port) + + later = asyncio.create_task(listen_later()) + async with connect(Runtime(f"tcp://127.0.0.1:{port}"), ready_timeout=10) as client: + manifest = client.manifest + server = await later + server.close() + + assert manifest is not None and manifest.session_id == "s-1" + + +@pytest.mark.parametrize( + ("reply", "message"), + [ + ( + answer(error={"code": -32000, "message": "frame exceeds size limit"}, id=None), + "hud rpc error -32000: frame exceeds size limit", + ), + ( + answer(error={"code": -32601, "message": "unknown method"}), + "hud rpc error -32601: unknown method", + ), + (answer(HELLO, id=99), "'hello': reply id did not match request"), + (answer(["not", "an", "object"]), "'hello': result was not an object"), + ], + ids=["an-error-frame", "an-error-reply", "a-mismatched-id", "a-result-that-is-not-an-object"], +) +async def test_a_handshake_the_env_breaks_fails_without_retrying(reply: Any, message: str) -> None: + async with control_peer([reply]) as peer: + with pytest.raises(HudProtocolError, match=message): + async with connect(Runtime(peer.url), ready_timeout=10): + pass + + assert peer.accepted == 1 + + +async def test_an_env_that_never_answers_is_given_up_on_at_the_runtimes_ready_timeout() -> None: + loop = asyncio.get_running_loop() + + async with control_peer([hang_up()]) as peer: + started = loop.time() + with pytest.raises(EOFError, match="env closed connection during 'hello'"): + async with connect(Runtime(peer.url, params={"ready_timeout": 0.3}), ready_timeout=60): + pass + + assert loop.time() - started < 5 + + +async def test_a_refused_port_is_given_up_on_at_the_runtimes_ready_timeout() -> None: + server = await asyncio.start_server(lambda reader, writer: None, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + server.close() + await server.wait_closed() + loop = asyncio.get_running_loop() + + started = loop.time() + with pytest.raises(OSError): + runtime = Runtime(f"tcp://127.0.0.1:{port}", params={"ready_timeout": 0.3}) + async with connect(runtime, ready_timeout=60): + pass + + assert loop.time() - started < 5 + + +@pytest.mark.parametrize("ready_timeout", [-1.0, math.nan, math.inf]) +async def test_a_ready_timeout_that_is_not_a_positive_finite_number_is_refused( + ready_timeout: float, +) -> None: + runtime = Runtime("tcp://127.0.0.1:9", params={"ready_timeout": ready_timeout}) + + with pytest.raises(ValueError, match="must be a positive finite number"): + async with connect(runtime): + pass + + +@pytest.mark.parametrize( + ("ref", "scheme"), + [("shell", "ssh"), ("ssh", "ssh"), ("ssh/2", "ssh"), ("screen-1", "rfb")], + ids=["by-name", "by-protocol", "by-versioned-protocol", "by-name-among-same-protocol"], +) +async def test_a_binding_resolves_to_a_local_tunnel_by_name_or_protocol( + ref: str, scheme: str +) -> None: + async with ( + control_peer([answer({**HELLO, "bindings": BINDINGS})]) as peer, + connect(Runtime(peer.url)) as client, + ): + url = urlsplit(client.binding(ref).url) + + assert (url.scheme, url.hostname) == (scheme, "127.0.0.1") + + +@pytest.mark.parametrize( + ("ref", "message"), + [ + ("rfb", "ambiguous capability 'rfb'; matches: screen-0 (rfb/3.8), screen-1 (rfb/3.8)"), + ( + "browser", + "no capability 'browser' (available: shell (ssh/2), screen-0 (rfb/3.8), " + "screen-1 (rfb/3.8), custom (custom/1))", + ), + ], + ids=["an-ambiguous-protocol", "a-missing-capability"], +) +async def test_a_binding_that_does_not_name_one_capability_says_what_there_is( + ref: str, message: str +) -> None: + async with ( + control_peer([answer({**HELLO, "bindings": BINDINGS})]) as peer, + connect(Runtime(peer.url)) as client, + ): + with pytest.raises(KeyError) as raised: + client.binding(ref) + + assert raised.value.args == (message,) + + +async def test_a_client_on_raw_streams_hands_out_bindings_as_the_env_wrote_them() -> None: + async with control_peer([answer({**HELLO, "bindings": BINDINGS})]) as peer: + reader, writer = await asyncio.open_connection("127.0.0.1", peer.port) + client = HudClient(reader, writer) + try: + await client.hello() + url = client.binding("shell").url + finally: + await client.close() + + assert url == "ssh://agent@environment:22" + + +async def test_a_stream_the_env_refuses_ends_the_local_connection( + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.WARNING, logger="hud.clients") + + async with ( + control_peer([answer({**HELLO, "bindings": BINDINGS[:1]})]) as peer, + connect(Runtime(peer.url)) as client, + ): + local = urlsplit(client.binding("shell").url) + reader, writer = await asyncio.open_connection(local.hostname, local.port) + received = await asyncio.wait_for(reader.read(), 10) + writer.close() + + assert received == b"" + assert any( + "tunnel.open 'shell' refused:" in m and "unknown capability" in m for m in caplog.messages + ) + + +async def test_a_stream_to_an_env_that_stopped_accepting_ends_the_local_connection( + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level(logging.WARNING, logger="hud.clients") + + async with ( + control_peer([answer({**HELLO, "bindings": BINDINGS[:1]})]) as peer, + connect(Runtime(peer.url)) as client, + ): + local = urlsplit(client.binding("shell").url) + peer.stop_accepting() + reader, writer = await asyncio.open_connection(local.hostname, local.port) + received = await asyncio.wait_for(reader.read(), 10) + writer.close() + + assert received == b"" + assert any( + m.startswith("tunnel peer 127.0.0.1:") and "connection failed" in m for m in caplog.messages + ) + + +async def test_a_task_list_that_breaks_the_protocol_raises_and_the_session_goes_on() -> None: + async with ( + control_peer( + [answer(HELLO), answer({"tasks": "fix"}), answer({"cancelled": True})] + ) as peer, + connect(Runtime(peer.url)) as client, + ): + with pytest.raises(HudProtocolError, match=r"tasks\.list: 'tasks' must be a list"): + await client.list_tasks() + await client.cancel() + + assert [request["method"] for request in peer.requests()] == [ + "hello", + "tasks.list", + "tasks.cancel", + ] + + +async def test_a_reply_to_another_request_raises_and_ends_the_connection() -> None: + async with ( + control_peer([answer(HELLO), answer({"score": 1.0}, id=7)]) as peer, + connect(Runtime(peer.url)) as client, + ): + with pytest.raises(HudProtocolError, match=r"'tasks\.grade': reply id did not match"): + await client.grade({"answer": "done"}) + with pytest.raises((EOFError, OSError)): + await client.cancel() + + assert len(peer.requests()) == 2 diff --git a/tests/clients/test_mcp.py b/tests/clients/test_mcp.py new file mode 100644 index 000000000..e0c605a88 --- /dev/null +++ b/tests/clients/test_mcp.py @@ -0,0 +1,267 @@ +"""What only ``MCPClient`` does: its transports, auth, gateway faults, and off-schema output. + +Tool calls go to a FastMCP server over each HTTP transport, or to a scripted +streamable-HTTP server on the fake services, whose ``tools/call`` answers come +from a script: a transient gateway status, a body that is not JSON, a dropped +connection, a result outside the tool's output schema. Every call resolves: +with a result once a transient error clears, or with a transport error. The +contract every client keeps is in ``test_clients.py``. +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any, Literal + +import pytest +from fastmcp import FastMCP +from fastmcp.exceptions import ToolError +from mcp.shared.exceptions import McpError + +from hud.capabilities import Capability, MCPClient +from tests.support import Reply, mcp_server, serve_asgi + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + + from starlette.types import ASGIApp, Receive, Scope, Send + + from tests.support import FakeServices, HudEnv, Request + + +def tools() -> FastMCP: + server = FastMCP("tools") + + @server.tool + def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + @server.tool + def refuse() -> str: + """Always fail.""" + raise ToolError("not today") + + @server.tool + def numbers() -> list[int]: + """A plain list, wrapped as structured content.""" + return [1, 2] + + @server.tool + def mistyped() -> int: + """Returns something other than its annotation.""" + wrong: Any = "not an int" + return wrong + + return server + + +@pytest.mark.parametrize(("transport", "path"), [("streamable-http", "/mcp"), ("sse", "/sse")]) +async def test_tool_calls_return_canonical_results_over_each_transport( + transport: Literal["streamable-http", "sse"], path: str +) -> None: + app = tools().http_app(transport=transport, path=path) + async with serve_asgi(app) as port: + client = await MCPClient.connect( + Capability.mcp(url=f"http://127.0.0.1:{port}{path}", transport=transport) + ) + try: + added = await client.call_tool("add", {"a": 2, "b": 3}) + refused = await client.call_tool("refuse", {}) + finally: + await client.close() + + assert (added.isError, added.structuredContent) == (False, {"result": 5}) + assert [block.model_dump(include={"type", "text"}) for block in added.content] == [ + {"type": "text", "text": "5"} + ] + assert refused.isError is True + assert [block.model_dump(include={"type", "text"}) for block in refused.content] == [ + {"type": "text", "text": "not today"} + ] + + +def bearer_only(app: ASGIApp, token: str, seen: list[str]) -> ASGIApp: + """``app`` behind a gate that answers 401 to any HTTP request without ``token``.""" + + async def gated(scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] == "http": + authorization = dict(scope["headers"]).get(b"authorization", b"").decode() + seen.append(authorization) + if authorization != f"Bearer {token}": + await send({"type": "http.response.start", "status": 401, "headers": []}) + await send({"type": "http.response.body", "body": b""}) + return + await app(scope, receive, send) + + return gated + + +async def test_an_auth_token_rides_every_request_and_is_needed_where_the_server_asks() -> None: + seen: list[str] = [] + app = bearer_only(tools().http_app(path="/mcp"), "s3cret", seen) + + async with serve_asgi(app) as port: + url = f"http://127.0.0.1:{port}/mcp" + client = await MCPClient.connect(Capability.mcp(url=url, auth_token="s3cret")) + try: + added = await client.call_tool("add", {"a": 2, "b": 3}) + finally: + await client.close() + authorized = set(seen) + with pytest.raises(McpError, match="HTTPStatusError"): + await MCPClient.connect(Capability.mcp(url=url)) + + assert added.structuredContent == {"result": 5} + assert authorized == {"Bearer s3cret"} + + +OUTPUT_SCHEMA = { + "type": "object", + "properties": {"n": {"type": "integer"}}, + "required": ["n"], +} + + +def result(request: Request, body: dict[str, Any]) -> Reply: + return Reply(json={"jsonrpc": "2.0", "id": request.json["id"], "result": body}) + + +class ScriptedMCPServer: + """A streamable-HTTP MCP server whose ``tools/call`` answers come from a script.""" + + def __init__(self, services: FakeServices, calls: list[Callable[[Request], Reply]]) -> None: + self.calls = calls + self.url = f"{services.url('runtime')}/mcp" + services.route("runtime", "POST", "/mcp", handler=self.post) + services.route("runtime", "GET", "/mcp", Reply(status=405)) + services.route("runtime", "DELETE", "/mcp", Reply(json={})) + self.services = services + + def post(self, request: Request) -> Reply: + method = request.json.get("method") + if method == "initialize": + reply = result( + request, + { + "protocolVersion": request.json["params"]["protocolVersion"], + "capabilities": {"tools": {}}, + "serverInfo": {"name": "fake", "version": "1"}, + }, + ) + reply.headers["mcp-session-id"] = "session-1" + return reply + if "id" not in request.json: + return Reply(status=202, body=b"") + if method == "tools/list": + tool = {"name": "count", "inputSchema": {"type": "object"}} + return result(request, {"tools": [{**tool, "outputSchema": OUTPUT_SCHEMA}]}) + answer = self.calls.pop(0) if len(self.calls) > 1 else self.calls[0] + return answer(request) + + def attempts(self) -> int: + return sum( + request.json.get("method") == "tools/call" + for request in self.services.requests("runtime", "POST", "/mcp") + ) + + async def call(self) -> Any: + client = await MCPClient.connect(Capability.mcp(url=self.url, transport="streamable-http")) + try: + return await asyncio.wait_for(client.call_tool("count", {}), timeout=30) + finally: + await client.close() + + +def counted(request: Request) -> Reply: + return result( + request, {"content": [{"type": "text", "text": "3"}], "structuredContent": {"n": 3}} + ) + + +def status(code: int) -> Callable[[Request], Reply]: + return lambda request: Reply(status=code, json={}) + + +def off_schema(request: Request) -> Reply: + return result( + request, {"content": [{"type": "text", "text": "x"}], "structuredContent": {"n": "x"}} + ) + + +def unparseable(request: Request) -> Reply: + del request + return Reply(body=b"{not json", content_type="application/json") + + +def dropped(request: Request) -> Reply: + del request + + def chunks() -> Iterator[bytes]: + yield b'{"jsonrpc": "2.0", ' + raise ConnectionResetError("server went away") + + return Reply(stream=chunks(), content_type="application/json") + + +async def test_a_tool_call_succeeds_after_a_transient_503(services: FakeServices) -> None: + server = ScriptedMCPServer(services, [status(503), counted]) + + called = await server.call() + + assert (called.isError, called.structuredContent) == (False, {"n": 3}) + assert server.attempts() == 2 + + +@pytest.mark.parametrize( + ("calls", "error_type", "attempts"), + [ + pytest.param([status(503)], "HTTPStatusError", 3, id="503-until-the-deadline"), + pytest.param([unparseable], "ValidationError", 1, id="unparseable-json"), + pytest.param([dropped], "RemoteProtocolError", 1, id="connection-dropped"), + ], +) +async def test_a_failing_tool_call_resolves_with_a_transport_error( + services: FakeServices, + hud_env: HudEnv, + calls: list[Callable[[Request], Reply]], + error_type: str, + attempts: int, +) -> None: + hud_env.set(HUD_CLIENT_TIMEOUT="1") + server = ScriptedMCPServer(services, calls) + + with pytest.raises(McpError) as raised: + await server.call() + + assert raised.value.error.code == -32000 + assert raised.value.error.message == f"Transport error: {error_type}" + assert server.attempts() == attempts + + +async def test_a_result_outside_its_output_schema_reaches_the_caller( + services: FakeServices, +) -> None: + called = await ScriptedMCPServer(services, [off_schema]).call() + + assert (called.isError, called.structuredContent) == (False, {"n": "x"}) + + +@pytest.mark.parametrize( + ("tool", "structured"), + [ + pytest.param("numbers", {"result": [1, 2]}, id="plain-result-wrapped"), + pytest.param("mistyped", {"result": "not an int"}, id="mistyped-result-not-validated"), + ], +) +async def test_a_served_fastmcp_tool_wraps_plain_results_and_skips_output_validation( + tool: str, structured: dict[str, Any] +) -> None: + async with mcp_server(tools()) as served: + client = await MCPClient.connect(served) + try: + called = await client.call_tool(tool, {}) + finally: + await client.close() + + assert (called.isError, called.structuredContent) == (False, structured) diff --git a/tests/clients/test_rfb.py b/tests/clients/test_rfb.py new file mode 100644 index 000000000..60942d2ff --- /dev/null +++ b/tests/clients/test_rfb.py @@ -0,0 +1,46 @@ +"""What only ``RFBClient`` does: encoding the framebuffer as the caller asks. + +The contract every client keeps, reconnecting after a drop among it, is in +``test_clients.py``. +""" + +from __future__ import annotations + +import io + +import pytest +from PIL import Image + +from hud.capabilities import Capability, RFBClient +from hud.capabilities.rfb import ScreenshotMimeType, WebPScreenshotEncoding +from tests.support import fake_screen + +COLOR = (10, 200, 30) + + +@pytest.mark.parametrize( + ("encoding", "mime_type", "image_format"), + [ + ("image/png", "image/png", "PNG"), + ("image/webp", "image/webp", "WEBP"), + (WebPScreenshotEncoding(quality=100), "image/webp", "WEBP"), + ], +) +async def test_a_screenshot_encodes_the_framebuffer_as_requested( + encoding: ScreenshotMimeType | WebPScreenshotEncoding, mime_type: str, image_format: str +) -> None: + async with fake_screen(width=8, height=6, color=COLOR) as screen: + client = await RFBClient.connect(Capability.rfb(url=screen.url)) + try: + data, encoded_as = await client.screenshot_png(encoding) + finally: + await client.close() + + image = Image.open(io.BytesIO(data)) + assert (encoded_as, image.format, image.size) == (mime_type, image_format, (8, 6)) + colors = image.convert("RGB").getcolors() + assert colors is not None + ((count, pixel),) = colors + assert isinstance(pixel, tuple) + assert count == 48 + assert max(abs(channel - expected) for channel, expected in zip(pixel, COLOR, strict=True)) <= 4 diff --git a/tests/clients/test_ssh.py b/tests/clients/test_ssh.py new file mode 100644 index 000000000..9a7fe915f --- /dev/null +++ b/tests/clients/test_ssh.py @@ -0,0 +1,249 @@ +"""What only ``SSHClient`` does: file transfer, Windows shells, its reconnect policy. + +Every test serves a real ``env.workspace`` on loopback and connects to its +``ssh`` binding through :func:`tests.support.relay`, which can sever the +connection mid-command, refuse reconnects, or stall a handshake. The contract +every client keeps (connecting, a drop, a cancelled or timed-out command) is in +``test_clients.py``. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +from contextlib import asynccontextmanager +from dataclasses import replace +from typing import TYPE_CHECKING +from urllib.parse import urlsplit + +import pytest + +from hud import Environment +from hud.capabilities import Connection, SSHClient +from hud.clients import connect +from hud.eval.runtime import Runtime +from tests.fixtures.environments import served +from tests.support import FlakyRelay, answer, control_peer, eventually, relay + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + from pathlib import Path + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="the workspace shell is POSIX") + +POWERSHELL = """\ +#!{python} +import base64, pathlib, re, sys + +script = base64.b64decode(sys.argv[-1]).decode("utf-16-le") +with open({log!r}, "a") as log: + log.write(script + "\\n") +literals = [text.replace("''", "'") for text in re.findall(r"'((?:[^']|'')*)'", script)] +path = literals[-1] +if "ReadAllBytes" in script: + print(base64.b64encode(pathlib.Path(path).read_bytes()).decode()) +elif "Get-ChildItem" in script: + print("\\n".join(sorted(entry.name for entry in pathlib.Path(path).iterdir()))) +elif "WriteAllBytes" in script: + pathlib.Path(path).write_bytes(b"") +else: + with open(path, "ab") as file: + file.write(base64.b64decode(literals[0])) +""" + + +@asynccontextmanager +async def shell( + tmp_path: Path, *, env: dict[str, str] | None = None +) -> AsyncIterator[tuple[SSHClient, FlakyRelay, Path]]: + """A client connected to a served workspace's ``ssh`` binding through a relay.""" + root = tmp_path / "workspace" + root.mkdir() + environment = Environment("ssh-faults") + environment.workspace(root, track_files=False, env=env) + async with served(environment) as control: + binding = control.binding("shell") + parts = urlsplit(binding.url) + assert parts.hostname is not None and parts.port is not None + async with relay(parts.hostname, parts.port) as flaky: + capability = replace(binding, url=f"ssh://{parts.username}@127.0.0.1:{flaky.port}") + client = await SSHClient.connect(capability) + try: + yield client, flaky, root + finally: + await client.close() + + +def written(path: Path) -> bool: + return path.is_file() and bool(path.read_text().strip()) + + +async def drop(client: SSHClient, flaky: FlakyRelay) -> None: + """Sever the relayed connection and wait until the client has seen it close.""" + flaky.sever() + await eventually(client.conn.is_closed) + + +async def test_files_round_trip_through_the_exec_channel(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, _, root): + await client.write_text("empty.txt", "") + await client.write_text("it's here.txt", "héllo\nwörld\n") + (root / "raw.bin").write_bytes(b"\x00\xff\x10") + + assert (root / "empty.txt").read_text() == "" + assert await client.read_text("empty.txt") == "" + assert await client.read_text(str(root / "it's here.txt")) == "héllo\nwörld\n" + assert await client.read_bytes("raw.bin") == b"\x00\xff\x10" + assert await client.listdir(".") == ["empty.txt", "it's here.txt", "raw.bin"] + + +async def test_windows_shells_move_files_as_base64_through_encoded_powershell( + tmp_path: Path, +) -> None: + tools = tmp_path / "tools" + tools.mkdir() + log = tmp_path / "powershell.log" + stub = tools / "powershell" + stub.write_text(POWERSHELL.format(python=sys.executable, log=str(log)), encoding="utf-8") + stub.chmod(0o755) + async with shell(tmp_path, env={"PATH": f"{tools}{os.pathsep}{os.environ['PATH']}"}) as ( + client, + _, + root, + ): + windows_client = SSHClient( + replace(client.capability, params={**client.capability.params, "shell": "powershell"}), + client.conn, + ) + content = "x" * 7000 + "é" + target = str(root / "it's.txt") + + await windows_client.write_text(target, content) + read = await windows_client.read_text(target) + listing = await windows_client.listdir(str(root)) + + assert (read, listing) == (content, ["it's.txt"]) + + +async def test_a_command_lost_in_flight_is_not_replayed_and_the_next_one_reconnects( + tmp_path: Path, +) -> None: + async with shell(tmp_path) as (client, flaky, root): + command = asyncio.create_task(client.run("echo ran >> marker; exec sleep 30")) + await eventually(lambda: written(root / "marker")) + + flaky.sever() + with pytest.raises(ConnectionError, match="SSH command ended without an exit status"): + await command + result = await client.run("cat marker", timeout=10) + process = await client.create_process("echo opened") + opened = await process.wait() + + assert result.stdout == "ran\n" + assert opened.stdout == b"opened\n" + assert flaky.accepted == 2 + + +async def test_reconnecting_gives_up_after_three_refused_attempts(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, flaky, _): + flaky.refuse(3) + await drop(client, flaky) + + with pytest.raises(ConnectionError, match="SSH reconnect failed after 3 attempts"): + await client.run("true") + + assert flaky.accepted == 4 + + +async def test_a_command_timeout_covers_a_stalled_reconnect(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, flaky, _): + flaky.stall() + await drop(client, flaky) + + with pytest.raises(TimeoutError): + await client.run("true", timeout=0.5) + flaky.release() + result = await client.run("echo back", timeout=10) + + assert result.stdout == "back\n" + + +async def test_closing_during_a_reconnect_discards_the_new_connection(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, flaky, _): + flaky.stall() + await drop(client, flaky) + command = asyncio.create_task(client.run("true")) + await flaky.wait_until(stalled=1) + + closing = asyncio.create_task(client.close()) + await asyncio.sleep(0) + flaky.release() + with pytest.raises(ConnectionError, match="SSH client is closed"): + await command + await closing + + await flaky.wait_until(accepted=2) + await eventually(lambda: flaky.open == 0) + + +@pytest.mark.parametrize( + ("advertised", "connection_capability", "message"), + [ + (False, "shell", "SSH capability does not support process-bound connections"), + (True, "browser", "connections do not belong to SSH capability 'shell': inference"), + ], +) +async def test_process_bound_connections_must_be_advertised_and_belong_to_the_shell( + advertised: bool, connection_capability: str, message: str, tmp_path: Path +) -> None: + connection = Connection( + name="inference", + capability=connection_capability, + url="https://inference.example", + headers={"Authorization": "Bearer secret"}, + ) + async with shell(tmp_path) as (client, _, _): + bound = SSHClient( + replace( + client.capability, + params={**client.capability.params, "process_connections": advertised}, + ), + client.conn, + ) + + with pytest.raises(ValueError, match=message): + await bound.create_process("true", connections=(connection,)) + + +async def test_an_opened_client_retries_an_upstream_that_refuses_and_closes_with_the_connection( + tmp_path: Path, +) -> None: + root = tmp_path / "workspace" + root.mkdir() + env = Environment("tunneled") + env.workspace(root, track_files=False) + async with served(env) as workspace: + shell_binding = workspace.binding("shell") + address = urlsplit(shell_binding.url) + assert address.hostname is not None and address.port is not None + async with relay(address.hostname, address.port) as flaky: + flaky.refuse(2) + binding = {**shell_binding.to_manifest(), "url": "ssh://agent@environment:22"} + hello = {"session_id": "s-1", "env": {"name": "stub", "version": "1.0"}} + async with control_peer( + [answer({**hello, "bindings": [binding]})], + upstreams={"shell": ("127.0.0.1", flaky.port)}, + ) as peer: + async with connect(Runtime(peer.url)) as client: + opened = await client.open("shell") + assert isinstance(opened, SSHClient) + result = await opened.run("echo through the tunnel", timeout=10) + again = await client.open("ssh") + + assert again is opened + assert opened.conn.is_closed() + await eventually(lambda: flaky.open == 0) + + assert result.stdout == "through the tunnel\n" + assert (flaky.accepted, peer.tunnels) == (3, 3) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..e95e7a570 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,243 @@ +"""Fixtures wiring the test support package (``tests/support``) into every test. + +Every test runs against an isolated SDK configuration: a temporary home, no +developer credentials, telemetry uploads off, and every HUD service URL pointed +at a closed local port. A test that talks to HUD services asks for the +``services`` fixture, which points the SDK at the fake backend instead. + +Tests marked ``e2e`` need something outside the process (a Docker daemon, a +Linux sandbox, a real HUD account) and only run when selected with ``-m``; +they skip themselves when what they need is missing. +""" + +from __future__ import annotations + +import importlib.util +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest +from dotenv import dotenv_values + +from hud.telemetry import flush +from tests.support import FakeServices, Hud, HudEnv, Models, Platform, Uv +from tests.support import platform as platform_module + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "tests" / "fixtures" +# The fixture environments that need nothing but the SDK, so a container can serve them. +IMAGE_FIXTURES = ("echo", "faulty", "sessions", "broken", "stalls") + +# The developer's or CI's own environment, captured before any test isolates it. +# Only ``live`` tests read credentials from it. +REAL_ENVIRONMENT = dict(os.environ) + +SERVICE_URL_VARIABLES = ( + "HUD_API_URL", + "HUD_TELEMETRY_URL", + "HUD_GATEWAY_URL", + "HUD_RUNTIME_URL", + "HUD_RL_URL", + "HUD_WEB_URL", +) + + +def real_hud_api_key() -> str | None: + """``HUD_API_KEY`` from the real environment or the real ``~/.hud/.env``.""" + if key := REAL_ENVIRONMENT.get("HUD_API_KEY"): + return key + home = REAL_ENVIRONMENT.get("HOME") + if home is None: + return None + return dotenv_values(Path(home) / ".hud" / ".env").get("HUD_API_KEY") + + +@pytest.fixture(autouse=True) +def hud_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> HudEnv: + """The SDK's configuration for this test; change it with ``hud_env.set(...)``.""" + return HudEnv(monkeypatch, tmp_path) + + +@pytest.fixture(scope="session") +def services_server() -> Iterator[FakeServices]: + server = FakeServices() + server.start() + yield server + server.stop() + + +@pytest.fixture +def services(services_server: FakeServices, hud_env: HudEnv) -> FakeServices: + """The fake HUD backend, with the SDK configured to reach it and no routes declared.""" + services_server.reset() + hud_env.set(**services_server.env()) + return services_server + + +@pytest.fixture +def platform(services: FakeServices, hud_env: HudEnv) -> Platform: + """The fake HUD platform, signed in with a HUD key.""" + hud_env.set(HUD_API_KEY=platform_module.API_KEY, HUD_WEB_URL=platform_module.WEB_URL) + return Platform(services) + + +@pytest.fixture +def reporting(services: FakeServices, hud_env: HudEnv) -> Iterator[FakeServices]: + """The HUD services accepting job and trace reports and span uploads, with telemetry on + and a key. + + Span uploads drain before the test ends, so none reaches a later test's services. + """ + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + for path in ("/v2/trace/job/{id}/enter", "/v2/trace/{id}/enter", "/v2/trace/{id}/exit"): + services.route("api", "POST", path, json={}) + services.route("telemetry", "POST", "/trace/{id}/telemetry-upload", json={}) + yield services + assert flush(timeout=10) + + +@pytest.fixture +def models(services: FakeServices) -> Models: + """Scripted model providers behind the fake gateway.""" + return Models(services) + + +@pytest.fixture +def hud(tmp_path: Path) -> Hud: + """The real ``hud`` CLI, run in a subprocess from ``tmp_path / "work"``.""" + return Hud(tmp_path / "work") + + +@pytest.fixture(scope="session") +def uv() -> Uv: + return Uv(REAL_ENVIRONMENT) + + +@pytest.fixture(scope="session") +def sdk_wheel(uv: Uv, tmp_path_factory: pytest.TempPathFactory) -> Path: + """This checkout's SDK as a wheel, built once per test worker.""" + out = tmp_path_factory.mktemp("sdk-wheel") + uv("build", "--wheel", "--no-create-gitignore", "--out-dir", str(out), str(ROOT), cwd=out) + (wheel,) = out.glob("hud-*.whl") + return wheel + + +@pytest.fixture(scope="session") +def fixture_image( + sdk_wheel: Path, tmp_path_factory: pytest.TempPathFactory +) -> Callable[[str], str]: + """The image serving fixture environment ``name``, built on first use; returns its tag.""" + context = tmp_path_factory.mktemp("fixture-image") + shutil.copy(sdk_wheel, context) + shutil.copy(FIXTURES / "Dockerfile", context) + (context / "envs").mkdir() + for name in IMAGE_FIXTURES: + shutil.copy(FIXTURES / "envs" / f"{name}.py", context / "envs") + built: dict[str, str] = {} + + def image(name: str) -> str: + if name not in built: + tag = f"hud-test-fixture:{name}" + subprocess.run( + ["docker", "build", "--quiet", "--build-arg", f"FIXTURE={name}", "--tag", tag, "."], + cwd=context, + env=REAL_ENVIRONMENT, + capture_output=True, + check=True, + ) + built[name] = tag + return built[name] + + return image + + +@pytest.fixture(scope="session") +def modal_image(sdk_wheel: Path) -> Callable[[str], Any]: + """A Modal image serving fixture environment ``name`` as ``/hud/env.py``, from this + checkout's SDK; Modal builds it on first use and caches it after.""" + import modal + + def image(name: str) -> Any: + return ( + modal.Image.debian_slim(python_version="3.12") + .add_local_file(sdk_wheel, f"/tmp/{sdk_wheel.name}", copy=True) + .run_commands(f"pip install /tmp/{sdk_wheel.name}") + .add_local_file(FIXTURES / "envs" / f"{name}.py", "/hud/env.py", copy=True) + ) + + return image + + +@pytest.fixture +def modal_account(hud_env: HudEnv) -> HudEnv: + """Let Modal find the real account the developer or CI configured.""" + home = REAL_ENVIRONMENT.get("HOME", "") + hud_env.set( + MODAL_CONFIG_PATH=str(Path(home) / ".modal.toml"), + MODAL_TOKEN_ID=REAL_ENVIRONMENT.get("MODAL_TOKEN_ID"), + MODAL_TOKEN_SECRET=REAL_ENVIRONMENT.get("MODAL_TOKEN_SECRET"), + ) + return hud_env + + +@pytest.fixture +def live(hud_env: HudEnv) -> HudEnv: + """Configure the SDK for a real HUD account: the real key and the real services.""" + key = real_hud_api_key() + if key is None: + pytest.skip("needs HUD_API_KEY") + hud_env.set( + HUD_API_KEY=key, + **{name: REAL_ENVIRONMENT.get(name) for name in SERVICE_URL_VARIABLES}, + ) + return hud_env + + +def _docker_available() -> bool: + if shutil.which("docker") is None: + return False + try: + probe = subprocess.run(["docker", "info"], capture_output=True, check=False, timeout=20) + except subprocess.TimeoutExpired: + return False + return probe.returncode == 0 + + +def _modal_available() -> bool: + home = REAL_ENVIRONMENT.get("HOME", "") + return "MODAL_TOKEN_ID" in REAL_ENVIRONMENT or (Path(home) / ".modal.toml").is_file() + + +def _sandbox_available() -> bool: + return sys.platform == "linux" and os.geteuid() == 0 and shutil.which("bwrap") is not None + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Skip selected ``e2e`` tests whose daemon, sandbox, or credentials are missing.""" + del config + checks = { + "docker": (_docker_available, "needs a running Docker daemon"), + "sandbox": (_sandbox_available, "needs Linux, root, and bubblewrap"), + "live": (lambda: real_hud_api_key() is not None, "needs HUD_API_KEY"), + "modal": (_modal_available, "needs a Modal account"), + "lerobot": ( + lambda: importlib.util.find_spec("lerobot") is not None, + "needs lerobot[dataset]", + ), + } + available: dict[str, bool] = {} + for item in items: + for mark, (check, reason) in checks.items(): + if mark not in item.keywords: + continue + if mark not in available: + available[mark] = check() + if not available[mark]: + item.add_marker(pytest.mark.skip(reason=reason)) diff --git a/hud/agents/openai/tools/tests/__init__.py b/tests/e2e/__init__.py similarity index 100% rename from hud/agents/openai/tools/tests/__init__.py rename to tests/e2e/__init__.py diff --git a/tests/e2e/lifecycle.py b/tests/e2e/lifecycle.py new file mode 100644 index 000000000..f0b3e2164 --- /dev/null +++ b/tests/e2e/lifecycle.py @@ -0,0 +1,50 @@ +"""Capped rollouts: the scripted answers they get and what a clean lifecycle leaves in the spans.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +from tests.support import say + +if TYPE_CHECKING: + from tests.support import ModelRequest, Turn + from tests.support.machine import Result + +# Tool steps may fail without failing the run: a real model can call a tool badly. +LIFECYCLE_SOURCES = {"task", "agent", "system"} + + +def task_calls(steps: list[dict[str, Any]]) -> list[dict[str, Any]]: + """The ``task_call`` of every task step, in order.""" + return [step["task_call"] for step in steps if step["source"] == "task"] + + +def assert_clean_lifecycle(steps: list[dict[str, Any]]) -> tuple[dict[str, Any], dict[str, Any]]: + """Assert the run set up its task, graded it, and errored nowhere; return both results.""" + errors = [ + step["error"] for step in steps if step["source"] in LIFECYCLE_SOURCES and step.get("error") + ] + assert not errors, errors + calls = task_calls(steps) + assert [call["phase"] for call in calls] == ["setup", "evaluate"], calls + setup, grade = (call["result"] for call in calls) + assert setup.get("prompt"), f"task setup returned no prompt: {setup}" + assert not grade.get("isError"), grade + assert isinstance(grade["score"], int | float), grade + assert 0 <= grade["score"] <= 1, grade + return setup, grade + + +def scripted(request: ModelRequest) -> Turn: + """The agent answers at once; the LLM judge finds every criterion met.""" + if request.protocol == "chat": + return say(json.dumps({"criterion_status": "MET", "explanation": "scripted judge"})) + return say("done") + + +def evaluated(result: Result) -> dict[str, Any]: + assert result.exit_code == 0, result + assert result.json["error_count"] == 0, result + (run,) = result.json["runs"] + return run diff --git a/tests/e2e/projects.py b/tests/e2e/projects.py new file mode 100644 index 000000000..6605e6623 --- /dev/null +++ b/tests/e2e/projects.py @@ -0,0 +1,130 @@ +"""Generated environment projects: ``hud init`` output with the SDK from this checkout installed.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from tests.support import Hud, Uv + +from .lifecycle import assert_clean_lifecycle + +ROOT = Path(__file__).resolve().parents[2] +TEMPLATES = sorted( + path.name for path in (ROOT / "environments").iterdir() if (path / "env.py").is_file() +) +ROWS = "ci-tasks.json" +CHECKS = ( + ("-m", "ruff", "format", ".", "--check"), + ("-m", "ruff", "check", "."), + ("-m", "pytest", "-q", "-p", "no:cacheprovider"), +) +EXPORT_TASKS = ( + "from hud.eval import Taskset; Taskset.from_file('tasks.py').to_file('.hud/tasks.json')" +) + + +def system_python() -> str: + """A CPython under /usr: inside the workspace sandbox, only /usr is visible to the grader.""" + for version in ("3.12", "3.11"): + if found := shutil.which(f"python{version}", path="/usr/local/bin:/usr/bin"): + return found + raise RuntimeError("needs Python 3.11 or 3.12 under /usr/local/bin or /usr/bin") + + +@dataclass(frozen=True) +class Project: + """A generated project and its own ``hud``, run from the project's environment.""" + + root: Path + python: Path + + @property + def hud(self) -> Hud: + return Hud(self.root, python=self.python) + + def run(self, *args: str, timeout: float = 600) -> subprocess.CompletedProcess[str]: + """Run the project's Python with ``args``, the way ``uv run python`` would.""" + return subprocess.run( + [str(self.python), *args], + cwd=self.root, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + def check(self) -> None: + """The template's own lint and tests, as a user runs them before deploying.""" + for args in CHECKS: + completed = self.run(*args) + assert completed.returncode == 0, ( + f"{' '.join(args)}\n{completed.stdout}\n{completed.stderr}" + ) + + def bundle_sdk_into_image(self) -> None: + """Copy the bundled SDK into the image ahead of its ``uv sync``, for a deploy to build.""" + dockerfile = self.root / "Dockerfile.hud" + lines = dockerfile.read_text().splitlines(keepends=True) + sync = next( + index + for index, line in enumerate(lines) + if line.startswith("RUN") and "uv sync" in line + ) + lines.insert(sync, "COPY .hud-sdk .hud-sdk\n") + dockerfile.write_text("".join(lines)) + with (self.root / ".dockerignore").open("a") as ignored: + ignored.write("!.hud-sdk/\n!.hud-sdk/**\n") + + def write_rows(self, data_file: str | None) -> Path: + """Write the first task as a JSON row beside the env source, attaching ``data_file``.""" + exported = self.run("-c", EXPORT_TASKS) + assert exported.returncode == 0, exported.stderr + row = json.loads((self.root / ".hud" / "tasks.json").read_text())[0] + if data_file is not None: + row["args"]["attachments"] = [{"file_id": data_file, "path": "notes.txt"}] + row["args"]["prompt"] = "Read files/notes.txt first. " + row["args"]["prompt"] + path = self.root / ROWS + path.write_text(json.dumps([row], indent=2)) + return path + + +def generate( + hud: Hud, + template: str, + directory: Path, + *, + wheel: Path, + uv: Uv, + environment: Path | None = None, +) -> Project: + """``hud init`` a project from ``template``, then install this checkout's SDK into it. + + The wheel is copied into the project, as a deploy needs it inside the build + context. ``environment`` installs the project's Python somewhere other than + ``.venv``, the way an image does. + """ + created = hud("init", directory.name, "--dir", str(directory.parent), "--template", template) + assert created.exit_code == 0, created + bundled = directory / ".hud-sdk" / wheel.name + bundled.parent.mkdir() + shutil.copy(wheel, bundled) + uv("add", "--no-sync", f".hud-sdk/{wheel.name}", cwd=directory) + placement = {} + if environment is not None: + placement = {"UV_PROJECT_ENVIRONMENT": str(environment), "UV_PYTHON": system_python()} + uv("sync", "--all-extras", cwd=directory, env=placement) + python = (environment or directory / ".venv") / "bin" / "python" + return Project(directory, python) + + +def assert_lifecycle(recorded: list[dict[str, Any]], data_file: str | None) -> dict[str, Any]: + """A clean single-run lifecycle; a declared data file reaches the start frame.""" + setup, grade = assert_clean_lifecycle(recorded) + if data_file is not None: + assert setup["data_files"] == [{"path": "files/notes.txt", "file_id": data_file}] + return grade diff --git a/tests/e2e/test_cookbooks.py b/tests/e2e/test_cookbooks.py new file mode 100644 index 000000000..cdc96f40e --- /dev/null +++ b/tests/e2e/test_cookbooks.py @@ -0,0 +1,123 @@ +"""The cookbooks against this checkout's SDK: every module imports, and the local runs complete. + +Cookbooks live outside the package and its tests, so an SDK change can break +them silently: a moved import path, a renamed export. Every ``hud`` name a +cookbook imports must exist; every cookbook module must import, unless a +third-party package it needs is not installed; and the cookbooks whose README +runs them with no outside account complete one capped rollout with a scripted +model. Nightly, through the ``e2e`` marker. +""" + +from __future__ import annotations + +import ast +import importlib +import importlib.util +import subprocess +import sys +from typing import TYPE_CHECKING + +import pytest + +from tests.support import steps + +from .lifecycle import assert_clean_lifecycle, evaluated, scripted +from .projects import ROOT + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import Hud, HudEnv, Models + +COOKBOOKS = ROOT / "cookbooks" +MODULES = sorted(path for path in COOKBOOKS.rglob("*.py") if ".venv" not in path.parts) +# The README commands that need no outside account: (cookbook, task source). +LOCAL_RUNS = [("connect4-selfplay", "env.py"), ("daytona-rl", "tasks.py")] + +pytestmark = pytest.mark.e2e + + +def module_id(path: Path) -> str: + return str(path.relative_to(COOKBOOKS)) + + +def cookbook_of(path: Path) -> Path: + return COOKBOOKS / path.relative_to(COOKBOOKS).parts[0] + + +def search_path(path: Path) -> list[Path]: + """Where a cookbook's scripts find its sibling modules: their directory and the cookbook's.""" + return list(dict.fromkeys([path.parent, cookbook_of(path)])) + + +def imports(path: Path) -> list[tuple[str, str | None]]: + """``(module, name)`` for every import in ``path``; ``name`` is None for ``import module``.""" + found: list[tuple[str, str | None]] = [] + for node in ast.walk(ast.parse(path.read_text("utf-8"))): + if isinstance(node, ast.Import): + found.extend((alias.name, None) for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + found.extend((node.module, alias.name) for alias in node.names) + return found + + +def resolves(module: str, name: str | None) -> bool: + """``module`` imports and has ``name``, as an attribute or a submodule.""" + imported = importlib.import_module(module) + if name is None or hasattr(imported, name): + return True + return ( + hasattr(imported, "__path__") and importlib.util.find_spec(f"{module}.{name}") is not None + ) + + +@pytest.mark.parametrize("path", MODULES, ids=module_id) +def test_every_hud_name_a_cookbook_imports_exists(path: Path) -> None: + hud_imports = [ + (module, name) for module, name in imports(path) if module.split(".")[0] == "hud" + ] + + missing = [f"{module}:{name}" for module, name in hud_imports if not resolves(module, name)] + + assert missing == [] + + +@pytest.mark.parametrize("path", MODULES, ids=module_id) +def test_a_cookbook_module_imports(path: Path) -> None: + local = {sibling.stem for directory in search_path(path) for sibling in directory.glob("*.py")} + third_party = { + module.split(".")[0] + for module, _ in imports(path) + if module.split(".")[0] not in {*sys.stdlib_module_names, "hud", "__future__", *local} + } + if missing := sorted(name for name in third_party if importlib.util.find_spec(name) is None): + pytest.skip(f"needs {', '.join(missing)}") + program = ( + f"import sys; sys.path[:0] = {[str(p) for p in search_path(path)]!r}; import {path.stem}" + ) + + imported = subprocess.run( + [sys.executable, "-c", program], + cwd=cookbook_of(path), + capture_output=True, + text=True, + timeout=120, + check=False, + ) + + assert imported.returncode == 0, imported.stderr + + +@pytest.mark.parametrize(("cookbook", "source"), LOCAL_RUNS, ids=[run[0] for run in LOCAL_RUNS]) +def test_a_cookbook_completes_its_local_run( + cookbook: str, source: str, models: Models, hud_env: HudEnv, hud: Hud +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(scripted) + + result = hud( + "eval", source, "claude", "--max-steps", "3", "-y", "--json", cwd=COOKBOOKS / cookbook + ) + + run = evaluated(result) + assert_clean_lifecycle(steps(run["trace_id"])) diff --git a/tests/e2e/test_templates.py b/tests/e2e/test_templates.py new file mode 100644 index 000000000..a73d720ca --- /dev/null +++ b/tests/e2e/test_templates.py @@ -0,0 +1,185 @@ +"""Every ``hud init`` template, from ``hud init`` to a graded rollout, the way a user takes it. + +Each case generates a project from a template, installs the SDK from this +checkout into it, lists its tasks, passes the template's own lint and tests, +and runs one capped rollout of its first task with ``hud eval``. Three lanes run +that lifecycle: + +- a scripted model behind the fake gateway, on every PR, or a cheap real model + through the HUD gateway in the ``live`` lane: the model is a row; +- ``hosted``: deployed to HUD and run on its runtime, with the trace read back + from the platform. The pre-release workflow runs it one template per job. + +A template that serves an isolated workspace (coding) needs the sandbox lane +locally; its environment is installed under /usr, where its image puts it. +""" + +from __future__ import annotations + +import asyncio +import json +import shutil +import time +import uuid +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from tests.conftest import REAL_ENVIRONMENT +from tests.support import Reply, fake_screen, steps + +from .lifecycle import evaluated, scripted +from .projects import ROWS, TEMPLATES, assert_lifecycle, generate + +if TYPE_CHECKING: + from collections.abc import Iterator + + from tests.support import FakeServices, Hud, HudEnv, Uv + from tests.support.machine import Result + +SANDBOXED = {"coding"} +TAKES_DATA_FILES = {"argument-hints"} +EVAL = ("eval", ROWS, "claude", "--max-steps", "10", "-y", "--json") +LIVE_MODEL = "claude-haiku-4-5" +HOSTED_MODEL = "claude-sonnet-4-6" +REAL_MODEL = ("--gateway", "--config", "max_tokens=2048") + + +def scripted_model( + request: pytest.FixtureRequest, template: str +) -> tuple[str | None, tuple[str, ...]]: + """A scripted model behind the fake gateway, and a data file the fake platform serves.""" + services: FakeServices = request.getfixturevalue("services") + request.getfixturevalue("models").respond(scripted) + request.getfixturevalue("hud_env").set(HUD_API_KEY="k") + services.route("api", "GET", "/v2/data/ci-notes", json={"filename": "notes.txt"}) + services.route( + "api", + "GET", + "/v2/data/ci-notes/download", + json={"url": f"{services.url('api')}/files/notes"}, + ) + services.route("api", "GET", "/files/notes", Reply(body=b"2, 3 and 5 are prime.")) + return ("ci-notes" if template in TAKES_DATA_FILES else None), () + + +def real_model(request: pytest.FixtureRequest, template: str) -> tuple[str | None, tuple[str, ...]]: + """A cheap real model through the HUD gateway, and the shared CI attachment.""" + request.getfixturevalue("live") + data_file = ( + REAL_ENVIRONMENT.get("HUD_CI_DATA_FILE_ID") if template in TAKES_DATA_FILES else None + ) + return data_file, (*REAL_MODEL, "--model", LIVE_MODEL) + + +MODELS = {"scripted": scripted_model, "live": real_model} +MODEL_LANES = {"scripted": [], "live": [pytest.mark.e2e, pytest.mark.live]} + + +def lane() -> list[Any]: + """One case per template and model; templates serving an isolated workspace join the + sandbox lane, and the real model runs in the live lane.""" + return [ + pytest.param( + name, + model, + id=f"{name}-{model}", + marks=[ + *marks, + *([pytest.mark.e2e, pytest.mark.sandbox] if name in SANDBOXED else []), + ], + ) + for model, marks in MODEL_LANES.items() + for name in TEMPLATES + ] + + +@pytest.fixture +def environment(request: pytest.FixtureRequest) -> Iterator[Path | None]: + """Where the project's Python goes: under /usr for a sandboxed template, else ``.venv``.""" + if request.node.callspec.params["template"] not in SANDBOXED: + yield None + return + location = Path("/usr/local/share/hud-e2e") / uuid.uuid4().hex + yield location + shutil.rmtree(location, ignore_errors=True) + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize(("template", "model"), lane()) +async def test_a_template_runs_from_init_to_a_graded_rollout( + template: str, + model: str, + environment: Path | None, + hud_env: HudEnv, + hud: Hud, + uv: Uv, + sdk_wheel: Path, + tmp_path: Path, + request: pytest.FixtureRequest, +) -> None: + data_file, model_args = MODELS[model](request, template) + + project = generate( + hud, template, tmp_path / f"ci-{template}", wheel=sdk_wheel, uv=uv, environment=environment + ) + listed = project.hud("task", "list", "--json") + assert listed.exit_code == 0, listed + assert listed.json, "the template lists no tasks" + if (project.root / "tests").is_dir(): + project.check() + project.write_rows(data_file) + async with fake_screen() as screen: + hud_env.set(VNC_PORT=str(screen.port)) + run = evaluated(await asyncio.to_thread(project.hud, *EVAL, *model_args, timeout=900)) + + assert_lifecycle(steps(run["trace_id"]), data_file) + + +@pytest.mark.e2e +@pytest.mark.live +@pytest.mark.hosted +@pytest.mark.timeout(2400) +@pytest.mark.parametrize("template", TEMPLATES) +def test_a_deployed_template_runs_on_hud_and_its_trace_reads_back( + template: str, live: HudEnv, hud: Hud, uv: Uv, sdk_wheel: Path, tmp_path: Path +) -> None: + data_file = None + if template in TAKES_DATA_FILES: + data_file = REAL_ENVIRONMENT.get("HUD_CI_DATA_FILE_ID") + assert data_file, "HUD_CI_DATA_FILE_ID must name the shared CI attachment" + traces = Path(REAL_ENVIRONMENT.get("HUD_E2E_TRACE_DIR") or tmp_path / "traces") / template + # A stable directory name keeps every run deploying to the same environment. + project = generate(hud, template, tmp_path / f"ci-{template}", wheel=sdk_wheel, uv=uv) + project.bundle_sdk_into_image() + deployed = project.hud("deploy", "--no-env", timeout=1800) + assert deployed.exit_code == 0, deployed + project.write_rows(data_file) + live.set(HUD_TELEMETRY_ENABLED="1", HUD_TELEMETRY_LOCAL_DIR=str(traces)) + + run = evaluated( + project.hud(*EVAL, *REAL_MODEL, "--model", HOSTED_MODEL, "--runtime", "hud", timeout=900) + ) + + assert_lifecycle(steps(run["trace_id"], traces), data_file) + # Without a local span directory, `hud trace get` reads the trace from the platform. + live.set(HUD_TELEMETRY_LOCAL_DIR=None) + events = read_back(project.hud, run["trace_id"]) + (traces / "remote-events.json").write_text(json.dumps(events, indent=2)) + + +def read_back(hud: Hud, trace_id: str, attempts: int = 12, interval: float = 5.0) -> list[Any]: + """The platform's events for ``trace_id``, polled until they include the agent's messages.""" + for _ in range(attempts - 1): + result = hud("trace", "get", trace_id, "--json") + if agent_spoke(result): + return result.json + time.sleep(interval) + result = hud("trace", "get", trace_id, "--json") + assert agent_spoke(result), f"no agent events on the platform after {attempts} reads:\n{result}" + return result.json + + +def agent_spoke(result: Result) -> bool: + return result.exit_code == 0 and any(event["kind"] == "agent_message" for event in result.json) diff --git a/hud/capabilities/tests/__init__.py b/tests/environment/__init__.py similarity index 100% rename from hud/capabilities/tests/__init__.py rename to tests/environment/__init__.py diff --git a/tests/environment/test_control_channel.py b/tests/environment/test_control_channel.py new file mode 100644 index 000000000..b06d2ce43 --- /dev/null +++ b/tests/environment/test_control_channel.py @@ -0,0 +1,451 @@ +"""The control channel as a peer speaks it: the handshake, the task catalog, a task's start +and grade, how a session's template is torn down, bad requests, and the frame limit.""" + +from __future__ import annotations + +from typing import Any, Literal + +import pytest +from pydantic import BaseModel + +from hud.capabilities import Capability +from hud.environment import ( + DataFileArg, + DataFileRef, + DataFilesArg, + Environment, + GradingArg, + PromptArg, +) +from hud.eval import LocalRuntime, Task +from hud.graders import EvaluationResult, SubScore +from tests.support import FRAME_LIMIT, SESSION_ID, encode, wire + +INVALID, NOT_FOUND, BAD_STATE, FAILED = -32602, -32601, -32600, -32000 + + +async def test_hello_names_the_session_the_env_and_what_its_hooks_published() -> None: + env = Environment("protocol", version="1.2.3") + + @env.initialize + async def publish_browser() -> None: + env.add_capability(Capability.cdp(url="ws://127.0.0.1:9/devtools/browser")) + + async with ( + LocalRuntime(env)(Task(env="protocol", id="x")) as runtime, + wire(runtime.url) as peer, + ): + hello = (await peer.call("hello", {}))["result"] + + assert (hello["session_id"], hello["env"]) == ( + SESSION_ID, + {"name": "protocol", "version": "1.2.3"}, + ) + assert [(b["name"], b["url"]) for b in hello["bindings"]] == [ + ("browser", "ws://127.0.0.1:9/devtools/browser") + ] + + +class Point(BaseModel): + x: int + y: int + + +class Criterion(BaseModel): + requirement: str + + +async def test_the_catalog_publishes_each_templates_arguments_and_io_types() -> None: + env = Environment("catalog") + + @env.template(description="Adds two numbers.") + async def add(a: int, suite: str = "math"): + yield "go" + yield 1.0 + + @env.template() + async def flexible(n: int, **rest: str): + yield "go" + yield 1.0 + + @env.template(input=Point, returns=Point) + async def structured(): + yield "go" + yield 1.0 + + @env.template() + async def files( + prompt: PromptArg, + attachment: DataFileArg[DataFileRef], + fixtures: DataFilesArg[DataFileRef], + criteria: GradingArg[Criterion], + ): + yield "go" + yield 1.0 + + async with LocalRuntime(env)(Task(env="catalog", id="x")) as runtime, wire(runtime.url) as peer: + listed = {t["id"]: t for t in (await peer.call("tasks.list"))["result"]["tasks"]} + + add_args = listed["add"]["args"] + assert (listed["add"]["description"], add_args["required"]) == ("Adds two numbers.", ["a"]) + assert add_args["properties"]["suite"]["default"] == "math" + assert [name for name, t in listed.items() if t["args"]["additionalProperties"]] == ["flexible"] + assert [name for name, t in listed.items() if {"input", "returns"} <= t.keys()] == [ + "structured" + ] + assert {k: v["x-hud-hint"] for k, v in listed["files"]["args"]["properties"].items()} == { + "prompt": "prompt", + "attachment": "data-file", + "fixtures": "data-files", + "criteria": "grading", + } + + +class Payload(BaseModel): + text: str + + +@pytest.mark.parametrize( + ("args", "prompt"), + [ + ({"mode": '"upper"', "payload": '{"text":"hello"}', "retries": "3"}, "HELLO!!!"), + ({"mode": "lower", "payload": {"text": "HeLLo"}}, "hello"), + ({"mode": "upper", "payload": {"text": "hi"}, "retries": None}, "HI"), + ], + ids=["json-strings-are-parsed-to-their-types", "native-values-validate", "none-is-kept"], +) +async def test_start_arguments_arrive_as_their_annotated_types( + args: dict[str, Any], prompt: str +) -> None: + env = Environment("typed") + + @env.template() + async def shout(mode: Literal["upper", "lower"], payload: Payload, retries: int | None = None): + text = payload.text.upper() if mode == "upper" else payload.text.lower() + yield text + "!" * (retries or 0) + yield 1.0 + + async with LocalRuntime(env)(Task(env="typed", id="x")) as runtime, wire(runtime.url) as peer: + started = await peer.call("tasks.start", {"id": "shout", "args": args}) + + assert started["result"] == {"prompt": prompt} + + +async def test_data_file_and_grading_arguments_arrive_as_models() -> None: + env = Environment("files") + + @env.template() + async def review(attachment: DataFileArg[DataFileRef], criteria: GradingArg[Criterion]): + yield f"{type(attachment).__name__}:{attachment.path} {type(criteria[0]).__name__}" + yield 1.0 + + args = { + "attachment": {"file_id": "file-1", "path": "brief.pdf"}, + "criteria": [{"requirement": "Answer it"}], + } + async with LocalRuntime(env)(Task(env="files", id="x")) as runtime, wire(runtime.url) as peer: + started = await peer.call("tasks.start", {"id": "review", "args": args}) + + assert started["result"] == {"prompt": "DataFileRef:brief.pdf Criterion"} + + +@pytest.mark.parametrize( + ("answer", "content"), + [('{"x": 1, "y": 2}', {"x": 1, "y": 2}), ("about (1, 2)", "about (1, 2)")], + ids=["an-answer-of-the-type-is-parsed", "any-other-answer-arrives-as-written"], +) +async def test_a_declared_return_type_parses_the_answer_and_keeps_it_raw( + answer: str, content: Any +) -> None: + env = Environment("structured") + + @env.template(returns=Point) + async def point(): + answer = yield "Name a point." + parsed = answer.content + yield { + "score": 1.0 if isinstance(parsed, Point) else 0.0, + "content": parsed.model_dump() if isinstance(parsed, Point) else parsed, + "raw": answer.raw, + } + + async with ( + LocalRuntime(env)(Task(env="structured", id="x")) as runtime, + wire(runtime.url) as peer, + ): + await peer.call("tasks.start", {"id": "point"}) + graded = (await peer.call("tasks.grade", {"answer": answer}))["result"] + + assert (graded["content"], graded["raw"]) == (content, answer) + + +async def test_an_evaluation_result_reaches_the_peer_with_its_details_and_breakdown() -> None: + env = Environment("graded") + + @env.template() + async def judged(): + yield "go" + yield EvaluationResult( + reward=0.75, + content="nice", + info={"max_tile": 256}, + subscores=[ + SubScore( + name="judge", + value=0.75, + children=[SubScore(name="criterion", value=1.0, info={"reason": "because"})], + ) + ], + ) + + async with LocalRuntime(env)(Task(env="graded", id="x")) as runtime, wire(runtime.url) as peer: + await peer.call("tasks.start", {"id": "judged"}) + graded = (await peer.call("tasks.grade", {"answer": "x"}))["result"] + + assert (graded["score"], graded["content"], graded["info"]) == (0.75, "nice", {"max_tile": 256}) + (judge,) = graded["subscores"] + assert (judge["name"], judge["children"][0]["info"]) == ("judge", {"reason": "because"}) + + +async def test_a_template_that_ends_without_grading_grades_zero() -> None: + env = Environment("ungraded") + + @env.template() + async def quits(): + yield "go" + + async with ( + LocalRuntime(env)(Task(env="ungraded", id="x")) as runtime, + wire(runtime.url) as peer, + ): + await peer.call("tasks.start", {"id": "quits"}) + graded = await peer.call("tasks.grade", {"answer": "x"}) + + assert graded["result"] == {"score": 0.0} + + +@pytest.mark.parametrize( + ("method", "params", "reply"), + [ + ("tasks.cancel", None, {"cancelled": True}), + ("bye", None, {"goodbye": True}), + ("tasks.grade", {"answer": "x"}, {"score": 1.0}), + ("tasks.start", {"id": "hold", "args": {"name": "second"}}, {"prompt": "holding second"}), + ], + ids=["cancel", "bye", "grade", "a-new-start"], +) +async def test_ending_a_sessions_task_closes_its_template( + method: str, params: dict[str, Any] | None, reply: dict[str, Any] +) -> None: + events = [] + env = Environment("held") + + @env.template() + async def hold(name: str): + try: + yield f"holding {name}" + yield 1.0 + finally: + events.append(f"closed {name}") + + async with LocalRuntime(env)(Task(env="held", id="x")) as runtime, wire(runtime.url) as peer: + await peer.call("tasks.start", {"id": "hold", "args": {"name": "first"}}) + answered = await peer.call(method, params) + closed = list(events) + + assert (answered["result"], closed) == (reply, ["closed first"]) + + +async def test_a_dropped_connection_parks_its_task_until_the_server_stops() -> None: + events = [] + env = Environment("held") + + @env.template() + async def hold(): + try: + yield "holding" + yield 1.0 + finally: + events.append("closed") + + async with LocalRuntime(env)(Task(env="held", id="x")) as runtime: + async with wire(runtime.url) as peer: + await peer.call("tasks.start", {"id": "hold"}) + parked = list(events) + + assert (parked, events) == ([], ["closed"]) + + +ROUTE = {"capability": "ssh", "host": "h", "port": "443"} +CONNECTION = { + "name": "inference", + "capability": "ssh", + "url": "https://inference.example", + "headers": {"Authorization": "Bearer t"}, +} + + +@pytest.mark.parametrize( + ("method", "params", "code"), + [ + ("tasks.start", {"id": "missing"}, INVALID), + ("tasks.start", {"id": "record", "args": ["x"]}, INVALID), + ("tasks.start", {"id": 5}, INVALID), + ("tasks.start", {"id": "record", "args": {"nmae": "x"}}, FAILED), + ("tasks.start", {"id": "lookup"}, FAILED), + ("tasks.grade", {"answer": "x"}, BAD_STATE), + ("tasks.stop", {}, NOT_FOUND), + ("hello", {"session_id": "sess-00000000"}, BAD_STATE), + ("hello", {"session_id": 7}, INVALID), + ("hello", {"workspace_routes": [ROUTE]}, INVALID), + ("hello", {"workspace_routes": "all"}, INVALID), + ("hello", {"connections": [CONNECTION]}, FAILED), + ("hello", {"connections": "all"}, INVALID), + ], + ids=[ + "an-unknown-template", + "arguments-that-are-not-an-object", + "a-template-id-that-is-not-a-string", + "arguments-the-template-does-not-take", + "a-template-that-raises-keyerror-is-not-an-unknown-task", + "a-grade-before-a-start", + "an-unknown-method", + "resuming-an-unknown-session", + "a-session-id-that-is-not-a-string", + "a-route-whose-port-is-not-a-number", + "routes-that-are-not-a-list", + "a-connection-without-a-workspace-to-bind-it", + "connections-that-are-not-a-list", + ], +) +async def test_a_bad_request_gets_an_error_reply_runs_nothing_and_the_session_goes_on( + method: str, params: dict[str, Any], code: int +) -> None: + started = [] + env = Environment("errors", capabilities=[Capability.cdp(url="ws://127.0.0.1:9/devtools")]) + + @env.template() + async def record(name: str): + started.append(name) + yield "go" + yield 1.0 + + @env.template() + async def lookup(): + settings: dict[str, str] = {} + yield settings["missing"] + + async with LocalRuntime(env)(Task(env="errors", id="x")) as runtime, wire(runtime.url) as peer: + refused = await peer.call(method, params) + listed = await peer.call("tasks.list") + + assert refused["error"]["code"] == code + assert ([t["id"] for t in listed["result"]["tasks"]], started) == (["record", "lookup"], []) + + +async def test_a_tunnel_opened_once_a_session_has_begun_is_an_unknown_method() -> None: + env = Environment("errors", capabilities=[Capability.cdp(url="ws://127.0.0.1:9/devtools")]) + + async with LocalRuntime(env)(Task(env="errors", id="x")) as runtime, wire(runtime.url) as peer: + await peer.call("hello", {}) + refused = await peer.call("tunnel.open", {"capability": "browser"}) + listed = await peer.call("tasks.list") + + assert (refused["error"]["code"], listed["result"]) == (NOT_FOUND, {"tasks": []}) + + +def start_of_size(size: int) -> dict[str, Any]: + """A ``tasks.start`` frame for ``echo`` of exactly ``size`` bytes, without its newline.""" + frame: dict[str, Any] = { + "jsonrpc": "2.0", + "id": 1, + "method": "tasks.start", + "params": {"id": "echo", "args": {"data": ""}}, + } + frame["params"]["args"]["data"] = "x" * (size - (len(encode(frame)) - 1)) + return frame + + +async def test_a_request_at_the_frame_limit_round_trips_its_payload() -> None: + env = Environment("limits") + + @env.template() + async def echo(data: str): + yield data + yield {"score": 1.0, "detail": data} + + frame = start_of_size(FRAME_LIMIT) + async with LocalRuntime(env)(Task(env="limits", id="x")) as runtime, wire(runtime.url) as peer: + await peer.write(encode(frame)) + started = await peer.read() + graded = await peer.call("tasks.grade", {"answer": "done"}) + + data = frame["params"]["args"]["data"] + assert started is not None + assert (started["result"]["prompt"], graded["result"]["detail"]) == (data, data) + + +async def test_a_request_over_the_frame_limit_is_refused_without_an_id_and_hung_up() -> None: + async with ( + LocalRuntime(Environment("limits"))(Task(env="limits", id="x")) as runtime, + wire(runtime.url) as peer, + ): + await peer.write(encode(start_of_size(FRAME_LIMIT + 1))) + refused = await peer.read() + after = await peer.read() + + assert refused is not None + assert (refused["id"], refused["error"]["code"], after) == (None, FAILED, None) + + +async def test_a_reply_too_large_for_its_own_id_is_an_error_and_the_session_goes_on() -> None: + frame: dict[str, Any] = {"jsonrpc": "2.0", "id": "", "method": "nope"} + frame["id"] = "x" * (FRAME_LIMIT - (len(encode(frame)) - 1)) + + async with ( + LocalRuntime(Environment("limits"))(Task(env="limits", id="x")) as runtime, + wire(runtime.url) as peer, + ): + await peer.write(encode(frame)) + refused = await peer.read() + listed = await peer.call("tasks.list") + + assert refused is not None + assert (refused["id"], refused["error"]["code"]) == (None, FAILED) + assert listed["result"] == {"tasks": []} + + +@pytest.mark.parametrize("too_large", ["prompt", "error"]) +async def test_a_prompt_or_error_over_the_frame_limit_is_refused_and_its_template_released( + too_large: str, +) -> None: + released = [] + env = Environment("bloated") + + @env.template() + async def huge_prompt(): + try: + yield "x" * FRAME_LIMIT + yield 1.0 + finally: + released.append("prompt") + + @env.template() + async def huge_error(): + try: + raise ValueError("x" * FRAME_LIMIT) + yield "never" + finally: + released.append("error") + + async with LocalRuntime(env)(Task(env="bloated", id="x")) as runtime: + async with wire(runtime.url) as peer: + refused = await peer.call("tasks.start", {"id": f"huge_{too_large}"}) + async with wire(runtime.url) as later: + regraded = await later.call("tasks.grade", {"answer": "done"}) + + assert (refused["error"]["code"], released, regraded["error"]["code"]) == ( + FAILED, + [too_large], + BAD_STATE, + ) diff --git a/tests/environment/test_file_tracking.py b/tests/environment/test_file_tracking.py new file mode 100644 index 000000000..b3db840ef --- /dev/null +++ b/tests/environment/test_file_tracking.py @@ -0,0 +1,190 @@ +"""``filetracking/1``: what a served workspace with ``track_files=True`` reports about its +files: what it tracks, what changed since the last baseline, the deliverables it captures, +and what never leaves the workspace.""" + +from __future__ import annotations + +import base64 +import hashlib +from typing import TYPE_CHECKING + +from hud.environment import Environment +from tests.fixtures.environments import served +from tests.support import wire, write_files + +if TYPE_CHECKING: + from pathlib import Path + +#: Larger than the per-file diff limit (1 MiB). +BIG_TEXT = 1024 * 1024 + 1 +#: Larger than the per-file capture limit (5 MiB). +BIG_PDF = 5 * 1024 * 1024 + 1 + + +def tracked(root: Path) -> Environment: + env = Environment("tracked") + env.workspace(root, track_files=True) + return env + + +async def test_a_snapshot_lists_the_files_it_tracks_leaving_out_what_is_not_the_work( + tmp_path: Path, +) -> None: + root = tmp_path / "root" + write_files( + root, + { + "a.txt": "line1\n", + "pkg/data.txt": "package data\n", + "pkg/.gitignore": "data.txt\n", + ".gitignore": "ignored.txt\n", + "ignored.txt": "ignored\n", + "node_modules/dep.js": "module.exports = 1;\n", + ".tmp/go-build123/_pkg_.a": "compiled\n", + ".git/config": "[core]\n", + ".gitmodules": "[submodule]\n", + ".env": "API_KEY=secret\n", + }, + ) + + async with served(tracked(root)) as client, wire(client.binding("filetracking").url) as peer: + snapshot = await peer.call("snapshot") + + files = {entry["path"]: entry["content_hash"] for entry in snapshot["result"]["files"]} + assert sorted(files) == [".gitignore", "a.txt", "pkg/.gitignore", "pkg/data.txt"] + assert files["a.txt"] == hashlib.sha256(b"line1\n").hexdigest() + + +async def test_a_diff_reports_what_changed_since_the_last_one(tmp_path: Path) -> None: + root = tmp_path / "root" + write_files(root, {"edited.txt": "line1\nline2\n", "removed.txt": "gone\n"}) + + async with served(tracked(root)) as client, wire(client.binding("filetracking").url) as peer: + await peer.call("snapshot") + before = await peer.call("diff") + write_files(root, {"edited.txt": "line1\nCHANGED\n", "added.txt": "hello\n"}) + (root / "removed.txt").unlink() + edited = await peer.call("diff") + again = await peer.call("diff") + + patches = {patch["path"]: patch for patch in edited["result"]["patches"]} + assert (before["result"]["patches"], again["result"]["patches"]) == ([], []) + assert {path: patch["status"] for path, patch in patches.items()} == { + "edited.txt": "modified", + "added.txt": "added", + "removed.txt": "deleted", + } + assert "+CHANGED" in patches["edited.txt"]["patch"] + + +async def test_secrets_never_leave_in_a_diff(tmp_path: Path) -> None: + root = tmp_path / "root" + write_files( + root, + {".env": "API_KEY=supersecretvalue\n", ".gitmodules": "url = https://token@x/p.git\n"}, + ) + + async with served(tracked(root)) as client, wire(client.binding("filetracking").url) as peer: + await peer.call("snapshot") + write_files( + root, + { + ".env": "API_KEY=supersecretvalue\nDB_PASSWORD=hunter2\n", + ".gitmodules": "url = https://new-token@x/p.git\n", + ".git/config": "url = https://new-token@x/repo.git\n", + }, + ) + edited = await peer.call("diff") + + assert {patch["path"] for patch in edited["result"]["patches"]} == {".env", ".gitmodules"} + for secret in ("supersecretvalue", "hunter2", "new-token"): + assert secret not in str(edited) + + +async def test_a_file_over_the_diff_limit_is_reported_without_its_contents( + tmp_path: Path, +) -> None: + root = tmp_path / "root" + write_files(root, {"big.txt": "a" * BIG_TEXT}) + + async with served(tracked(root)) as client, wire(client.binding("filetracking").url) as peer: + await peer.call("snapshot") + write_files(root, {"big.txt": "b" * (BIG_TEXT + 1)}) + edited = await peer.call("diff") + + ((path, status, patch),) = [ + (p["path"], p["status"], p["patch"]) for p in edited["result"]["patches"] + ] + assert (path, status) == ("big.txt", "modified") + assert len(patch) < 200 + + +async def test_setup_and_advance_each_move_the_baseline_later_diffs_start_from( + tmp_path: Path, +) -> None: + root = tmp_path / "root" + write_files(root, {"src.py": "x = 1\n"}) + + async with served(tracked(root)) as client, wire(client.binding("filetracking").url) as peer: + await peer.call("snapshot") + write_files(root, {"generated.txt": "setup\n", "src.py": "x = 2\n"}) + setup = await peer.call("setup") + after_setup = await peer.call("diff") + write_files(root, {"legacy.txt": "legacy\n"}) + await peer.call("advance") + after_advance = await peer.call("diff") + + assert {p["path"]: p["status"] for p in setup["result"]["patches"]} == { + "generated.txt": "added", + "src.py": "modified", + } + assert (after_setup["result"]["patches"], after_advance["result"]["patches"]) == ([], []) + + +async def test_a_flush_diffs_the_deliverables_and_captures_the_documents_under_the_limit( + tmp_path: Path, +) -> None: + root = tmp_path / "root" + write_files(root, {".env.xlsx": "API_KEY=before"}) + + async with served(tracked(root)) as client, wire(client.binding("filetracking").url) as peer: + await peer.call("snapshot") + write_files( + root, + { + "analysis.py": "print('tracked by diff')\n", + "deliverable.xlsx": "workbook bytes", + "report.html": "report\n", + ".env.xlsx": "API_KEY=after", + "large.pdf": "%" * BIG_PDF, + }, + ) + flushed = (await peer.call("flush"))["result"] + + assert {p["path"]: p["status"] for p in flushed["diff"]["patches"]} == { + "analysis.py": "added", + "deliverable.xlsx": "added", + "report.html": "added", + ".env.xlsx": "modified", + "large.pdf": "added", + } + capture = flushed["capture"] + captured = {f["path"]: base64.b64decode(f["file"]["data"]) for f in capture["files"]} + assert captured == { + "deliverable.xlsx": b"workbook bytes", + "report.html": b"report\n", + } + assert (capture["files_skipped"], capture["truncated"]) == (1, True) + assert "API_KEY=after" not in str(flushed) + + +async def test_an_unknown_request_is_refused_and_tracking_goes_on(tmp_path: Path) -> None: + root = tmp_path / "root" + write_files(root, {"a.txt": "one\n"}) + + async with served(tracked(root)) as client, wire(client.binding("filetracking").url) as peer: + refused = await peer.call("rewind") + snapshot = await peer.call("snapshot") + + assert refused["error"]["code"] == -32000 + assert [entry["path"] for entry in snapshot["result"]["files"]] == ["a.txt"] diff --git a/tests/environment/test_serving.py b/tests/environment/test_serving.py new file mode 100644 index 000000000..4c555b52b --- /dev/null +++ b/tests/environment/test_serving.py @@ -0,0 +1,204 @@ +"""Serving an environment: the entry points that own a process, and the lifecycle they run. + +Both entry points, ``python -m hud.environment.server`` (what images run) and +``hud serve``, are rows of each table. They resolve a reference in a fixture tree +whose environments leave a ``stopped-`` marker from a shutdown hook, print +``HUD_SERVE_PORT=`` once the control channel accepts, and stop cleanly on a +signal, even with a session still grading. A reference that does not load ends +the process with the error and without a port. +""" + +from __future__ import annotations + +import asyncio +import os +import signal +import sys +from typing import TYPE_CHECKING + +import pytest + +from hud.environment import Environment +from tests.fixtures.environments import served +from tests.support import wire + +if TYPE_CHECKING: + from pathlib import Path + + +def declare(name: str, variable: str = "env") -> str: + """Source declaring Environment ``name``, whose shutdown hook leaves a marker in cwd.""" + return ( + "from pathlib import Path\n" + "from hud import Environment\n" + f"{variable} = Environment({name!r})\n" + f"@{variable}.shutdown\n" + f"async def _stopped_{variable}():\n" + f" Path('stopped-{name}').write_text('stopped')\n" + ) + + +TREE = { + "env.py": declare("from-env-py"), + "pkg/__init__.py": ( + "from pathlib import Path\n" + "from hud.environment import Environment\n" + "def make_env(name='from-factory'):\n" + " env = Environment(name)\n" + " @env.shutdown\n" + " async def _stopped():\n" + " Path(f'stopped-{name}').write_text('stopped')\n" + " return env\n" + ), + "multi.py": declare("env-one", "first") + declare("env-two", "second"), + "stalls.py": declare("stalls") + + "import asyncio\n" + + '@env.template(id="stall")\nasync def stall():\n yield "go"\n' + + " await asyncio.Event().wait()\n yield 1.0\n", +} + +SERVER = [sys.executable, "-m", "hud.environment.server"] +HUD_SERVE = [sys.executable, "-m", "hud.cli", "serve", "--port", "0"] + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + root = tmp_path / "project" + for name, content in TREE.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return root + + +async def announced_port(process: asyncio.subprocess.Process) -> int: + assert process.stdout is not None + async with asyncio.timeout(60): + while line := await process.stdout.readline(): + text = line.decode().strip() + if text.startswith("HUD_SERVE_PORT="): + return int(text.removeprefix("HUD_SERVE_PORT=")) + assert process.stderr is not None + raise AssertionError(f"exited without a port:\n{(await process.stderr.read()).decode()}") + + +@pytest.mark.parametrize( + ("command", "served_name", "stop", "grading"), + [ + pytest.param([*SERVER, "env.py"], "from-env-py", signal.SIGTERM, False, id="server"), + pytest.param([*HUD_SERVE, "env:env"], "from-env-py", signal.SIGINT, False, id="hud-serve"), + pytest.param( + [*HUD_SERVE, "pkg:make_env", "--arg", "name=demo"], + "demo", + signal.SIGTERM, + False, + id="hud-serve-factory", + ), + pytest.param( + [*SERVER, "stalls.py"], "stalls", signal.SIGTERM, True, id="server-while-grading" + ), + pytest.param( + [*HUD_SERVE, "stalls:env"], + "stalls", + signal.SIGTERM, + True, + id="hud-serve-while-grading", + ), + ], +) +async def test_a_serving_process_announces_its_port_and_runs_shutdown_hooks_on_a_signal( + tree: Path, command: list[str], served_name: str, stop: signal.Signals, grading: bool +) -> None: + process = await asyncio.create_subprocess_exec( + *command, + cwd=tree, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env={**os.environ, "NO_COLOR": "1"}, + ) + try: + port = await announced_port(process) + async with wire(f"tcp://127.0.0.1:{port}") as control: + hello = await control.call("hello", {}) + if grading: + await control.call("tasks.start", {"id": "stall", "args": {}}) + await control.send("tasks.grade", {"answer": "x"}) + process.send_signal(stop) + code = await asyncio.wait_for(process.wait(), timeout=10) + finally: + if process.returncode is None: + process.kill() + await process.wait() + + assert hello["result"]["env"]["name"] == served_name + assert code == 0 + assert [name for name in os.listdir(tree) if name.startswith("stopped-")] == [ + f"stopped-{served_name}" + ] + + +@pytest.mark.parametrize( + ("command", "code", "error"), + [ + pytest.param( + [*SERVER, "multi.py"], 1, "multiple Environments in multi.py", id="server-ambiguous" + ), + pytest.param( + [*HUD_SERVE, "multi.py"], + 2, + "multiple Environments in multi.py", + id="hud-serve-ambiguous", + ), + pytest.param( + [*HUD_SERVE, "pkg:make_env", "--arg", "name"], 2, "--arg", id="hud-serve-malformed-arg" + ), + pytest.param( + [*SERVER, "missing.py"], 1, "no environment source at missing.py", id="server-missing" + ), + pytest.param( + [*HUD_SERVE, "missing.py"], + 1, + "no environment source at missing.py", + id="hud-serve-missing", + ), + ], +) +async def test_a_serving_process_exits_with_the_load_error( + tree: Path, command: list[str], code: int, error: str +) -> None: + process = await asyncio.create_subprocess_exec( + *command, cwd=tree, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=60) + + assert process.returncode == code + assert b"HUD_SERVE_PORT=" not in stdout + assert error in stderr.decode() + + +async def test_initialize_hooks_finish_before_hello_and_shutdown_hooks_run_in_reverse() -> None: + env = Environment("ordered") + log: list[str] = [] + + @env.initialize + async def first_up() -> None: + await asyncio.sleep(0.1) + log.append("first up") + + @env.initialize + async def second_up() -> None: + log.append("second up") + + @env.shutdown + async def first_down() -> None: + log.append("first down") + + @env.shutdown + async def second_down() -> None: + log.append("second down") + + async with served(env): + seen_at_hello = list(log) + + assert seen_at_hello == ["first up", "second up"] + assert log == ["first up", "second up", "second down", "first down"] diff --git a/tests/environment/test_sessions.py b/tests/environment/test_sessions.py new file mode 100644 index 000000000..24921b4e6 --- /dev/null +++ b/tests/environment/test_sessions.py @@ -0,0 +1,105 @@ +"""Control sessions: each connection drives its own task, and a dropped one parks its task +for a later connection to grade, by resuming the session (``hello`` with its id) or, +when exactly one session is parked, by a plain ``tasks.grade``.""" + +from __future__ import annotations + +from hud.environment import Environment +from hud.eval import LocalRuntime, Task +from tests.support import wire + + +def tagged() -> Environment: + env = Environment("sessions") + + @env.template() + async def echo(tag: str): + yield f"go {tag}" + yield {"score": 1.0, "tag": tag} + + return env + + +async def test_concurrent_sessions_each_grade_their_own_task() -> None: + async with ( + LocalRuntime(tagged())(Task(env="sessions", id="echo")) as runtime, + wire(runtime.url) as a, + wire(runtime.url) as b, + ): + await a.call("tasks.start", {"id": "echo", "args": {"tag": "a"}}) + await b.call("tasks.start", {"id": "echo", "args": {"tag": "b"}}) + graded = [ + (await peer.call("tasks.grade", {"answer": "x"}))["result"]["tag"] for peer in (a, b) + ] + + assert graded == ["a", "b"] + + +async def test_a_new_start_replaces_only_its_own_sessions_task() -> None: + async with ( + LocalRuntime(tagged())(Task(env="sessions", id="echo")) as runtime, + wire(runtime.url) as a, + wire(runtime.url) as b, + ): + await b.call("tasks.start", {"id": "echo", "args": {"tag": "b"}}) + await a.call("tasks.start", {"id": "echo", "args": {"tag": "first"}}) + await a.call("tasks.start", {"id": "echo", "args": {"tag": "second"}}) + graded = [ + (await peer.call("tasks.grade", {"answer": "x"}))["result"]["tag"] for peer in (a, b) + ] + + assert graded == ["second", "b"] + + +async def test_a_session_dropped_before_its_prompt_is_resumed_by_its_id_and_graded() -> None: + async with LocalRuntime(tagged())(Task(env="sessions", id="echo")) as runtime: + async with wire(runtime.url) as a: + session = (await a.call("hello", {}))["result"]["session_id"] + await a.send("tasks.start", {"id": "echo", "args": {"tag": "parked"}}) + await a.hang_up() + async with wire(runtime.url) as b: + resumed = (await b.call("hello", {"session_id": session}))["result"]["session_id"] + graded = await b.call("tasks.grade", {"answer": "x"}) + + assert (resumed, graded["result"]["tag"]) == (session, "parked") + + +async def test_a_plain_grade_adopts_the_only_parked_session() -> None: + async with LocalRuntime(tagged())(Task(env="sessions", id="echo")) as runtime: + async with wire(runtime.url) as a: + await a.call("tasks.start", {"id": "echo", "args": {"tag": "parked"}}) + async with wire(runtime.url) as b: + graded = await b.call("tasks.grade", {"answer": "x"}) + + assert graded["result"]["tag"] == "parked" + + +async def test_with_several_parked_sessions_a_grade_must_name_one() -> None: + sessions = [] + async with LocalRuntime(tagged())(Task(env="sessions", id="echo")) as runtime: + for tag in ("one", "two"): + async with wire(runtime.url) as peer: + sessions.append((await peer.call("hello", {}))["result"]["session_id"]) + await peer.call("tasks.start", {"id": "echo", "args": {"tag": tag}}) + async with wire(runtime.url) as c: + refused = await c.call("tasks.grade", {"answer": "x"}) + await c.call("hello", {"session_id": sessions[1]}) + graded = await c.call("tasks.grade", {"answer": "x"}) + + assert refused["error"]["message"] == ( + f"2 parked sessions ({', '.join(sorted(sessions))}); " + "resume one by sending hello with its session_id" + ) + assert graded["result"]["tag"] == "two" + + +async def test_a_session_with_a_live_connection_cannot_be_resumed() -> None: + async with ( + LocalRuntime(tagged())(Task(env="sessions", id="echo")) as runtime, + wire(runtime.url) as a, + wire(runtime.url) as b, + ): + session = (await a.call("hello", {}))["result"]["session_id"] + refused = await b.call("hello", {"session_id": session}) + + assert refused["error"]["message"] == f"session {session!r} has a live connection" diff --git a/tests/environment/test_tunnel.py b/tests/environment/test_tunnel.py new file mode 100644 index 000000000..1ad3374fa --- /dev/null +++ b/tests/environment/test_tunnel.py @@ -0,0 +1,147 @@ +"""Capability tunneling: environment daemons reached through the control port. + +A capability address belongs to the environment's network. The manifest keeps +that address for processes running there, while a client binding points at a +local forwarder. Each connection to the forwarder becomes one connection to the +control port, opened with a ``tunnel.open`` preface frame and then spliced raw to +the daemon. These scenarios front a TCP echo daemon with a served env. +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +import pytest + +from hud.capabilities import Capability +from hud.environment import Environment +from hud.eval import LocalRuntime, Task +from tests.fixtures.environments import served +from tests.support import FRAME_LIMIT, encode, wire + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +@pytest.fixture +async def echo_port() -> AsyncIterator[int]: + """A daemon on the environment's network that echoes every byte back.""" + + async def echo(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + finally: + writer.close() + + server = await asyncio.start_server(echo, "127.0.0.1", 0) + yield server.sockets[0].getsockname()[1] + server.close() + await server.wait_closed() + + +def echo_env(port: int) -> Environment: + return Environment( + "echo-env", + capabilities=[Capability(name="echo", protocol="rfb/3.8", url=f"rfb://127.0.0.1:{port}")], + ) + + +async def round_trip(host: str | None, port: int | None, payload: bytes) -> bytes: + reader, writer = await asyncio.open_connection(host, port) + writer.write(payload) + await writer.drain() + data = await reader.readexactly(len(payload)) + writer.close() + await writer.wait_closed() + return data + + +async def test_concurrent_tunnel_streams_do_not_interleave(echo_port: int) -> None: + payloads = [f"stream-{i}".encode() * 100 for i in range(8)] + + async with served(echo_env(echo_port)) as client: + binding = urlsplit(client.binding("echo").url) + echoed = await asyncio.gather( + *(round_trip(binding.hostname, binding.port, payload) for payload in payloads) + ) + + assert echoed == payloads + + +async def test_closing_the_client_closes_its_forwarders(echo_port: int) -> None: + async with served(echo_env(echo_port)) as client: + binding = urlsplit(client.binding("echo").url) + + with pytest.raises(OSError): + await asyncio.open_connection(binding.hostname, binding.port) + + +@pytest.mark.parametrize("preface_size", [128, FRAME_LIMIT]) +async def test_raw_bytes_sent_with_the_preface_reach_the_daemon( + echo_port: int, preface_size: int +) -> None: + preface: dict[str, Any] = { + "jsonrpc": "2.0", + "id": 1, + "method": "tunnel.open", + "params": {"capability": "echo"}, + "padding": "", + } + preface["padding"] = "x" * (preface_size - (len(encode(preface)) - 1)) + env = echo_env(echo_port) + + async with ( + LocalRuntime(env)(Task(env=env.name, id="tunnel")) as runtime, + wire(runtime.url) as stream, + ): + await stream.write(encode(preface) + b"raw bytes") + opened = await stream.read() + echoed = await asyncio.wait_for(stream.reader.readexactly(9), timeout=30) + + assert opened == {"jsonrpc": "2.0", "id": 1, "result": {"capability": "echo"}} + assert echoed == b"raw bytes" + + +@pytest.mark.parametrize( + ("params", "code"), + [ + pytest.param( + {"capability": "nope"}, + -32000, + id="unknown-capability", + ), + ], +) +async def test_a_refused_tunnel_gets_one_error_frame_and_is_closed( + echo_port: int, params: dict[str, Any], code: int +) -> None: + env = echo_env(echo_port) + + async with ( + LocalRuntime(env)(Task(env=env.name, id="tunnel")) as runtime, + wire(runtime.url) as stream, + ): + refused = await stream.call("tunnel.open", params) + after = await stream.read() + + assert (refused["id"], refused["error"]["code"]) == (1, code) + assert after is None + + +async def test_manifest_keeps_environment_addresses_and_client_bindings_are_forwarded() -> None: + local = Capability.cdp(name="browser", url="ws://127.0.0.1:9222") + remote = Capability.ssh(name="box", url="ssh://box.example.com:22", host_pubkey="ssh-ed25519 x") + env = Environment("mixed-env", capabilities=[local, remote]) + + async with served(env) as client: + assert client.manifest is not None + assert client.manifest.bindings == [local, remote] + for declared in (local, remote): + forwarded = client.binding(declared.name) + assert forwarded.url != declared.url + assert urlsplit(forwarded.url).hostname == "127.0.0.1" + assert (forwarded.name, forwarded.protocol) == (declared.name, declared.protocol) diff --git a/hud/environment/tests/__init__.py b/tests/environment/workspace/__init__.py similarity index 100% rename from hud/environment/tests/__init__.py rename to tests/environment/workspace/__init__.py diff --git a/tests/environment/workspace/conftest.py b/tests/environment/workspace/conftest.py new file mode 100644 index 000000000..5befb0b49 --- /dev/null +++ b/tests/environment/workspace/conftest.py @@ -0,0 +1,42 @@ +"""A served workspace, reached the way an agent reaches it: over its ``ssh/2`` binding. + +``shell`` serves ``env.workspace(root, ...)`` on loopback and yields the +:class:`~hud.environment.Workspace` it attached (for what the environment itself +does with it: verifier runs, launches) with an SSH client on its published binding. + +Rows marked ``SANDBOX`` need Linux, root and bubblewrap; they run in the sandbox +lane (``-m "e2e and sandbox"``), where every session runs in the bubblewrap +sandbox. Unmarked rows run wherever the suite runs, sandboxed or not. +""" + +from __future__ import annotations + +import sys +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any, cast + +import pytest + +from hud.environment import Environment +from tests.fixtures.environments import served + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + from pathlib import Path + + from hud.capabilities import SSHClient + from hud.environment import Workspace + +SANDBOX = (pytest.mark.e2e, pytest.mark.sandbox) + +if sys.platform == "win32": + collect_ignore_glob = ["test_*.py"] + + +@asynccontextmanager +async def shell(root: Path, **options: Any) -> AsyncIterator[tuple[Workspace, SSHClient]]: + """Serve a workspace over ``root`` and connect to its shell binding.""" + env = Environment("workspace") + workspace = env.workspace(root, track_files=False, **options) + async with served(env) as client: + yield workspace, cast("SSHClient", await client.open("shell")) diff --git a/tests/environment/workspace/test_egress.py b/tests/environment/workspace/test_egress.py new file mode 100644 index 000000000..b6d0fca4c --- /dev/null +++ b/tests/environment/workspace/test_egress.py @@ -0,0 +1,748 @@ +"""A bounded workspace's ways out: the policy proxy, peer forwards and the bridge. + +The proxy and each peer forward serve on a unix socket, and the bridge offers +them as ports inside the workspace's network. Scenarios speak to them over real +sockets; the internet behind the proxy is :func:`tests.support.internet`. +""" + +from __future__ import annotations + +import asyncio +import base64 +import http.client +import json +import shutil +import socket +import struct +import subprocess +import tempfile +import threading +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.capabilities import Connection +from hud.environment import Peer +from hud.environment.egress import ( + ANY_HOST, + ConnectionRelay, + Egress, +) +from tests.support import eventually, internet + +from .conftest import shell + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + +REQUEST = ( + b"POST /v1/echo HTTP/1.1\r\nHost: upstream\r\nContent-Length: 12\r\n" + b"Connection: close\r\n\r\nrequest body" +) +BODY = b"complete response" * 32768 +RESPONSE = b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s" % ( + len(BODY), + BODY, +) + + +@contextmanager +def sockets() -> Iterator[Path]: + """A socket directory outside the pytest tmp dir: unix socket paths cap near 104 bytes.""" + directory = Path(tempfile.mkdtemp(dir="/tmp")) + try: + yield directory + finally: + shutil.rmtree(directory, ignore_errors=True) + + +@contextmanager +def proxy(allowed: set[str], **options: Any) -> Iterator[Egress]: + with sockets() as directory: + egress = Egress(directory, allowed, **options) + egress.start() + try: + yield egress + finally: + egress.stop() + + +def exchange(path: Path, request: bytes) -> bytes: + """One request to a unix-socket proxy, and its whole response.""" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(5) + client.connect(str(path)) + client.sendall(request) + response = b"" + while b"\r\n\r\n" not in response: + response += client.recv(65536) + head, _, body = response.partition(b"\r\n\r\n") + lengths = [ + int(line.split(b":", 1)[1]) + for line in head.split(b"\r\n") + if line.lower().startswith(b"content-length:") + ] + if not lengths: + return response + drain(client) + while len(body) < lengths[0]: + body += client.recv(65536) + return head + b"\r\n\r\n" + body + + +def basic(token: str) -> str: + return base64.b64encode(f"hud:{token}".encode()).decode() + + +def drain(client: socket.socket) -> bytes: + received = bytearray() + while chunk := client.recv(65536): + received.extend(chunk) + return bytes(received) + + +@pytest.mark.parametrize( + ("allowed", "token", "request_bytes", "status", "reason"), + [ + pytest.param( + {"example.com"}, + None, + b"GET http://other.example/ HTTP/1.1\r\nHost: other.example\r\n\r\n", + b"403", + b"blocked-by-allowlist", + id="a-host-outside-the-allowlist", + ), + pytest.param( + {"pypi.org"}, + None, + b"GET http://notpypi.org/ HTTP/1.1\r\nHost: notpypi.org\r\n\r\n", + b"403", + b"blocked-by-allowlist", + id="a-host-sharing-an-allowed-suffix", + ), + pytest.param( + {"pypi.org"}, + None, + b"GET http://pypi.org.evil.com/ HTTP/1.1\r\nHost: pypi.org.evil.com\r\n\r\n", + b"403", + b"blocked-by-allowlist", + id="an-allowed-name-as-a-prefix", + ), + pytest.param( + {ANY_HOST}, + None, + b"GET / HTTP/1.1\r\n\r\n", + b"403", + b"blocked-by-allowlist", + id="no-host-at-all", + ), + pytest.param( + {"pypi.org"}, + None, + b"GET http://files.pypi.org/ HTTP/1.1\r\nHost: files.pypi.org\r\n\r\n", + b"502", + b"upstream-failure", + id="a-subdomain-of-an-allowed-host-is-carried", + ), + pytest.param( + {ANY_HOST}, + None, + b"GET http://anything.example/ HTTP/1.1\r\nHost: anything.example\r\n\r\n", + b"502", + b"upstream-failure", + id="any-host-is-carried", + ), + pytest.param( + {ANY_HOST}, + None, + b"CONNECT 127.0.0.1:8765 HTTP/1.1\r\nHost: 127.0.0.1:8765\r\n\r\n", + b"403", + b"blocked-by-address", + id="connect-to-a-substrate-address", + ), + pytest.param( + {ANY_HOST}, + None, + b"GET http://internal.example:8765/ HTTP/1.1\r\nHost: internal.example\r\n\r\n", + b"403", + b"blocked-by-address", + id="a-name-resolving-into-the-substrate", + ), + pytest.param( + {"example.com"}, + "secret", + b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n", + b"407", + b"Proxy-Authenticate", + id="a-visitor-proxy-without-its-credential", + ), + pytest.param( + {"example.com"}, + "secret", + b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n" + b"Proxy-Authorization: Basic " + basic("another").encode() + b"\r\n\r\n", + b"407", + b"Proxy-Authenticate", + id="a-visitor-proxy-with-another-visitors-credential", + ), + pytest.param( + {"example.com"}, + None, + b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n", + b"502", + b"upstream-failure", + id="an-upstream-that-cannot-be-reached", + ), + ], +) +def test_the_proxy_carries_only_what_its_allowlist_permits_and_says_why_not( + monkeypatch: pytest.MonkeyPatch, + allowed: set[str], + token: str | None, + request_bytes: bytes, + status: bytes, + reason: bytes, +) -> None: + internet(monkeypatch, resolve={"internal.example": "127.0.0.1"}) + with proxy(allowed, token=token) as egress: + response = exchange(egress.socket_path, request_bytes) + + assert response.split(b"\r\n", 1)[0].split()[1] == status + assert reason in response + + +def test_the_proxy_normalizes_http_framing_in_both_directions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: dict[str, Any] = {} + + class Chunked(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + seen.update( + host=self.headers.get("Host"), + length=self.headers.get("Content-Length"), + transfer=self.headers.get("Transfer-Encoding"), + ) + seen["body"] = self.rfile.read(int(seen["length"] or 0)) + self.send_response(200) + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + self.wfile.write(b"5\r\nhello\r\n0\r\n\r\n") + + def log_message(self, format: str, *args: Any) -> None: + pass + + with serving(Chunked) as port: + internet(monkeypatch, servers={"upstream.example": port}) + with proxy({"upstream.example"}) as egress: + response = exchange( + egress.socket_path, + b"POST http://upstream.example/ HTTP/1.1\r\nHost: forbidden.example\r\n" + b"Transfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n", + ) + + headers, _, body = response.partition(b"\r\n\r\n") + assert b"200" in headers.split(b"\r\n", 1)[0] + assert b"transfer-encoding" not in headers.lower() + assert body == b"hello" + assert seen == { + "host": "upstream.example", + "length": "11", + "transfer": None, + "body": b"hello world", + } + + +@pytest.mark.parametrize( + ("header", "relayed"), + [ + pytest.param(b"Content-Type: text/plain", True, id="an-ordinary-header"), + pytest.param(b"X-Meta:", True, id="an-empty-value"), + pytest.param(b"X-Evil: a\r\n b", False, id="a-folded-value"), + pytest.param(b"X-Evil: a\x00b", False, id="a-control-character"), + ], +) +def test_the_proxy_relays_an_upstream_header_only_as_the_upstream_sent_it( + monkeypatch: pytest.MonkeyPatch, header: bytes, relayed: bool +) -> None: + """An upstream header is remote text, and a folded value keeps its CRLF + through http.client. Relayed verbatim it would carry headers of its own + into the response the workspace reads, so a field outside the grammar + fails the whole response rather than being quietly repaired.""" + with raw_upstream(b"HTTP/1.1 200 OK\r\n" + header + b"\r\nContent-Length: 2\r\n\r\nok") as port: + internet(monkeypatch, servers={"upstream.example": port}) + with proxy({"upstream.example"}) as egress: + response = exchange( + egress.socket_path, + b"GET http://upstream.example/ HTTP/1.1\r\nHost: upstream.example\r\n\r\n", + ) + + if relayed: + assert response.startswith(b"HTTP/1.1 200") + assert header.split(b":")[0] in response + else: + assert response.startswith(b"HTTP/1.1 502") + assert b"X-Proxy-Error: unrelayable-upstream-header" in response + + +@contextmanager +def serving(handler: type[BaseHTTPRequestHandler]) -> Iterator[int]: + server = HTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server.server_address[1] + finally: + server.shutdown() + server.server_close() + thread.join() + + +@contextmanager +def raw_upstream(answer: bytes, listener: socket.socket | None = None) -> Iterator[int]: + """A server that reads one request to its blank line and answers ``answer``.""" + server = listener or socket.create_server(("127.0.0.1", 0)) + server.settimeout(5) + + def serve() -> None: + with server, server.accept()[0] as connection: + request = b"" + while b"\r\n\r\n" not in request: + request += connection.recv(65536) + connection.sendall(answer) + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + try: + yield server.getsockname()[1] if server.family == socket.AF_INET else 0 + finally: + thread.join(5) + + +@contextmanager +def half_close_upstream(listener: socket.socket) -> Iterator[list[bytes]]: + """Reads a connection to its EOF, records it, then answers ``RESPONSE``.""" + requests: list[bytes] = [] + listener.settimeout(5) + + def serve() -> None: + with listener, listener.accept()[0] as connection: + requests.append(drain(connection)) + connection.sendall(RESPONSE) + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + try: + yield requests + finally: + thread.join(5) + assert not thread.is_alive() + + +@contextmanager +def bridge( + egress: Egress, cwd: Path, **options: Any +) -> Iterator[tuple[int, subprocess.Popen[bytes]]]: + """The bridge process serving ``egress`` on a free loopback port.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as holder: + holder.bind(("127.0.0.1", 0)) + port = holder.getsockname()[1] + argv, config = egress.bridge_command(port=port, **options) + process = subprocess.Popen( + argv, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + try: + assert process.stdin is not None and process.stdout is not None + process.stdin.write(config) + process.stdin.flush() + assert process.stdout.readline() == b"ready\n" + yield port, process + finally: + process.terminate() + process.wait(timeout=5) + + +@contextmanager +def through_the_bridge( + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[tuple[socket.socket, list[bytes]]]: + del monkeypatch + with sockets() as directory: + egress = Egress(directory, {"example.com"}) + upstream = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + upstream.bind(str(egress.socket_path)) + upstream.listen() + with ( + half_close_upstream(upstream) as requests, + bridge(egress, directory) as (port, _process), + socket.create_connection(("127.0.0.1", port), timeout=5) as client, + ): + yield client, requests + + +@contextmanager +def through_a_connect_tunnel( + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[tuple[socket.socket, list[bytes]]]: + upstream = socket.create_server(("127.0.0.1", 0)) + internet(monkeypatch, servers={"example.com": upstream.getsockname()[1]}) + with ( + half_close_upstream(upstream) as requests, + proxy({"example.com"}) as egress, + socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client, + ): + client.settimeout(5) + client.connect(str(egress.socket_path)) + client.sendall(b"CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n") + established = b"" + while b"\r\n\r\n" not in established: + established += client.recv(4096) + assert b"200 Connection established" in established + yield client, requests + + +@contextmanager +def through_a_peer_forward(_: pytest.MonkeyPatch) -> Iterator[tuple[socket.socket, list[bytes]]]: + upstream = socket.create_server(("127.0.0.1", 0)) + with ( + half_close_upstream(upstream) as requests, + proxy(set(), peers=[Peer("service", upstream.getsockname()[1])]) as egress, + socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client, + ): + ((_, _, peer_socket),) = json.loads(egress.bridge_command()[1]) + client.settimeout(5) + client.connect(peer_socket) + yield client, requests + + +@contextmanager +def through_a_connection_relay( + _: pytest.MonkeyPatch, +) -> Iterator[tuple[socket.socket, list[bytes]]]: + requests: list[bytes] = [] + + class Upstream(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + requests.append(self.rfile.read(int(self.headers["Content-Length"]))) + self.send_response(200) + self.send_header("Content-Length", str(len(BODY))) + self.end_headers() + self.wfile.write(BODY) + + def log_message(self, format: str, *args: Any) -> None: + pass + + with serving(Upstream) as port: + relay = ConnectionRelay( + Connection( + name="inference", + capability="ssh", + url=f"http://127.0.0.1:{port}/v1", + headers={"Authorization": "Bearer scoped-runtime-token"}, + ) + ) + relay.start() + try: + with socket.create_connection(("127.0.0.1", relay.port), timeout=5) as client: + yield client, requests + finally: + relay.stop() + + +@pytest.mark.parametrize( + "route", + [ + pytest.param(through_the_bridge, id="bridge"), + pytest.param(through_a_connect_tunnel, id="connect-tunnel"), + pytest.param(through_a_peer_forward, id="peer-forward"), + pytest.param(through_a_connection_relay, id="connection-relay"), + ], +) +def test_a_relay_delivers_the_whole_response_after_the_client_half_closes( + monkeypatch: pytest.MonkeyPatch, + route: Callable[[pytest.MonkeyPatch], Any], +) -> None: + with route(monkeypatch) as (client, requests): + client.settimeout(5) + client.sendall(REQUEST) + client.shutdown(socket.SHUT_WR) + received = drain(client) + + assert received.endswith(BODY) + assert requests[0].endswith(b"request body") + + +def test_the_bridge_ignores_workspace_imports_and_closes_reset_connections( + tmp_path: Path, +) -> None: + workspace = tmp_path / "workspace" + package = workspace / "hud" / "environment" + package.mkdir(parents=True) + (workspace / "hud" / "__init__.py").write_text("") + (package / "__init__.py").write_text("") + marker = workspace / "imported" + (package / "utils.py").write_text( + f"from pathlib import Path\nPath({str(marker)!r}).write_text('imported')\n" + ) + + with sockets() as directory: + egress = Egress(directory, {"example.com"}) + upstream_listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + upstream_listener.bind(str(egress.socket_path)) + upstream_listener.listen() + upstream_listener.settimeout(5) + with upstream_listener, bridge(egress, workspace) as (port, process): + command_line = subprocess.run( + ["ps", "-o", "command=", "-p", str(process.pid)], + capture_output=True, + text=True, + check=True, + ).stdout + client = socket.create_connection(("127.0.0.1", port), timeout=5) + upstream, _ = upstream_listener.accept() + with upstream: + upstream.settimeout(1) + client.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + client.close() + closed = upstream.recv(1) + + assert not marker.exists() + # Where the sockets live is configuration, not something a process listing shows. + assert str(directory) not in command_line + assert closed == b"" + + +VISITOR = """ +import base64, http.client, os, pathlib, sys, time, urllib.parse + +name, own_host, other_host = sys.argv[1:] +workspace = pathlib.Path.cwd() +(workspace / f"started-{name}").write_text(os.environ["http_proxy"]) +while not (workspace / f"release-{name}").exists(): + time.sleep(0.01) + +def status(host, proxy_url): + proxy = urllib.parse.urlsplit(proxy_url) + secret = urllib.parse.unquote(proxy.password) + connection = http.client.HTTPConnection("127.0.0.1", proxy.port, timeout=5) + credential = base64.b64encode(f"{proxy.username}:{secret}".encode()).decode() + authorization = {"Proxy-Authorization": f"Basic {credential}"} + connection.request("GET", f"http://{host}/", headers=authorization) + return str(connection.getresponse().status) + +proxy_url = os.environ["http_proxy"] +statuses = [status(own_host, proxy_url), status(other_host, proxy_url)] +stale = workspace / "stale-proxy" +if stale.exists(): + statuses.append(status(own_host, stale.read_text())) +print(" ".join(statuses)) +""" + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_overlapping_verifier_runs_keep_their_own_policies_and_credentials( + tmp_path: Path, +) -> None: + """Each verifier run is held to its own allowlist behind its own credential, + and a second run waits for the first to release the visitor route.""" + root = tmp_path / "root" + + def visitor(name: str, own: str, other: str) -> Any: + return workspace.run( + ["python3", "-c", VISITOR, name, own, other], allowed_hosts={own}, identity=None + ) + + with sockets() as credentials: + async with shell(root, require_isolation=True, credentials_dir=credentials) as ( + workspace, + _, + ): + first = asyncio.create_task(visitor("first", "first.example", "second.example")) + await eventually(lambda: (root / "started-first").exists()) + second = asyncio.create_task(visitor("second", "second.example", "first.example")) + await asyncio.sleep(0.2) + second_waited = not (root / "started-second").exists() + (root / "release-first").touch() + first_result = await first + + await eventually(lambda: (root / "started-second").exists()) + first_proxy = (root / "started-first").read_text() + (root / "stale-proxy").write_text(first_proxy) + (root / "release-second").touch() + second_result = await second + second_proxy = (root / "started-second").read_text() + route_left_open = (credentials / "visitor" / "egress.sock").exists() + + assert second_waited + assert first_proxy != second_proxy + # An allowed host is carried (and fails upstream: nothing resolves in here). + assert first_result.stdout.decode().split() == ["502", "403"] + assert second_result.stdout.decode().split() == ["502", "403", "407"] + assert not route_left_open + + +@pytest.mark.e2e +@pytest.mark.sandbox +@pytest.mark.parametrize( + ("options", "allowed", "expected"), + [ + pytest.param( + {"allowed_hosts": {"pypi.org"}}, + None, + {"https_proxy": "http://127.0.0.1:3128"}, + id="no-policy-of-its-own-uses-the-sessions-proxy", + ), + pytest.param( + {"allowed_hosts": {"pypi.org"}}, + {"pypi.org"}, + {"https_proxy": "http://127.0.0.1:3128"}, + id="the-sessions-policy-uses-the-sessions-proxy", + ), + pytest.param( + {"peers": [Peer("db", 5432)], "ports": [5432]}, + {"pypi.org"}, + {"https_proxy": "@127.0.0.1:3129", "NO_PROXY": "db,127.0.0.2"}, + id="a-policy-of-its-own-gets-the-visitor-route", + ), + ], +) +async def test_a_verifier_run_is_pointed_at_the_proxy_for_its_policy( + tmp_path: Path, options: dict[str, Any], allowed: set[str] | None, expected: dict[str, str] +) -> None: + async with shell(tmp_path / "root", require_isolation=True, **options) as (workspace, _): + result = await workspace.run(["env"], allowed_hosts=allowed, identity=None) + environment = parsed(result.stdout) + + for name, value in expected.items(): + if name == "https_proxy": + assert environment[name].endswith(value) + else: + assert set(value.split(",")) <= set(environment[name].split(",")) + + +def parsed(output: bytes) -> dict[str, str]: + return dict(line.split("=", 1) for line in output.decode().splitlines() if "=" in line) + + +@contextmanager +def relay_to(upstream: type[BaseHTTPRequestHandler], **connection: Any) -> Iterator[int]: + """A connection relay in front of ``upstream``, holding the controller's credential.""" + with serving(upstream) as port: + relay = ConnectionRelay( + Connection( + name="inference", + capability="ssh", + url=f"http://127.0.0.1:{port}{connection.get('prefix', '')}", + headers=connection.get("headers", {"Authorization": "Bearer scoped-runtime-token"}), + ) + ) + relay.start() + try: + yield relay.port + finally: + relay.stop() + + +def test_a_connection_relay_replaces_credentials_and_preserves_streaming() -> None: + seen: dict[str, Any] = {} + + class Upstream(BaseHTTPRequestHandler): + def do_POST(self) -> None: + seen.update( + authorization=self.headers.get("Authorization"), + api_key=self.headers.get("X-Api-Key"), + body=self.rfile.read(int(self.headers.get("Content-Length", "0"))), + ) + self.send_response(200) + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + self.wfile.write(b"5\r\nfirst\r\n6\r\nsecond\r\n0\r\n\r\n") + + def log_message(self, format: str, *args: Any) -> None: + pass + + token = "scoped-runtime-token" + headers = {"Authorization": f"Bearer {token}", "X-Api-Key": token} + with relay_to(Upstream, prefix="/v1", headers=headers) as port: + client = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + client.request( + "POST", + "/v1/messages", + body=b"request", + headers={"Authorization": "Bearer model-visible", "X-Api-Key": "model-visible"}, + ) + response = client.getresponse() + status, body = response.status, response.read() + client.close() + + assert (status, body) == (200, b"firstsecond") + assert seen == {"authorization": f"Bearer {token}", "api_key": token, "body": b"request"} + + +def test_a_connection_relay_refuses_paths_that_could_leave_its_prefix() -> None: + forwarded: list[str] = [] + + class Upstream(BaseHTTPRequestHandler): + def do_GET(self) -> None: + forwarded.append(self.path) + self.send_response(204) + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: + pass + + answers: dict[str, tuple[int, str | None]] = {} + with relay_to(Upstream, prefix="/v1") as port: + for path in ( + "/v1/../admin", + "/v1/./models", + "/v1/%2e%2e/admin", + "/v1/%2E%2E/admin", + "/v1/..%2fadmin", + "/v1/..%5cadmin", + "/v1/models", + ): + client = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + client.request("GET", path) + response = client.getresponse() + response.read() + answers[path] = (response.status, response.getheader("X-Connection-Error")) + client.close() + + assert forwarded == ["/v1/models"] + assert answers.pop("/v1/models") == (204, None) + assert set(answers.values()) == {(400, "non-canonical-path")} + + +@pytest.mark.parametrize("placeholder", ["X-Api-Key", "Api-Key", "X-Goog-Api-Key"]) +def test_a_connection_relay_drops_process_credentials_the_connection_does_not_set( + placeholder: str, +) -> None: + received: dict[str, str] = {} + + class Upstream(BaseHTTPRequestHandler): + def do_POST(self) -> None: + received.update({key.casefold(): value for key, value in self.headers.items()}) + self.send_response(204) + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: + pass + + with relay_to(Upstream) as port: + client = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + client.request("POST", "/v1/messages", body=b"{}", headers={placeholder: "model-visible"}) + status = client.getresponse().status + client.close() + + assert status == 204 + assert received["authorization"] == "Bearer scoped-runtime-token" + assert placeholder.casefold() not in received + assert "model-visible" not in received.values() diff --git a/tests/environment/workspace/test_process_connections.py b/tests/environment/workspace/test_process_connections.py new file mode 100644 index 000000000..74c664ccc --- /dev/null +++ b/tests/environment/workspace/test_process_connections.py @@ -0,0 +1,401 @@ +"""Controller connections bound to one process: only that process reaches them. + +A control session hands the environment a :class:`~hud.capabilities.Connection`; +the workspace serves it behind a credential-owning relay that only the process +it was requested for may connect to. The guard that enforces this has one +backend per kernel facility (seccomp user notification, ptrace), each held to +the same table; the served workspace is the sandbox-lane scenario. +""" + +from __future__ import annotations + +import asyncio +import shlex +import shutil +import subprocess +import sys +import tempfile +import threading +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal, cast + +import pytest + +from hud.capabilities import Connection, SSHClient +from hud.clients import HudProtocolError, connect +from hud.environment import Environment, Peer, WorkspaceRoute, process_guard +from hud.environment.egress import ANY_HOST, BRIDGE_PORT +from hud.environment.process_guard import ProcessConnectionGuard, process_connections_supported +from hud.eval import LocalRuntime, Task + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Iterator + +pytestmark = pytest.mark.skipif( + not process_connections_supported(), reason="process connection guards are unavailable" +) + +ECHO = Task(env="sessions", id="echo") + + +class Recorded(BaseHTTPRequestHandler): + """Answers ``ok`` and records the Authorization header it was sent.""" + + authorizations: list[str | None] + + def do_GET(self) -> None: + self.authorizations.append(self.headers.get("Authorization")) + self.send_response(200) + self.send_header("Content-Length", "2") + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, format: str, *args: Any) -> None: + pass + + +@contextmanager +def upstream() -> Iterator[tuple[int, list[str | None]]]: + authorizations: list[str | None] = [] + handler = type("Upstream", (Recorded,), {"authorizations": authorizations}) + server = HTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server.server_address[1], authorizations + finally: + server.shutdown() + server.server_close() + thread.join() + + +def fetch(url: str) -> str: + return ( + "import urllib.request;" + f"request=urllib.request.Request({url!r},headers={{'Authorization':'Bearer visible'}});" + "print(urllib.request.urlopen(request,timeout=5).read().decode())" + ) + + +def threaded_fetch(url: str) -> str: + return ( + "import threading,urllib.request;" + "result=[];" + "thread=threading.Thread(target=lambda:result.append(" + f"urllib.request.urlopen({url!r},timeout=5).read().decode()));" + "thread.start();thread.join();print(result[0])" + ) + + +def in_a_child(source: str) -> str: + """Run ``source`` in a child process; print the child's exit status.""" + return ( + f"import subprocess,sys;print(subprocess.run([sys.executable,'-c',{source!r}]).returncode)" + ) + + +@contextmanager +def guard_directory() -> Iterator[Path]: + """Outside the pytest tmp dir: the guard's unix socket path caps near 104 bytes.""" + directory = Path(tempfile.mkdtemp(dir="/tmp")) + try: + yield directory / "guard" + finally: + shutil.rmtree(directory, ignore_errors=True) + + +def test_the_guard_projects_a_helper_that_runs_on_its_own(tmp_path: Path) -> None: + with guard_directory() as directory: + guard = ProcessConnectionGuard(directory, set(), set(), backend="notify") + guard.start() + projected = guard.helper_path.read_bytes() + mode = guard.helper_path.stat().st_mode & 0o777 + standalone = subprocess.run( + [sys.executable, "-I", str(guard.helper_path), "--help"], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + guard.close() + + assert projected.startswith(b'"""Linux process-bound network connection enforcement."""') + assert mode == 0o500 + assert (standalone.returncode, standalone.stderr) == (0, "") + + +def backend_works(backend: str) -> bool: + return ( + sys.platform == "linux" + and subprocess.run( + [sys.executable, process_guard.__file__, "--probe", backend], + capture_output=True, + check=False, + ).returncode + == 0 + ) + + +@pytest.mark.e2e +@pytest.mark.sandbox +@pytest.mark.parametrize( + "backend", + [ + pytest.param( + name, + marks=pytest.mark.skipif(not backend_works(name), reason=f"no {name} guard backend"), + ) + for name in ("notify", "ptrace") + ], +) +async def test_only_the_guarded_process_reaches_its_protected_destination( + backend: Literal["notify", "ptrace"], +) -> None: + with upstream() as (protected, _), upstream() as (ordinary, _): + protected_url = f"http://127.0.0.1:{protected}" + child = ( + "import subprocess,sys,threading,urllib.request;" + f"print(urllib.request.urlopen({protected_url!r},timeout=5).read().decode());" + f"exec({threaded_fetch(protected_url)!r});" + f"blocked=subprocess.run([sys.executable,'-c',{fetch(protected_url)!r}]);" + "print(f'descendant={blocked.returncode}',flush=True);" + f"permitted=subprocess.run([sys.executable,'-c',{fetch(f'http://127.0.0.1:{ordinary}')!r}]);" + "print(f'ordinary={permitted.returncode}')" + ) + target = ("127.0.0.1", protected) + with guard_directory() as directory: + guard = ProcessConnectionGuard(directory, {target}, {target}, backend=backend) + guard.start() + process = await asyncio.create_subprocess_exec( + sys.executable, + str(guard.helper_path), + "--backend", + backend, + str(guard.socket_path), + "--", + sys.executable, + "-c", + child, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + await guard.wait_ready() + stdout, stderr = await asyncio.wait_for(process.communicate(), 20) + finally: + if process.returncode is None: + process.kill() + await process.wait() + guard.close() + + assert process.returncode == 0, stderr.decode(errors="replace") + lines = stdout.decode().splitlines() + assert lines[:2] == ["ok", "ok"] + assert lines[2] != "descendant=0" + assert lines[3:] == ["ok", "ordinary=0"] + + +def python(source: str) -> str: + """A session command running ``source`` on the image's interpreter.""" + return f"exec python3 -c {shlex.quote(source)}" + + +IO_URING_DENIED = ( + "import ctypes,errno,sys;" + "libc=ctypes.CDLL(None,use_errno=True);" + "result=libc.syscall(425,0,None);" + "sys.exit(0 if result == -1 and ctypes.get_errno() == errno.EPERM else 1)" +) +PROXY_ENVIRONMENT = ( + "import os;print(os.environ.get('http_proxy',''));" + "print(os.environ.get('https_proxy',''));print(os.environ.get('no_proxy',''))" +) + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_only_the_bound_session_process_reaches_a_controller_connection( + tmp_path: Path, +) -> None: + with upstream() as (protected, protected_seen), upstream() as (ordinary, ordinary_seen): + connection = Connection( + name="inference", + capability="ssh", + url=f"http://127.0.0.1:{protected}", + headers={"Authorization": "Bearer scoped-runtime-token"}, + ) + url = connection.client_url + env = Environment("process-connection") + env.workspace( + tmp_path / "root", + peers=(Peer("ordinary.hud.invalid", 80, target=("127.0.0.1", ordinary)),), + allowed_hosts={ANY_HOST}, + require_isolation=True, + track_files=False, + ) + cases = { + "unbound": (fetch(url), False), + "bound": (fetch(url), True), + "bound-thread": (threaded_fetch(url), True), + "bound-descendant": (in_a_child(fetch(url)), True), + "bound-descendant-by-mapped-address": ( + in_a_child( + "import http.client;" + f"c=http.client.HTTPConnection('::ffff:127.0.0.1',{connection.port},timeout=5);" + "c.request('GET','/');print(c.getresponse().read().decode())" + ), + True, + ), + "bound-descendant-through-the-proxy": ( + in_a_child( + "import http.client,sys;" + f"c=http.client.HTTPConnection('127.0.0.1',{BRIDGE_PORT},timeout=5);" + f"c.request('GET',{url!r});r=c.getresponse();" + "sys.exit(0 if r.status==200 and r.read()==b'ok' else 9)" + ), + True, + ), + "bound-io-uring": (IO_URING_DENIED, True), + "bound-proxy-environment": (PROXY_ENVIRONMENT, True), + "unbound-ordinary-peer": (fetch("http://ordinary.hud.invalid"), False), + } + results: dict[str, tuple[int | None, str]] = {} + async with ( + LocalRuntime(env)(Task(env=env.name, id="test")) as runtime, + connect(runtime, connections=(connection,)) as client, + ): + ssh = cast("SSHClient", await client.open("ssh")) + advertised = ssh.capability.params["process_connections"] + for case, (source, bound) in cases.items(): + process = await ssh.create_process( + python(source), connections=(connection,) if bound else () + ) + completed = await process.wait() + results[case] = (completed.returncode, cast("bytes", completed.stdout).decode()) + + assert advertised is True + assert results["unbound"][0] != 0 + assert results["bound"] == results["bound-thread"] == (0, "ok\n") + for case in ( + "bound-descendant", + "bound-descendant-by-mapped-address", + "bound-descendant-through-the-proxy", + ): + assert results[case][0] == 0 and results[case][1].strip() != "0", case + assert results["bound-io-uring"][0] == 0 + http_proxy, https_proxy, no_proxy = results["bound-proxy-environment"][1].splitlines() + assert http_proxy == https_proxy == f"http://127.0.0.1:{BRIDGE_PORT}" + assert "ordinary.hud.invalid" in no_proxy.split(",") + assert results["unbound-ordinary-peer"] == (0, "ok\n") + assert set(protected_seen) == {"Bearer scoped-runtime-token"} + assert set(ordinary_seen) == {"Bearer visible"} + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_a_later_session_cannot_launch_against_a_released_connection( + tmp_path: Path, +) -> None: + with upstream() as (protected, _): + connection = Connection( + name="inference", + capability="ssh", + url=f"http://127.0.0.1:{protected}", + headers={"Authorization": "Bearer scoped-runtime-token"}, + ) + env = Environment("process-connection") + env.workspace( + tmp_path / "root", allowed_hosts={ANY_HOST}, require_isolation=True, track_files=False + ) + async with LocalRuntime(env)(Task(env=env.name, id="test")) as runtime: + async with connect(runtime, connections=(connection,)) as first: + ssh = cast("SSHClient", await first.open("ssh")) + bound = await ssh.create_process( + python(fetch(connection.client_url)), connections=(connection,) + ) + first_status = (await bound.wait()).returncode + async with connect(runtime) as later: + ssh = cast("SSHClient", await later.open("ssh")) + stale = await ssh.create_process( + python(fetch(connection.client_url)), connections=(connection,) + ) + result = await stale.wait() + + assert first_status == 0 + assert result.returncode == 1 + assert b"process connection is not bound: inference" in cast("bytes", result.stderr) + + +def echo_env() -> Environment: + env = Environment("sessions") + + @env.template() + async def echo(tag: str): + yield f"go {tag}" + yield {"score": 1.0, "tag": tag} + + return env + + +def inference(token: str) -> Connection: + return Connection( + name="inference", + capability="ssh", + url="https://inference.hud.so", + headers={"Authorization": f"Bearer {token}"}, + ) + + +async def a_finished_session_releases_its_connection(env: Environment, root: Path) -> None: + workspace = env.workspace(root, network=False) + async with LocalRuntime(env)(ECHO) as runtime: + async with connect(runtime, connections=[inference("first")]) as first: + await first.start_task("echo", {"tag": "a"}) + await first.grade({"answer": "x"}) + async with connect(runtime, connections=[inference("rotated")]) as second: + await second.start_task("echo", {"tag": "b"}) + names = [peer.name for peer in workspace.peers] + assert names.count(inference("rotated").host) == 1 + assert (await second.grade({"answer": "x"}))["tag"] == "b" + + +async def a_live_session_keeps_its_connection_name(env: Environment, root: Path) -> None: + env.workspace(root, network=False) + async with LocalRuntime(env)(ECHO) as runtime, connect(runtime, connections=[inference("a")]): + with pytest.raises(HudProtocolError, match="bound to another control session"): + async with connect(runtime, connections=[inference("b")]): + pass + + +async def a_finished_session_takes_its_route_with_it(env: Environment, root: Path) -> None: + workspace = env.workspace(root, network=False) + route = WorkspaceRoute("ssh", "inference.hud.so", 443) + async with LocalRuntime(env)(ECHO) as runtime: + async with connect(runtime, workspace_routes=[route, route]) as first: + await first.start_task("echo", {"tag": "a"}) + assert workspace.peers == ( + Peer("inference.hud.so", 443, target=("inference.hud.so", 443)), + ) + await first.grade({"answer": "x"}) + async with connect(runtime) as later: + await later.start_task("echo", {"tag": "later"}) + assert workspace.peers == () + await later.grade({"answer": "x"}) + + +@pytest.mark.e2e +@pytest.mark.sandbox +@pytest.mark.parametrize( + "scenario", + [ + a_finished_session_releases_its_connection, + a_live_session_keeps_its_connection_name, + a_finished_session_takes_its_route_with_it, + ], +) +async def test_a_sessions_workspace_bindings_end_with_the_session( + tmp_path: Path, scenario: Callable[[Environment, Path], Awaitable[None]] +) -> None: + await scenario(echo_env(), tmp_path / "root") diff --git a/tests/environment/workspace/test_publication.py b/tests/environment/workspace/test_publication.py new file mode 100644 index 000000000..284692e12 --- /dev/null +++ b/tests/environment/workspace/test_publication.py @@ -0,0 +1,122 @@ +"""A workspace publishes its ``ssh/2`` binding when the environment serves, never before. + +``env.workspace(root)`` is pure declaration: keys, sockets and the directory +materialize at serve time, the manifest carries the published address, and +stopping the environment takes the daemon down. +""" + +from __future__ import annotations + +import asyncio +import stat +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +import asyncssh +import pytest + +from hud.clients import connect +from hud.environment import Environment, Workspace +from hud.environment.workspace import usable_bwrap +from hud.eval import LocalRuntime, Task +from tests.fixtures.environments import served + +from .conftest import SANDBOX, shell + +if TYPE_CHECKING: + from tests.support import HudEnv + +TEMP = Path(tempfile.gettempdir()).resolve() + + +async def test_a_workspace_lives_exactly_while_its_environment_serves(tmp_path: Path) -> None: + env = Environment("ws-env") + env.workspace(tmp_path / "root") + declared = (list(env.capabilities), (tmp_path / "root").exists()) + + async with LocalRuntime(env)(Task(env=env.name, id="test")) as runtime: + async with connect(runtime) as client: + first = client.binding("shell").params["host_pubkey"] + async with connect(runtime) as client: + again = client.binding("shell").params["host_pubkey"] + backing_port = urlsplit(env.capability("shell").url).port + async with served(env): + restarted = [capability.name for capability in env.capabilities] + + assert declared == ([], False) + assert again == first + assert restarted == ["shell", "filetracking"] + with pytest.raises(OSError): + _, writer = await asyncio.open_connection("127.0.0.1", backing_port) + writer.close() + + +@pytest.mark.parametrize( + ("options", "variables", "isolation", "published"), + [ + pytest.param({}, {}, "none", ["shell", "filetracking"], id="tracked-by-default"), + pytest.param( + {}, {"HUD_FILE_TRACKING_ENABLED": "false"}, "none", ["shell"], id="tracking-off" + ), + pytest.param({"track_files": False}, {}, "none", ["shell"], id="the-workspace-opts-out"), + pytest.param( + {}, {}, "bwrap", ["shell", "filetracking"], id="sandboxed-host", marks=SANDBOX + ), + ], +) +async def test_serving_publishes_the_bindings_its_settings_ask_for( + tmp_path: Path, + hud_env: HudEnv, + options: dict[str, Any], + variables: dict[str, str], + isolation: str, + published: list[str], +) -> None: + if isolation == "none" and usable_bwrap() is not None: + pytest.skip("this host sandboxes every workspace") + hud_env.set(**{"HUD_FILE_TRACKING_ENABLED": None, **variables}) + env = Environment("ws-env") + env.workspace(tmp_path / "root", **options) + + async with served(env) as client: + assert client.manifest is not None + bindings = {binding.name: binding for binding in client.manifest.bindings} + shell_binding = client.binding("shell") + tracking = client.binding("filetracking") if "filetracking" in published else None + + assert list(bindings) == published + assert bindings["shell"].params["isolation"] == isolation + assert (shell_binding.protocol, urlsplit(shell_binding.url).scheme) == ("ssh/2", "ssh") + assert shell_binding.params["host_pubkey"].startswith("ssh-ed25519") + if tracking is not None: + assert tracking.protocol == "filetracking/1" + assert tracking.params == { + "root": (tmp_path / "root").resolve().as_posix(), + "setup_diff": True, + } + + +async def test_session_credentials_live_privately_outside_the_served_root(tmp_path: Path) -> None: + """The agent's shell root must not hold its own SSH key material.""" + async with shell(tmp_path / "root") as (workspace, client): + credentials = Path(client.capability.params["client_key_path"]).parent + mode = stat.S_IMODE(credentials.stat().st_mode) + listing = await client.run("ls -A", check=True) + other = Workspace(tmp_path / "other") + other_credentials = Path(other.capability().params["client_key_path"]).parent + await other.stop() + + assert credentials.resolve().is_relative_to(TEMP) + assert not credentials.is_relative_to(workspace.root) + assert mode == 0o700 + assert other_credentials != credentials + assert listing.stdout == "" + assert not credentials.exists() + + +async def test_the_ssh_daemon_serves_no_sftp_subsystem(tmp_path: Path) -> None: + async with shell(tmp_path / "root") as (_, client): + with pytest.raises(asyncssh.ChannelOpenError): + await client.conn.start_sftp_client() diff --git a/tests/environment/workspace/test_sandbox.py b/tests/environment/workspace/test_sandbox.py new file mode 100644 index 000000000..333521b21 --- /dev/null +++ b/tests/environment/workspace/test_sandbox.py @@ -0,0 +1,405 @@ +"""The sandbox a workspace's processes run in, observed from inside them. + +Sessions run as the identity the workspace was given, in one sandbox they all +share, behind the network the task declared. Most rows here need Linux, root +and bubblewrap (the sandbox lane); the probe that decides whether bubblewrap +can sandbox at all runs anywhere, against stand-in ``bwrap`` executables. +""" + +from __future__ import annotations + +import asyncio +import os +import shutil +import socket +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.environment import Peer, Workspace +from hud.environment.workspace import Bubblewrap, usable_bwrap + +from .conftest import SANDBOX, shell + +if TYPE_CHECKING: + from tests.support import HudEnv + +IDENTITY = "echo $(id -u) $(id -g) $(id -G)" + + +def namespace(kind: str) -> str: + return os.readlink(f"/proc/self/ns/{kind}") + + +@pytest.mark.parametrize( + ("options", "expected"), + [ + pytest.param({}, None, id="serving-identity"), + pytest.param( + {"shell_uid": 1000}, + None, + id="shell-uid-is-a-no-op-off-root", + marks=pytest.mark.skipif(os.geteuid() == 0, reason="needs a non-root server"), + ), + pytest.param({"require_isolation": True}, "0 0 0", id="sandboxed-root", marks=SANDBOX), + pytest.param( + {"require_isolation": True, "shell_uid": 1000}, + "1000 1000 1000", + id="dropped-to-shell-uid", + marks=SANDBOX, + ), + pytest.param( + {"require_isolation": True, "shell_uid": 1000, "shell_gid": 2000}, + "1000 2000 2000", + id="dropped-keeps-the-declared-group", + marks=SANDBOX, + ), + ], +) +async def test_a_session_runs_as_the_identity_the_workspace_was_given( + tmp_path: Path, options: dict[str, Any], expected: str | None +) -> None: + async with shell(tmp_path / "root", **options) as (_, client): + identity = (await client.run(IDENTITY, check=True)).stdout + no_new_privs = (await client.run("grep NoNewPrivs /proc/self/status", check=False)).stdout + + if expected is None: + assert str(identity).split()[0] == str(os.getuid()) + else: + assert str(identity).strip() == expected + if "shell_uid" in options and expected is not None: + # A setuid binary must not let the dropped shell regain root. + assert str(no_new_privs).split() == ["NoNewPrivs:", "1"] + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_a_dropped_session_has_a_world_writable_tmp(tmp_path: Path) -> None: + async with shell(tmp_path / "root", require_isolation=True, shell_uid=1000) as (_, client): + result = await client.run("stat -c %a /tmp && touch /tmp/scratch", check=False) + + assert (result.exit_status, str(result.stdout).strip()) == (0, "1777") + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_only_the_workspace_directory_is_handed_to_the_shell_uid(tmp_path: Path) -> None: + """The handoff is O(1): recursing over baked content (node_modules, a venv) + on the serving path would delay the control-port bind past the deploy + readiness probe, so pre-staged content stays its author's.""" + root = tmp_path / "root" + (root / "pkg").mkdir(parents=True) + (root / "pkg" / "mod.py").write_text("x = 1\n") + + async with shell(root, shell_uid=1000): + owners = { + path.name: path.stat().st_uid for path in (root, root / "pkg", root / "pkg" / "mod.py") + } + + assert owners == {"root": 1000, "pkg": 0, "mod.py": 0} + + +def isolation_unavailable(_: pytest.MonkeyPatch, hud_env: HudEnv) -> dict[str, Any]: + del hud_env + if usable_bwrap() is not None: + pytest.skip("this host can sandbox") + return {"require_isolation": True} + + +def setpriv_missing(_: pytest.MonkeyPatch, hud_env: HudEnv) -> dict[str, Any]: + hud_env.set(PATH="/nonexistent") + return {"shell_uid": 1000} + + +def chown_denied(monkeypatch: pytest.MonkeyPatch, hud_env: HudEnv) -> dict[str, Any]: + del hud_env + + def deny(*_: object) -> None: + raise PermissionError("operation not permitted") + + monkeypatch.setattr(os, "lchown", deny) + return {"shell_uid": 1000} + + +@pytest.mark.parametrize( + ("cause", "error", "message"), + [ + pytest.param( + isolation_unavailable, + RuntimeError, + "isolation was required", + id="isolation-required-without-bwrap", + ), + pytest.param( + setpriv_missing, + RuntimeError, + "privileges cannot be dropped", + id="root-without-setpriv", + marks=SANDBOX, + ), + pytest.param( + chown_denied, PermissionError, "not permitted", id="handoff-chown-denied", marks=SANDBOX + ), + ], +) +async def test_a_workspace_that_cannot_wall_its_sessions_refuses_to_serve( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + hud_env: HudEnv, + cause: Any, + error: type[Exception], + message: str, +) -> None: + workspace = Workspace(tmp_path / "root", **cause(monkeypatch, hud_env)) + + with pytest.raises(error, match=message): + await workspace.start() + + +@pytest.mark.e2e +@pytest.mark.sandbox +@pytest.mark.parametrize("guest_path", ["/workspace", "/tmp/hud-coding/123/workspace"]) +async def test_the_shell_user_reads_and_writes_the_guest_workspace( + tmp_path: Path, guest_path: str +) -> None: + root = tmp_path / "root" + root.mkdir() + (root / "input.txt").write_text("hello\n") + command = f"id -u; cat {guest_path}/input.txt; echo written > {guest_path}/output.txt" + async with shell( + root, guest_path=guest_path, network=False, shell_uid=1000, require_isolation=True + ) as (workspace, client): + session = await client.run(command, check=False) + await workspace.terminate_sessions() + captured = await workspace.run(["sh", "-c", command], scope="environment") + + assert (session.exit_status, session.stdout) == (0, "1000\nhello\n") + assert (captured.returncode, captured.stdout) == (0, b"1000\nhello\n") + assert (root / "output.txt").read_text() == "written\n" + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_concurrent_sessions_share_one_sandbox(tmp_path: Path) -> None: + """An agent issues parallel tool calls; if each started its own sandbox, + what one backgrounds would be invisible to the next.""" + async with shell( + tmp_path / "root", guest_path="/app", network=True, require_isolation=True + ) as (_, client): + seen = await asyncio.gather( + *(client.run("readlink /proc/self/ns/pid; pwd", check=True) for _ in range(4)) + ) + started = await client.run("sleep 300 >/dev/null 2>&1 & echo $!", check=True) + visible = await client.run(f"kill -0 {str(started.stdout).strip()}", check=False) + + assert len({result.stdout for result in seen}) == 1 + assert str(seen[0].stdout).splitlines()[1] == "/app" + assert visible.exit_status == 0 + + +@pytest.mark.e2e +@pytest.mark.sandbox +@pytest.mark.parametrize( + ("allowed_hosts", "network", "owns"), + [ + pytest.param({"example.com"}, True, True, id="a-policy-needs-a-network-to-apply-to"), + pytest.param(set(), True, True, id="declared-unreachable"), + pytest.param(None, False, True, id="no-network"), + pytest.param(None, True, False, id="the-substrates-network"), + ], +) +async def test_sessions_get_a_network_of_their_own_exactly_when_the_task_bounds_it( + tmp_path: Path, allowed_hosts: set[str] | None, network: bool, owns: bool +) -> None: + async with shell( + tmp_path / "root", allowed_hosts=allowed_hosts, network=network, require_isolation=True + ) as (workspace, client): + session = str((await client.run("readlink /proc/self/ns/net", check=True)).stdout).strip() + + assert workspace.owns_netns is owns + assert (session != namespace("net")) is owns + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_a_verifier_run_joins_the_sandbox_and_an_isolated_one_gets_its_own_network( + tmp_path: Path, +) -> None: + namespaces = "readlink /proc/self/ns/pid; readlink /proc/self/ns/net" + probe = ["sh", "-c", namespaces] + async with shell(tmp_path / "root", require_isolation=True) as (workspace, client): + session = str((await client.run(namespaces, check=True)).stdout) + joined = await workspace.run(probe, identity=None, max_wait=10) + isolated = await workspace.run(probe, isolated=True, identity=None, max_wait=10) + + session_pid, session_net = session.split() + assert joined.stdout.decode().split() == [session_pid, session_net] + assert isolated.returncode == 0 + assert isolated.stdout.decode().split()[1] not in {session_net, namespace("net")} + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_a_verifier_gets_a_disposable_writable_hosts_file(tmp_path: Path) -> None: + async with shell(tmp_path / "root", require_isolation=True) as (workspace, client): + before = (await client.run("cat /etc/hosts", check=True)).stdout + written = await workspace.run( + ["sh", "-c", "echo '127.0.0.1 verifier-added' >> /etc/hosts && cat /etc/hosts"], + identity=None, + writable_hosts=True, + ) + after = (await client.run("cat /etc/hosts", check=True)).stdout + + assert written.returncode == 0 + assert written.stdout.decode() == f"{before}127.0.0.1 verifier-added\n" + assert after == before + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_ending_the_sessions_keeps_environment_processes_and_the_sandbox() -> None: + # Somewhere another identity can traverse: the launch runs bwrap as uid 1000. + parent = Path(tempfile.mkdtemp(dir="/tmp")) + os.chmod(parent, 0o755) + try: + async with shell(parent / "root", network=True, require_isolation=True) as ( + workspace, + client, + ): + infrastructure = await workspace.launch( + ["sh", "-c", "id -u; id -g; exec sleep 300"], + identity=(1000, 2000), + no_new_privs=False, + persistent=True, + scope="environment", + ) + ids = [await infrastructure.stdout.readline() for _ in range(2)] + session = await client.conn.create_process("sleep 300") + sandbox = (await client.run("readlink /proc/self/ns/pid", check=True)).stdout + + await workspace.terminate_sessions() + + await asyncio.wait_for(session.wait_closed(), 5) + after = (await client.run("readlink /proc/self/ns/pid", check=True)).stdout + still_running = infrastructure.returncode is None + await infrastructure.terminate() + finally: + shutil.rmtree(parent, ignore_errors=True) + + assert ids == [b"1000\n", b"2000\n"] + assert still_running + assert after == sandbox + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_the_sandbox_keeps_every_id_the_container_has(tmp_path: Path) -> None: + uid_map = await asyncio.to_thread(Path("/proc/self/uid_map").read_text) + ranges = [line.split() for line in uid_map.splitlines()] + async with shell(tmp_path / "root", require_isolation=True) as (_, client): + mapped = str((await client.run("cat /proc/self/uid_map", check=True)).stdout) + + assert [line.split() for line in mapped.splitlines()] == [ + [inside, inside, count] for inside, _, count in ranges + ] + + +@pytest.mark.e2e +@pytest.mark.sandbox +@pytest.mark.parametrize( + ("options", "present", "absent"), + [ + pytest.param( + { + "peers": [Peer("db", 5432)], + "local_aliases": ["main"], + "ports": [5432], + "allowed_hosts": {"pypi.org"}, + "shell_uid": 1000, + }, + ["127.0.0.1\tmain", "127.0.0.2\tdb", f"127.0.0.1\t{socket.gethostname()}"], + [], + id="peers-and-aliases-resolve-for-any-identity", + ), + pytest.param({"allowed_hosts": set()}, ["localhost"], [], id="localhost-without-peers"), + pytest.param( + {"peers": [Peer("db", 5432)], "network": True}, + [], + ["\tdb"], + id="the-substrates-hosts-where-the-network-is-shared", + ), + ], +) +async def test_a_declared_peer_is_a_name_sessions_resolve( + tmp_path: Path, options: dict[str, Any], present: list[str], absent: list[str] +) -> None: + async with shell(tmp_path / "root", require_isolation=True, **options) as (_, client): + hosts = str((await client.run("cat /etc/hosts", check=True)).stdout) + + assert [line for line in present if line not in hosts] == [] + assert [line for line in absent if line in hosts] == [] + + +FAKE_BWRAP = """#!/bin/sh +for arg in "$@"; do + case "$arg" in {refused}) exit 1 ;; esac +done +{succeed} +""" + + +@pytest.mark.parametrize( + ("refused", "with_unshare", "expected"), + [ + pytest.param("*", False, None, id="unusable-install"), + pytest.param("--never", False, {"tmpfs_modes": True}, id="direct"), + pytest.param( + "--unshare-pid", + True, + {"pid_unshare": "unshare", "tmpfs_modes": True}, + id="staged-when-direct-pid-mounting-is-blocked", + ), + pytest.param("--perms", False, {"tmpfs_modes": False}, id="without-tmpfs-modes"), + ], +) +def test_bubblewrap_is_used_only_in_a_launch_mode_that_works( + tmp_path: Path, refused: str, with_unshare: bool, expected: dict[str, Any] | None +) -> None: + """An installed bwrap that cannot create namespaces must not be used.""" + tools = tmp_path / "bin" + tools.mkdir() + for name, script in { + "bwrap": FAKE_BWRAP.format(refused=refused, succeed="exit 0"), + "true": "#!/bin/sh\nexit 0\n", + **({"unshare": '#!/bin/sh\nshift 3\nexec "$@"\n'} if with_unshare else {}), + }.items(): + (tools / name).write_text(script) + (tools / name).chmod(0o755) + + probe = subprocess.run( + [ + sys.executable, + "-c", + "from hud.environment.workspace import usable_bwrap; print(repr(usable_bwrap()))", + ], + env={**os.environ, "PATH": str(tools)}, + capture_output=True, + text=True, + check=True, + ) + + if expected is None: + assert probe.stdout.strip() == "None" + else: + unshare = expected.get("pid_unshare") + assert probe.stdout.strip() == repr( + Bubblewrap( + str(tools / "bwrap"), + pid_unshare=str(tools / unshare) if unshare else None, + tmpfs_modes=expected["tmpfs_modes"], + ) + ) diff --git a/tests/environment/workspace/test_session_environment.py b/tests/environment/workspace/test_session_environment.py new file mode 100644 index 000000000..9f20006f9 --- /dev/null +++ b/tests/environment/workspace/test_session_environment.py @@ -0,0 +1,156 @@ +"""What environment a process the workspace starts sees. + +HUD's own variables in the serving process are a credential leak where they hold +a key, and a tell everywhere else: an agent that finds ``HUD_`` anything knows +what is running it. So no entry point hands them on. What the task declares is +the task's, HUD-shaped name or not, and the image's ``PATH`` (the interpreters +and tools the task installed) reaches every process. +""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Any, cast + +import pytest + +from hud.capabilities import Connection, SSHClient +from hud.clients import connect +from hud.environment import Environment +from hud.environment.process_guard import process_connections_supported +from hud.eval import LocalRuntime, Task + +from .conftest import SANDBOX, shell + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + from pathlib import Path + + from tests.support import HudEnv + +TASK_ENV = {"WORKSPACE_ENV": "present", "HUD_TASK_DECLARED": "mine"} + +# What the workspace handed the session's shell, read before a login profile +# rewrites it where /proc shows it (Debian's /etc/profile resets PATH). +SHELL_ENV = "if [ -r /proc/$$/environ ]; then tr '\\0' '\\n' < /proc/$$/environ; else env; fi" + + +def parsed(output: str | bytes) -> dict[str, str]: + text = output.decode() if isinstance(output, bytes) else output + return dict(line.split("=", 1) for line in text.splitlines() if "=" in line) + + +async def ssh_session(root: Path, **options: Any) -> dict[str, str]: + async with shell(root, env=TASK_ENV, **options) as (_, client): + return parsed((await client.run(SHELL_ENV, check=True)).stdout or "") + + +async def unsandboxed_session(root: Path) -> dict[str, str]: + return await ssh_session(root) + + +async def sandboxed_session(root: Path) -> dict[str, str]: + return await ssh_session(root, require_isolation=True) + + +async def dropped_session(root: Path) -> dict[str, str]: + return await ssh_session(root, require_isolation=True, shell_uid=1000) + + +async def bound_connection_session(root: Path) -> dict[str, str]: + if not process_connections_supported(): + pytest.skip("process connection guards are unavailable") + connection = Connection( + name="inference", + capability="ssh", + url="http://127.0.0.1:9", + headers={"Authorization": "Bearer scoped-runtime-token"}, + ) + env = Environment("bound") + env.workspace( + root, env=TASK_ENV, allowed_hosts={"pypi.org"}, require_isolation=True, track_files=False + ) + async with ( + LocalRuntime(env)(Task(env=env.name, id="bound")) as runtime, + connect(runtime, connections=(connection,)) as client, + ): + ssh = cast("SSHClient", await client.open("shell")) + process = await ssh.create_process(SHELL_ENV, connections=(connection,)) + return parsed((await process.wait(check=True)).stdout or b"") + + +async def verifier_run(root: Path) -> dict[str, str]: + async with shell(root, env=TASK_ENV, require_isolation=True) as (workspace, _): + result = await workspace.run( + ["env"], + env={"VERIFIER_ONLY": "yes"}, + identity=None, + inherit_workspace_env=False, + allowed_hosts={"pypi.org"}, + ) + return parsed(result.stdout) + + +async def isolated_run(root: Path) -> dict[str, str]: + async with shell(root, env=TASK_ENV, require_isolation=True) as (workspace, _): + result = await workspace.run(["env"], isolated=True, identity=None) + return parsed(result.stdout) + + +@pytest.mark.parametrize( + ("entry", "expected"), + [ + pytest.param( + unsandboxed_session, {**TASK_ENV, "ORDINARY": "kept"}, id="ssh-session-unsandboxed" + ), + pytest.param( + sandboxed_session, + {**TASK_ENV, "ORDINARY": "kept"}, + id="ssh-session-sandboxed", + marks=SANDBOX, + ), + pytest.param( + dropped_session, + {**TASK_ENV, "ORDINARY": None, "HOME": "/workspace"}, + id="ssh-session-dropped-to-shell-uid", + marks=SANDBOX, + ), + pytest.param( + bound_connection_session, + {**TASK_ENV, "ORDINARY": "kept", "http_proxy": "http://127.0.0.1:3128"}, + id="ssh-session-with-a-bound-connection", + marks=SANDBOX, + ), + pytest.param( + verifier_run, + { + **dict.fromkeys(TASK_ENV), + "ORDINARY": "kept", + "VERIFIER_ONLY": "yes", + }, + id="verifier-run-without-workspace-env", + marks=SANDBOX, + ), + pytest.param( + isolated_run, {**TASK_ENV, "ORDINARY": "kept"}, id="isolated-run", marks=SANDBOX + ), + ], +) +async def test_hud_configuration_never_reaches_a_process_but_the_task_env_and_path_do( + tmp_path: Path, + hud_env: HudEnv, + entry: Callable[[Path], Awaitable[dict[str, str]]], + expected: dict[str, str | None], +) -> None: + hud_env.set( + HUD_API_KEY="super-secret", + ORDINARY="kept", + PATH=f"/task/bin{os.pathsep}{os.environ['PATH']}", + ) + + seen = await entry(tmp_path / "root") + + assert not [name for name in seen if name.startswith("HUD_") and name not in TASK_ENV] + assert "super-secret" not in seen.values() + assert "/task/bin" in seen["PATH"].split(os.pathsep) + assert {name: seen.get(name) for name in expected} == expected diff --git a/tests/environment/workspace/test_shell_io.py b/tests/environment/workspace/test_shell_io.py new file mode 100644 index 000000000..5da5911b4 --- /dev/null +++ b/tests/environment/workspace/test_shell_io.py @@ -0,0 +1,387 @@ +"""What an agent's shell session carries over the SSH wire, and what survives it. + +Every scenario connects to a served workspace's ``ssh/2`` binding. The +``backing`` axis runs the same scenario on whatever the host gives (sessions run +directly where bubblewrap is unavailable) and in the sandbox lane, where each +session is spawned by the sandbox's namespace host. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import shlex +import shutil +import signal +import subprocess +import sys +import time +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +import asyncssh +import pytest + +from hud.capabilities.ssh import PROCESS_CONNECTIONS_REQUEST +from hud.environment.workspace import usable_bwrap +from tests.support import eventually + +from .conftest import SANDBOX, shell + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + from pathlib import Path + + from hud.capabilities import SSHClient + +BACKINGS = [ + pytest.param({}, id="host"), + pytest.param({"require_isolation": True}, id="sandbox", marks=SANDBOX), +] + +UNSANDBOXED = pytest.mark.skipif( + usable_bwrap() is not None, + reason="a sandboxed command keeps its process group until the sandbox is discarded", +) + + +def alive(pid: int) -> bool: + """Whether ``pid`` is a live process on this host (an unreaped zombie counts as gone).""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + status = subprocess.run( + ["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True, check=False + ).stdout.strip() + return bool(status) and not status.startswith("Z") + + +async def test_output_arrives_while_the_command_is_still_running(tmp_path: Path) -> None: + """Held until exit, a long build tells the agent nothing while it runs and + a session that never exits says nothing at all.""" + async with shell(tmp_path / "root") as (_, client): + started = time.monotonic() + process = await client.conn.create_process("echo first; sleep 5; echo second") + first = await asyncio.wait_for(process.stdout.readline(), 10) + elapsed = time.monotonic() - started + process.channel.close() + + assert first.strip() == "first" + assert elapsed < 2.0, f"first line took {elapsed:.1f}s — output is not streaming" + + +PAYLOAD = b"0123456789abcdef" * 131072 + b"tail" +STREAMS = [ + pytest.param( + "IFS= read -r line; printf 'out:%s' \"$line\"; printf err >&2", + b"input\n", + (b"out:input", b"err", 0), + id="stdin-reaches-the-command-and-both-streams-come-back", + ), + pytest.param( + "cat; printf stderr-tail >&2", + PAYLOAD, + (PAYLOAD, b"stderr-tail", 0), + id="large-streams-and-the-status-survive", + ), + pytest.param( + "printf done", + b"x" * 500_000, + (b"done", b"", 0), + id="large-input-to-an-early-exit-does-not-hang", + ), + pytest.param( + "head -c 8388608 /dev/zero; head -c 8388608 /dev/zero >&2; exit 7", + b"", + (bytes(8388608), bytes(8388608), 7), + id="output-beyond-every-buffer-drains-before-the-status", + ), +] + + +@pytest.mark.parametrize("backing", BACKINGS) +@pytest.mark.parametrize(("command", "stdin", "expected"), STREAMS) +async def test_a_session_carries_its_streams_and_exit_status( + tmp_path: Path, + backing: dict[str, Any], + command: str, + stdin: bytes, + expected: tuple[bytes, bytes, int], +) -> None: + async with shell(tmp_path / "root", **backing) as (_, client): + async with asyncio.timeout(30): + result = await client.conn.run(command, input=stdin, encoding=None) + + assert (result.stdout, result.stderr, result.exit_status) == expected + + +@pytest.mark.parametrize("backing", BACKINGS) +async def test_a_session_that_asks_for_a_terminal_gets_one( + tmp_path: Path, backing: dict[str, Any] +) -> None: + """Programs branch on isatty: without a pty they take their batch path, so + a terminal task is graded on behaviour a terminal would never produce.""" + async with shell(tmp_path / "root", **backing) as (_, client): + with_pty = await client.conn.run( + "test -t 0 && test -t 1 && echo TTY || echo NOT_TTY; stty size", + term_type="xterm-256color", + term_size=(120, 40), + check=False, + ) + without = await client.conn.run("test -t 1 && echo TTY || echo NOT_TTY", check=False) + + assert "TTY" in str(with_pty.stdout) and "NOT_TTY" not in str(with_pty.stdout) + assert "40 120" in str(with_pty.stdout) + assert "NOT_TTY" in str(without.stdout) + + +@pytest.mark.parametrize("backing", BACKINGS) +async def test_a_resize_does_not_cost_the_session_its_keyboard( + tmp_path: Path, backing: dict[str, Any] +) -> None: + """asyncssh delivers a resize as an exception on the stdin read, and it is + not an asyncssh.Error — unhandled it escapes the relay and input stops.""" + async with shell(tmp_path / "root", **backing) as (_, client): + process = await client.conn.create_process( + "cat", term_type="xterm-256color", term_size=(80, 24) + ) + process.channel.change_terminal_size(132, 43) + await asyncio.sleep(0.2) + process.stdin.write("still-listening\n") + echoed = await asyncio.wait_for(process.stdout.readline(), 5) + process.channel.close() + + assert "still-listening" in echoed + + +@pytest.mark.parametrize("backing", BACKINGS) +async def test_terminal_output_survives_a_client_that_reads_slowly( + tmp_path: Path, backing: dict[str, Any] +) -> None: + async with shell(tmp_path / "root", **backing) as (_, client): + binding = client.capability + address = urlsplit(binding.url) + assert address.hostname is not None and address.port is not None + async with asyncssh.connect( + address.hostname, + address.port, + username=binding.params["user"], + client_keys=[asyncssh.import_private_key(binding.params["client_key"])], + known_hosts=None, + window=32768, + ) as conn: + result = await asyncio.wait_for( + conn.run( + "head -c 500000 /dev/zero | tr '\\000' x", + term_type="xterm", + encoding=None, + ), + 30, + ) + + assert (result.exit_status, result.stdout) == (0, b"x" * 500000) + + +async def test_a_session_that_cannot_be_prepared_is_told_why(tmp_path: Path) -> None: + async with shell(tmp_path / "root") as (_, client): + result = await client.conn.run( + "echo unreachable", env={PROCESS_CONNECTIONS_REQUEST: '["missing"]'}, check=False + ) + + assert result.exit_status == 1 + assert "workspace: cannot prepare shell:" in str(result.stderr) + assert "unreachable" not in str(result.stdout) + + +@pytest.mark.e2e +@pytest.mark.sandbox +@pytest.mark.parametrize( + ("command", "marker"), + [ + pytest.param( + "sh -c 'sleep .1; exec >/dev/null 2>&1; : > silent-done' & printf started", + "silent-done", + id="inherited-eof", + ), + pytest.param( + "sh -c \"sleep .1; trap '' PIPE; printf late-out || :; " + "printf late-err >&2 || :; exec >/dev/null 2>&1; " + ': > late-done" & printf started', + "late-done", + id="late-output", + ), + pytest.param( + "cd . && nohup sh -c 'sleep .1' stdin.log 2>&1 && " + "exec >/dev/null 2>&1 && : > stdin-done & printf started", + "stdin-done", + id="stdin-detached", + ), + pytest.param( + "(cd . && exec sh -c 'sleep .1; : > redirected-done') " + "redirected.log 2>&1 & printf started", + "redirected-done", + id="fully-redirected", + ), + ], +) +async def test_a_backgrounded_job_finishing_leaves_the_next_sandboxed_command_working( + tmp_path: Path, command: str, marker: str +) -> None: + root = tmp_path / "root" + async with shell(root, require_isolation=True) as (_, client): + launched = await asyncio.wait_for(client.conn.run(f"bash -c {shlex.quote(command)}"), 10) + await eventually(lambda: (root / marker).exists()) + probe = await asyncio.wait_for(client.conn.run("printf healthy"), 10) + + assert (launched.exit_status, launched.stdout) == (0, "started") + assert (probe.exit_status, probe.stdout) == (0, "healthy") + + +async def output_split_mid_character_is_replaced(client: SSHClient, root: Path) -> None: + del root + completed = await client.run("printf '\\226\\214 split-utf8'", check=True) + assert completed.stdout == "�� split-utf8" + + +async def a_channel_closed_while_its_command_is_torn_down(client: SSHClient, root: Path) -> None: + # The pid file doubles as the child's readiness signal: it is written after + # the TERM trap is installed, and teardown's TERM only fires once the + # foreground command exits. Exiting before the child is ready would kill it + # while still untrapped. + remote = await client.conn.create_process( + "sh -c 'trap \"echo started > teardown-started; " + 'while :; do sleep 1; done" TERM; ' + "echo $$ > close-race-child.pid; while :; do sleep 1; done' & " + "while [ ! -s close-race-child.pid ]; do sleep 0.01; done; " + "printf output" + ) + pid = 0 + try: + await eventually(lambda: (root / "teardown-started").exists()) + pid = int((root / "close-race-child.pid").read_text()) + remote.close() + await asyncio.wait_for(remote.wait_closed(), 5.0) + await eventually(lambda: not alive(pid), within=5.0) + finally: + remote.close() + if pid and alive(pid): + os.kill(pid, signal.SIGKILL) + + +@pytest.mark.parametrize( + "fault", + [ + pytest.param(output_split_mid_character_is_replaced, id="invalid-utf8-output"), + pytest.param( + a_channel_closed_while_its_command_is_torn_down, + id="channel-closed-during-teardown", + marks=UNSANDBOXED, + ), + ], +) +async def test_the_connection_survives_a_command_that_goes_wrong( + tmp_path: Path, fault: Callable[[SSHClient, Path], Awaitable[None]] +) -> None: + root = tmp_path / "root" + async with shell(root) as (_, client): + connection = client.conn + await fault(client, root) + after = await client.run("printf healthy", check=True, timeout=5.0) + + assert after.stdout == "healthy" + assert client.conn is connection + + +async def a_completed_command_leaves_a_background_child(client: SSHClient, root: Path) -> int: + await client.conn.run("sleep 120 >/dev/null 2>&1 < /dev/null & echo $! > child.pid", check=True) + return int((root / "child.pid").read_text()) + + +async def a_session_closed_while_its_child_ignores_term(client: SSHClient, root: Path) -> int: + remote = await client.conn.create_process("trap '' TERM; sleep 120 & echo $! > child.pid; wait") + try: + await eventually( + lambda: bool((root / "child.pid").exists() and (root / "child.pid").read_text().strip()) + ) + pid = int((root / "child.pid").read_text()) + assert alive(pid) + finally: + remote.close() + await asyncio.wait_for(remote.wait_closed(), 5.0) + return pid + + +@UNSANDBOXED +@pytest.mark.parametrize( + "ending", + [ + pytest.param(a_completed_command_leaves_a_background_child, id="command-completed"), + pytest.param(a_session_closed_while_its_child_ignores_term, id="session-closed"), + ], +) +async def test_the_end_of_an_unsandboxed_command_kills_its_descendants( + tmp_path: Path, ending: Callable[[SSHClient, Path], Awaitable[int]] +) -> None: + root = tmp_path / "root" + pid = 0 + try: + async with shell(root) as (_, client): + pid = await ending(client, root) + await eventually(lambda: not alive(pid), within=5.0) + finally: + if pid and alive(pid): + os.kill(pid, signal.SIGKILL) + + +@UNSANDBOXED +async def test_a_command_returns_although_a_detached_descendant_holds_its_output( + tmp_path: Path, +) -> None: + root = tmp_path / "root" + setsid = shutil.which("setsid") + if setsid is not None: + detached_command = ( + f"{shlex.quote(setsid)} sh -c " + "'echo $$ > detached.pid; printf ready; kill -KILL $PPID; sleep 120'" + ) + else: + code = ( + "import os,signal,time;" + "from pathlib import Path;" + "parent=os.getppid();" + "os.setsid();" + "Path('detached.pid').write_text(str(os.getpid()));" + "os.write(1,b'ready');" + "os.kill(parent,signal.SIGKILL);" + "time.sleep(120)" + ) + detached_command = f"{shlex.quote(sys.executable)} -c {shlex.quote(code)}" + + pid = 0 + try: + async with shell(root) as (_, client): + remote = await client.conn.create_process(f"{detached_command} & wait") + try: + await eventually( + lambda: bool( + (root / "detached.pid").exists() + and (root / "detached.pid").read_text().strip() + ) + ) + pid = int((root / "detached.pid").read_text()) + completed = await remote.wait(timeout=5.0) + finally: + remote.close() + await asyncio.wait_for(remote.wait_closed(), 5.0) + survived = alive(pid) + finally: + if pid and alive(pid): + with contextlib.suppress(ProcessLookupError): + os.killpg(pid, signal.SIGKILL) + + assert completed.stdout == "ready" + assert survived diff --git a/hud/eval/tests/__init__.py b/tests/eval/__init__.py similarity index 100% rename from hud/eval/tests/__init__.py rename to tests/eval/__init__.py diff --git a/tests/eval/test_chat.py b/tests/eval/test_chat.py new file mode 100644 index 000000000..18445a477 --- /dev/null +++ b/tests/eval/test_chat.py @@ -0,0 +1,94 @@ +"""``Chat``: a conversation folded over a chat-style task, one rollout per turn.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest +from mcp.types import ImageContent, TextContent + +from hud import Environment +from hud.agents.base import Agent +from hud.agents.types import AgentStep, Citation +from hud.eval import Chat, LocalRuntime, Task + +if TYPE_CHECKING: + from hud.eval import Run + + +def assistant(prompts: list[Any]) -> Environment: + """A chat-style environment: its prompt is the whole conversation so far.""" + env = Environment("chat") + + @env.template() + async def reply(messages: list[dict[str, Any]]): + prompts.append(messages) + yield messages + yield 1.0 + + return env + + +class Echo(Agent): + """Replies ``echo:`` and the last turn's text, and fails on ``fail``.""" + + async def __call__(self, run: Run) -> None: + last = run.prompt_messages[-1].content + text = last.text if isinstance(last, TextContent) else "" + if text == "fail": + raise RuntimeError("model unavailable") + run.trace.content = f"echo:{text}" + + +async def test_each_turn_runs_on_the_whole_conversation_and_adds_the_reply_to_it() -> None: + prompts: list[Any] = [] + chat = Chat(Task(env="chat", id="reply"), Echo(), runtime=LocalRuntime(assistant(prompts))) + picture = [ + TextContent(type="text", text="what is this?"), + ImageContent(type="image", data="aW1n", mimeType="image/png"), + ] + + first = await chat.send("hello") + second = await chat.send(picture) + + assert (first.content, second.content) == ("echo:hello", "echo:") + assert [message["role"] for message in chat.messages] == ["user", "assistant"] * 2 + texts = [m["content"]["text"] for m in chat.messages if isinstance(m["content"], dict)] + assert texts == ["hello", "echo:hello", "echo:"] + assert [block["type"] for block in chat.messages[2]["content"]] == ["text", "image"] + assert prompts == [chat.messages[:1], chat.messages[:3]] + + +async def test_every_turn_reports_under_the_conversations_one_job() -> None: + chat = Chat(Task(env="chat", id="reply"), Echo(), runtime=LocalRuntime(assistant([]))) + + await chat.send("hello") + await chat.send("again") + + assert chat.job is not None + assert (chat.job.name, [run.job_id for run in chat.job.runs]) == ("reply", [chat.job.id] * 2) + + +async def test_a_reply_keeps_the_citations_its_agent_gave() -> None: + class Cites(Agent): + async def __call__(self, run: Run) -> None: + citation = Citation(type="url_citation", text="docs", source="https://docs.test") + run.record(AgentStep(content="see the docs", citations=[citation])) + run.trace.content = "see the docs" + + chat = Chat(Task(env="chat", id="reply"), Cites(), runtime=LocalRuntime(assistant([]))) + + await chat.send("where?") + + assert chat.messages[-1]["citations"] == [ + {"type": "url_citation", "text": "docs", "source": "https://docs.test"} + ] + + +async def test_a_failed_turn_raises_and_leaves_no_reply_in_the_conversation() -> None: + chat = Chat(Task(env="chat", id="reply"), Echo(), runtime=LocalRuntime(assistant([]))) + + with pytest.raises(RuntimeError, match="model unavailable"): + await chat.send("fail") + + assert [message["role"] for message in chat.messages] == ["user"] diff --git a/tests/eval/test_docker.py b/tests/eval/test_docker.py new file mode 100644 index 000000000..42c1f63c4 --- /dev/null +++ b/tests/eval/test_docker.py @@ -0,0 +1,256 @@ +"""DockerRuntime: a row runs in a confined container or Compose project, removed after. + +The scenarios run real containers from the fixture image (this checkout's SDK +serving a fixture environment) on the local Docker daemon, and check what the +daemon reports about them. What every runtime does (outcomes, deadlines, cancel, +cleanup, session files for a verifier, refusals) runs on Docker as a row of the +rollout tables; this module holds what only Docker does. +""" + +from __future__ import annotations + +import io +import json +import subprocess +import tarfile +import textwrap +import uuid +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.eval import ( + ComposeProject, + DockerRuntime, + RuntimeConfig, + RuntimeLimits, + RuntimeResources, + Task, + rollout, +) +from hud.eval.runtime import DockerBindMount +from hud.eval.runtime.docker import DockerEndpoint +from tests.fixtures.agents import ScriptedAgent +from tests.support import eventually + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from tests.support import HudEnv + +ECHO = Task(env="echo", id="echo", args={"word": "tangerine"}) + + +def inspect(container: str) -> dict[str, Any]: + out = subprocess.run( + ["docker", "inspect", container], capture_output=True, text=True, check=True + ).stdout + return json.loads(out)[0] + + +def containers(**labels: str) -> list[str]: + filters = [arg for key, value in labels.items() for arg in ("--filter", f"label={key}={value}")] + out = subprocess.run( + ["docker", "ps", "--all", "--quiet", *filters], capture_output=True, text=True, check=True + ).stdout + return out.split() + + +def volume_exists(name: str) -> bool: + probe = subprocess.run(["docker", "volume", "inspect", name], capture_output=True, check=False) + return probe.returncode == 0 + + +def labelled() -> tuple[str, list[str]]: + """A label unique to this test and the ``docker run`` flags that set it.""" + label = uuid.uuid4().hex + return label, ["--label", f"hud-test={label}"] + + +@pytest.mark.e2e +@pytest.mark.docker +async def test_an_image_row_runs_in_a_confined_container_that_is_removed_after( + fixture_image: Callable[[str], str], +) -> None: + label, flags = labelled() + + async with DockerRuntime(fixture_image("echo"), run_args=flags)(ECHO) as runtime: + assert isinstance(runtime, DockerEndpoint) + held = inspect(runtime.container) + run = await rollout(ECHO, ScriptedAgent("tangerine"), runtime=runtime) + + sessions = [mount["Name"] for mount in held["Mounts"] if mount["Type"] == "volume"] + assert run.reward == 1.0 + assert any(option.startswith("seccomp=") for option in held["HostConfig"]["SecurityOpt"]) + assert held["HostConfig"]["Privileged"] is False + assert containers(**{"hud-test": label}) == [] + assert sessions and not any(volume_exists(name) for name in sessions) + + +@pytest.mark.e2e +@pytest.mark.docker +async def test_the_rows_settings_win_over_the_providers_and_reach_the_container( + fixture_image: Callable[[str], str], tmp_path: Path +) -> None: + (tmp_path / "data").mkdir() + _, flags = labelled() + provider = DockerRuntime( + "hud-test-fixture:missing", + run_args=flags, + env_vars={"MODE": "eval"}, + bind_mounts=[DockerBindMount(str(tmp_path / "data"), "/data")], + runtime_config=RuntimeConfig(resources=RuntimeResources(cpu=2, memory_mb=1024)), + ) + row = ECHO.model_copy( + update={ + "runtime_config": RuntimeConfig( + image=fixture_image("echo"), resources=RuntimeResources(cpu=1, memory_mb=512) + ) + } + ) + + async with provider(row) as runtime: + assert isinstance(runtime, DockerEndpoint) + held = inspect(runtime.container) + run = await rollout(row, ScriptedAgent("tangerine"), runtime=runtime) + + (data,) = [mount for mount in held["Mounts"] if mount["Destination"] == "/data"] + assert run.reward == 1.0 + assert (held["HostConfig"]["NanoCpus"], held["HostConfig"]["Memory"]) == (10**9, 512 * 2**20) + assert "MODE=eval" in held["Config"]["Env"] + assert data["RW"] is False + + +@pytest.mark.e2e +@pytest.mark.docker +async def test_a_row_that_names_an_image_runs_there_without_a_runtime( + fixture_image: Callable[[str], str], +) -> None: + row = ECHO.model_copy(update={"runtime_config": RuntimeConfig(image=fixture_image("echo"))}) + + job = await row.run(ScriptedAgent("tangerine")) + + assert job.reward == 1.0 + + +@pytest.mark.e2e +@pytest.mark.docker +async def test_more_disk_than_the_container_has_is_refused_and_the_container_removed( + fixture_image: Callable[[str], str], +) -> None: + label, flags = labelled() + row = ECHO.model_copy( + update={"runtime_config": RuntimeConfig(resources=RuntimeResources(storage_mb=10**9))} + ) + + with pytest.raises(RuntimeError, match=r"requires 1000000000 MB of free disk"): + async with DockerRuntime(fixture_image("echo"), run_args=flags)(row): + pytest.fail("the request should be refused") + + assert containers(**{"hud-test": label}) == [] + + +@pytest.mark.e2e +@pytest.mark.docker +async def test_a_container_past_its_run_limit_is_removed_while_held( + fixture_image: Callable[[str], str], +) -> None: + label, flags = labelled() + row = ECHO.model_copy( + update={"runtime_config": RuntimeConfig(limits=RuntimeLimits(run_timeout_s=2))} + ) + + async with DockerRuntime(fixture_image("echo"), run_args=flags)(row): + await eventually(lambda: containers(**{"hud-test": label}) == []) + + +@pytest.mark.e2e +@pytest.mark.docker +async def test_a_restored_session_archive_may_hold_only_files_and_directories( + fixture_image: Callable[[str], str], tmp_path: Path +) -> None: + archive = tmp_path / "session.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + link = tarfile.TarInfo("escape") + link.type = tarfile.SYMTYPE + link.linkname = "/etc/passwd" + tar.addfile(link, io.BytesIO()) + + async with DockerRuntime(fixture_image("echo"))(ECHO) as runtime: + with pytest.raises( + ValueError, match="runtime session archive contains an unsupported entry" + ): + await runtime.restore_session("sess-12345678", archive) + + +def compose_file(tmp_path: Path, document: str, env_file: str = "") -> ComposeProject: + (tmp_path / ".env").write_text(env_file) + path = tmp_path / "compose.yaml" + path.write_text(textwrap.dedent(document)) + return ComposeProject(document=path) + + +@pytest.mark.e2e +@pytest.mark.docker +async def test_a_compose_project_serves_main_confined_through_its_network_owner( + fixture_image: Callable[[str], str], tmp_path: Path, hud_env: HudEnv +) -> None: + hud_env.set(HOST_ONLY="from-the-host") + project = compose_file( + tmp_path, + f""" + services: + main: + image: {fixture_image("echo")} + network_mode: service:gateway + environment: + MODE: ${{MODE}} + LITERAL: $$HOST_ONLY + depends_on: [gateway] + gateway: + image: python:3.12-slim + command: sleep infinity + healthcheck: {{test: [CMD, "true"], interval: 1s}} + """, + env_file="MODE=eval\n", + ) + row = ECHO.model_copy(update={"runtime_config": RuntimeConfig(compose=project)}) + + async with DockerRuntime()(row) as runtime: + assert isinstance(runtime, DockerEndpoint) + held = inspect(runtime.container) + run = await rollout(row, ScriptedAgent("tangerine"), runtime=runtime) + + compose = held["Config"]["Labels"]["com.docker.compose.project"] + assert run.reward == 1.0 + assert any(option.startswith("seccomp=") for option in held["HostConfig"]["SecurityOpt"]) + assert {"MODE=eval", "LITERAL=$HOST_ONLY"} <= set(held["Config"]["Env"]) + assert containers(**{"com.docker.compose.project": compose}) == [] + + +@pytest.mark.e2e +@pytest.mark.docker +async def test_a_compose_project_whose_network_owner_dies_is_taken_down( + fixture_image: Callable[[str], str], tmp_path: Path +) -> None: + project = compose_file( + tmp_path, + f""" + services: + main: + image: {fixture_image("echo")} + network_mode: service:gateway + gateway: + image: python:3.12-slim + command: python -c "print('gateway gave up'); raise SystemExit(1)" + """, + ) + row = ECHO.model_copy(update={"runtime_config": RuntimeConfig(compose=project)}) + + with pytest.raises(RuntimeError): + async with DockerRuntime()(row): + pytest.fail("the project should not serve") + + working_dir = {"com.docker.compose.project.working_dir": str(tmp_path)} + assert containers(**working_dir) == [] diff --git a/tests/eval/test_hosted.py b/tests/eval/test_hosted.py new file mode 100644 index 000000000..d32f7c670 --- /dev/null +++ b/tests/eval/test_hosted.py @@ -0,0 +1,309 @@ +"""``HostedRuntime``: the platform runs the whole rollout; this process submits it, polls +its trace to a terminal state, and folds the result into a ``Run``. A deadline or a +cancel here cancels the rollout on the platform.""" + +from __future__ import annotations + +import asyncio +import uuid +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.agents import OpenAIChatAgent +from hud.agents.base import Agent +from hud.agents.types import OpenAIChatConfig +from hud.eval import HostedRuntime, Task, Taskset +from hud.types import AgentType +from tests.support import Reply, eventually + +if TYPE_CHECKING: + from pathlib import Path + + from hud.eval import Run + from tests.support import FakeServices, HudEnv + +SUBMIT = "/v2/rollouts/submit" +POLL = "/v2/trace/{id}" +CANCEL = "/v2/rollouts/cancel" + + +@pytest.fixture(autouse=True) +def signed_in(hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="k") + + +def chat_agent() -> OpenAIChatAgent: + return OpenAIChatAgent( + OpenAIChatConfig(model="test-model", api_key="secret", base_url="http://model.test") + ) + + +async def test_a_hosted_rollout_submits_the_row_and_its_agent_then_polls_to_its_end( + services: FakeServices, tmp_path: Path +) -> None: + (tmp_path / "compose.yaml").write_text("services:\n main:\n image: sums:latest\n") + services.route("api", "POST", SUBMIT, json={"status": "queued"}) + services.route( + "api", + "GET", + POLL, + Reply(json={"status": "pending"}), + Reply(json={"status": "running"}), + Reply(json={"status": "completed", "reward": 0.5}), + ) + row = Task.model_validate( + { + "env": "sums", + "id": "add", + "args": {"a": 1, "b": 2}, + "agent_config": {"timeout_seconds": 45.0}, + "runtime_config": {"compose": {"document": "compose.yaml"}}, + "verifier": {"env": "judge", "id": "verify", "runtime_config": {"image": "judge:1"}}, + }, + context={"base_path": tmp_path}, + ) + + job = await Taskset("sums", [row]).run(chat_agent(), runtime=HostedRuntime(poll_interval=0)) + + (run,) = job.runs + (submitted,) = services.requests("api", "POST", SUBMIT) + body = submitted.json + assert (body["trace_id"], body["job_id"], body["group_id"]) == ( + str(uuid.UUID(run.trace_id)), + str(uuid.UUID(job.id)), + run.group_id, + ) + assert (body["env"], body["task"], body["slug"], body["args"]) == ( + "sums", + "add", + row.slug, + {"a": 1, "b": 2}, + ) + assert (body["agent"]["type"], body["agent"]["config"]["timeout_seconds"]) == ( + "openai_compatible", + 45.0, + ) + assert list(body["runtime_config"]) == ["compose"] + assert body["runtime_config"]["compose"]["document"]["services"]["main"]["image"] == ( + "sums:latest" + ) + assert (body["verifier"]["env"], body["verifier"]["runtime_config"]) == ( + "judge", + {"image": "judge:1"}, + ) + assert "secret" not in str(body) + assert (run.reward, run.trace.status, run.runtime) == ( + 0.5, + "completed", + f"hud://trace/{run.trace_id}", + ) + assert len(services.requests("api", "GET", POLL)) == 3 + assert services.requests("api", "POST", "/v2/trace/{id}/exit") == [] + + +@pytest.mark.parametrize( + ("state", "reward", "status", "error", "graded"), + [ + pytest.param( + {"status": "completed", "reward": 0.5, "evaluation_result": {"score": 0.5}}, + 0.5, + "completed", + None, + True, + id="completed-with-its-evaluation", + ), + pytest.param( + {"status": "error", "reward": None, "error": "env exploded"}, + 0.0, + "error", + "env exploded", + False, + id="errored-without-a-reward", + ), + pytest.param( + {"status": "error", "reward": 0.75, "error": "agent exploded"}, + 0.75, + "error", + "agent exploded", + True, + id="errored-but-graded", + ), + pytest.param( + {"status": "cancelled", "reward": None}, + 0.0, + "cancelled", + None, + False, + id="cancelled-before-grading", + ), + pytest.param( + {"status": "completed", "evaluation_result": {"score": "high"}}, + 0.0, + "error", + "hud rpc error -32603: tasks.grade: result must include a numeric 'score'", + False, + id="an-evaluation-without-a-numeric-score", + ), + ], +) +async def test_the_trace_the_platform_ends_with_is_the_runs_outcome( + state: dict[str, Any], + reward: float, + status: str, + error: str | None, + graded: bool, + services: FakeServices, +) -> None: + services.route("api", "POST", SUBMIT, json={"status": "queued"}) + services.route("api", "GET", POLL, json=state) + + job = await Task(env="sums", id="add").run(chat_agent(), runtime=HostedRuntime(poll_interval=0)) + + (run,) = job.runs + assert (run.reward, run.trace.status, run.trace.error, run in job.errors) == ( + reward, + status, + error, + not graded, + ) + + +async def test_a_rejected_submission_is_a_failed_run_that_polls_nothing( + services: FakeServices, +) -> None: + services.route("api", "POST", SUBMIT, status=422, json={"detail": "unknown env 'sums'"}) + + job = await Task(env="sums", id="add").run(chat_agent(), runtime=HostedRuntime(poll_interval=0)) + + assert "unknown env 'sums'" in (job.runs[0].trace.error or "") + assert services.requests("api", "GET") == [] + + +async def test_an_agent_the_platform_cannot_run_fails_without_a_submission( + services: FakeServices, +) -> None: + class Local(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "x" + + job = await Task(env="sums", id="add").run(Local(), runtime=HostedRuntime(poll_interval=0)) + + assert (job.runs[0].trace.error or "").startswith( + "hosted execution supports the registered agent types" + ) + assert services.requests() == [] + + +async def test_hosted_execution_needs_a_hud_api_key( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY=None) + + job = await Task(env="sums", id="add").run(chat_agent(), runtime=HostedRuntime(poll_interval=0)) + + assert job.runs[0].trace.error == "HUD_API_KEY is required" + assert services.requests() == [] + + +@pytest.mark.parametrize( + "submit_delay", [0, 5], ids=["while-the-platform-runs-it", "before-the-platform-accepts-it"] +) +async def test_a_hosted_rollout_past_its_deadline_is_cancelled_on_the_platform( + submit_delay: float, services: FakeServices +) -> None: + services.route("api", "POST", SUBMIT, Reply(json={"status": "queued"}, delay=submit_delay)) + services.route("api", "GET", POLL, json={"status": "running"}) + services.route("api", "POST", CANCEL, json={"status": "accepted"}) + + job = await Task(env="sums", id="add").run( + chat_agent(), runtime=HostedRuntime(poll_interval=0), rollout_timeout=1 + ) + + (run,) = job.runs + assert (run.trace.status, run.trace.stop_reason, run.trace.error) == ( + "error", + "timeout", + f"hosted rollout {run.trace_id} did not finish within 1s", + ) + await eventually(lambda: len(services.requests("api", "POST", CANCEL)) == 1, within=10) + assert services.bodies("api", "POST", CANCEL) == [{"trace_id": str(uuid.UUID(run.trace_id))}] + + +async def test_the_deprecated_run_timeout_is_the_hosted_rollouts_deadline( + services: FakeServices, +) -> None: + services.route("api", "POST", SUBMIT, json={"status": "queued"}) + services.route("api", "GET", POLL, json={"status": "running"}) + services.route("api", "POST", CANCEL, json={"status": "accepted"}) + with pytest.warns(DeprecationWarning, match="rollout_timeout=... to Task.run"): + runtime = HostedRuntime(poll_interval=0, run_timeout=1) + + job = await Task(env="sums", id="add").run(chat_agent(), runtime=runtime) + + assert job.runs[0].trace.stop_reason == "timeout" + await eventually(lambda: len(services.requests("api", "POST", CANCEL)) == 1, within=10) + + +async def test_cancelling_a_hosted_rollout_cancels_it_on_the_platform( + services: FakeServices, +) -> None: + services.route("api", "POST", SUBMIT, json={"status": "queued"}) + services.route("api", "GET", POLL, json={"status": "running"}) + services.route("api", "POST", CANCEL, json={"status": "accepted"}) + + pending = asyncio.create_task( + Task(env="sums", id="add").run(chat_agent(), runtime=HostedRuntime(poll_interval=0)) + ) + await eventually(lambda: services.requests("api", "POST", SUBMIT) != []) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + + await eventually(lambda: len(services.requests("api", "POST", CANCEL)) == 1, within=10) + + +async def test_a_hosted_row_whose_timeouts_contradict_raises_before_submitting( + services: FakeServices, +) -> None: + row = Task.model_validate( + { + "env": "sums", + "id": "add", + "agent_config": {"timeout_seconds": 600}, + "runtime_config": {"limits": {"run_timeout_s": 600}}, + } + ) + + with pytest.raises( + ValueError, match=r"must be less than runtime_config\.limits\.run_timeout_s" + ): + await row.run(chat_agent(), runtime=HostedRuntime(poll_interval=0)) + + assert services.requests() == [] + + +@pytest.mark.parametrize( + ("agent_type", "config"), + [ + ("claude", {"model": "claude-sonnet-4-6", "max_steps": 5, "tool_timeout_seconds": 30}), + ("openai", {"model": "gpt-5.6", "prompt_cache_key": "shared"}), + ("gemini", {"model": "gemini-m", "max_output_tokens": 512}), + ("openai_compatible", {"model": "m", "completion_kwargs": {"temperature": 0.2}}), + ("claude_cli", {"reasoning_effort": "max", "max_steps": 3}), + ("codex_cli", {"sandbox": "read-only", "reasoning_effort": "high"}), + ], +) +async def test_the_platform_rebuilds_the_submitted_agent_from_its_type_and_config( + agent_type: str, config: dict[str, Any], services: FakeServices +) -> None: + services.route("api", "POST", SUBMIT, json={"status": "queued"}) + services.route("api", "GET", POLL, json={"status": "completed", "reward": 1.0}) + kind = AgentType(agent_type).cls + agent = kind(kind.config_cls.model_validate({**config, "gateway": True})) + + await Task(env="sums", id="add").run(agent, runtime=HostedRuntime(poll_interval=0)) + + (body,) = services.bodies("api", "POST", SUBMIT) + rebuilt = AgentType(body["agent"]["type"]).cls.load(body["agent"]["config"]) + assert (type(rebuilt), rebuilt.dump()) == (kind, agent.dump()) diff --git a/tests/eval/test_hud_tunnel.py b/tests/eval/test_hud_tunnel.py new file mode 100644 index 000000000..cf07d3161 --- /dev/null +++ b/tests/eval/test_hud_tunnel.py @@ -0,0 +1,203 @@ +"""``HUDRuntime``: the agent loop runs here against an environment the platform hosts, +reached through a runtime session and a WebSocket tunnel. + +The fake runtime service relays each tunnel's bytes to an environment served in this +process, so a tunneled rollout really completes and its reward proves it. +""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from typing import TYPE_CHECKING, Any + +import pytest + +from hud import Environment +from hud.agents.base import Agent +from hud.eval import HUDRuntime, LocalRuntime, Task, Taskset +from tests.fixtures.agents import ScriptedAgent +from tests.support import Reply, eventually +from tests.support.platform import SESSION, SESSIONS, TUNNEL, host_on_runtime + +if TYPE_CHECKING: + from starlette.websockets import WebSocket + + from hud.eval import Run + from tests.support import FakeServices, HudEnv + + +def checked() -> Environment: + env = Environment("checked") + + @env.template() + async def check(): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + return env + + +async def test_a_platform_taskset_runs_over_the_tunnel_and_releases_its_session( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + row = Task(env="checked", id="check") + + async with LocalRuntime(checked())(row) as served: + host_on_runtime(services, int(served.url.rsplit(":", 1)[1])) + job = await Taskset("remote", [row], taskset_id="ts-1").run(ScriptedAgent("yes")) + + (run,) = job.runs + assert run.reward == 1.0 + (created,) = services.requests("runtime", "POST", SESSIONS) + assert (created.bearer, created.json) == ( + "k", + {"environment": "checked", "trace_id": str(uuid.UUID(run.trace_id))}, + ) + assert { + (tunnel.params["id"], tunnel.bearer) for tunnel in services.requests("runtime", "WS") + } == {("sess-1", "k")} + assert [request.path for request in services.requests("runtime", "DELETE")] == [ + "/runtime/sessions/sess-1" + ] + + +async def test_runtime_url_points_the_tunnel_at_another_runtime_service( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_RUNTIME_URL="http://127.0.0.1:9") + row = Task(env="checked", id="check") + + async with LocalRuntime(checked())(row) as served: + host_on_runtime(services, int(served.url.rsplit(":", 1)[1])) + job = await row.run( + ScriptedAgent("yes"), runtime=HUDRuntime(runtime_url=services.url("runtime") + "/") + ) + + assert job.reward == 1.0 + + +async def dropped(websocket: WebSocket, params: dict[str, str]) -> None: + await websocket.accept() + await websocket.close() + + +@pytest.mark.parametrize( + ("session", "error"), + [ + pytest.param( + Reply(status=503, json={"detail": "no capacity"}), + "[provisioning] httpx.HTTPStatusError: Server error '503 Service Unavailable'", + id="a-rejected-session", + ), + pytest.param( + Reply(json={"status": "ok"}), + "[provisioning] RuntimeError: Runtime gateway did not return a session id", + id="a-session-without-an-id", + ), + ], +) +async def test_a_session_the_runtime_service_does_not_grant_fails_the_run( + session: Reply, error: str, services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + services.route("runtime", "POST", SESSIONS, session) + + job = await Task(env="checked", id="check").run(ScriptedAgent("yes"), runtime=HUDRuntime()) + + (run,) = job.runs + assert (run.trace.error or "").startswith(error) + assert [request.method for request in services.requests()] == ["POST"] + + +async def test_a_tunnel_that_keeps_dropping_ends_at_the_deadline_and_releases_the_session( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + services.route("runtime", "POST", SESSIONS, json={"id": "sess-1"}) + services.route("runtime", "DELETE", SESSION, json={}) + services.websocket("runtime", TUNNEL, dropped) + + job = await Task(env="checked", id="check").run( + ScriptedAgent("yes"), runtime=HUDRuntime(), rollout_timeout=0.5 + ) + + assert job.runs[0].trace.error == "rollout timed out after 0.5s during starting task" + await eventually(lambda: len(services.requests("runtime", "DELETE", SESSION)) == 1) + + +async def test_resources_the_deployed_env_cannot_change_are_warned_about_once( + services: FakeServices, hud_env: HudEnv, caplog: pytest.LogCaptureFixture +) -> None: + hud_env.set(HUD_API_KEY="k") + sized = {"image": "checked:1", "resources": {"cpu": 4, "memory_mb": 8192}} + rows = [Task(env="checked", id="check", slug=slug, runtime_config=sized) for slug in "ab"] + + async with LocalRuntime(checked())(Task(env="checked", id="check")) as served: + host_on_runtime(services, int(served.url.rsplit(":", 1)[1])) + job = await Taskset("sized", rows).run(ScriptedAgent("yes"), runtime=HUDRuntime()) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + assert [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] == [ + "HUDRuntime cannot honor task runtime_config ['resources'] on an already-deployed env; " + "rollouts proceed on the platform's defaults" + ] + + +@pytest.mark.parametrize( + "runtime_config", + [ + {"resources": {"gpu": {"type": "H100"}}}, + {"resources": {"os": "windows"}}, + {"resources": {"tpu": {"type": "v5e", "topology": "2x2"}}}, + {"limits": {"startup_timeout_s": 60}}, + ], + ids=["a-gpu", "an-os", "a-tpu", "limits"], +) +async def test_requirements_the_deployed_env_cannot_meet_fail_the_run_before_a_session( + runtime_config: dict[str, Any], services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + row = Task(env="checked", id="check", runtime_config=runtime_config) + + job = await row.run(ScriptedAgent("yes"), runtime=HUDRuntime()) + + assert (job.runs[0].trace.error or "").startswith( + "[provisioning] ValueError: HUDRuntime cannot honor this task's declared placement " + "requirements or limits" + ) + assert services.requests() == [] + + +async def test_the_tunnel_needs_a_hud_api_key(services: FakeServices) -> None: + job = await Task(env="checked", id="check").run(ScriptedAgent("yes"), runtime=HUDRuntime()) + + assert job.runs[0].trace.error == ( + "[provisioning] RuntimeError: HUD runtime tunnel requires HUD_API_KEY" + ) + assert services.requests() == [] + + +async def test_the_deprecated_run_timeout_still_ends_a_tunneled_run( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + row = Task(env="checked", id="check") + + class Waits(Agent): + async def __call__(self, run: Run) -> None: + await asyncio.sleep(30) + + async with LocalRuntime(checked())(row) as served: + host_on_runtime(services, int(served.url.rsplit(":", 1)[1])) + with pytest.warns(DeprecationWarning, match="rollout_timeout=... to Task.run"): + runtime = HUDRuntime(run_timeout=0.5) + job = await row.run(Waits(), runtime=runtime) + await eventually(lambda: len(services.requests("runtime", "DELETE", SESSION)) == 1) + + assert (job.runs[0].trace.stop_reason, job.runs[0].trace.error) == ( + "timeout", + "rollout timed out after 0.5s during agent loop", + ) diff --git a/tests/eval/test_rollout.py b/tests/eval/test_rollout.py new file mode 100644 index 000000000..2e075b0f0 --- /dev/null +++ b/tests/eval/test_rollout.py @@ -0,0 +1,581 @@ +"""One rollout: what the agent is handed, and how what the agent, the grader or a +deadline did decides the run's reward, status and error.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +import pytest + +from hud import Environment +from hud.agents.base import Agent +from hud.agents.types import AgentConfig +from hud.capabilities import Connection +from hud.eval import LocalRuntime, Runtime, Task, rollout +from hud.telemetry.context import get_current_trace_id +from tests.fixtures.agents import ScriptedAgent +from tests.support import eventually + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + from pathlib import Path + + from hud.eval import Run + + +@pytest.mark.parametrize( + ("prompt", "turns", "text"), + [ + pytest.param("add 2 3", [("user", "add 2 3")], "add 2 3", id="text-is-one-user-turn"), + pytest.param(None, [("user", "")], "", id="no-prompt-is-one-empty-user-turn"), + pytest.param( + [ + {"role": "system", "content": "be brief"}, + {"role": "assistant", "content": {"type": "text", "text": "hi"}}, + {"role": "user", "content": "add 2 3"}, + ], + [("user", "be brief"), ("assistant", "hi"), ("user", "add 2 3")], + "be brief\n\nhi\n\nadd 2 3", + id="chat-turns-keep-their-roles-and-system-becomes-user", + ), + pytest.param( + [ + {"role": "user", "content": "first"}, + {"role": "user", "content": {"type": "image", "data": "aW1n", "mimeType": "x"}}, + {"role": "user", "content": ""}, + "second", + ], + [("user", "first"), ("user", None), ("user", ""), ("user", "second")], + "first\n\nsecond", + id="non-text-and-empty-turns-are-left-out-of-the-text", + ), + pytest.param( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + {"type": "image", "data": "aW1n", "mimeType": "image/png"}, + ], + } + ], + [("user", "what is this?"), ("user", None)], + "what is this?", + id="a-turn-of-several-blocks-is-one-message-per-block", + ), + ], +) +async def test_the_agent_is_handed_the_prompt_as_turns_and_as_text( + prompt: Any, turns: list[tuple[str, str | None]], text: str +) -> None: + env = Environment("prompts") + + @env.template() + async def ask(): + yield prompt + yield 1.0 + + handed = [] + + class Reader(Agent): + async def __call__(self, run: Run) -> None: + roles = [(m.role, getattr(m.content, "text", None)) for m in run.prompt_messages] + handed.append((roles, run.prompt_text)) + + await ask().run(Reader()) + + assert handed == [(turns, text)] + + +async def test_the_agent_is_handed_its_episodes_bindings_inside_its_trace() -> None: + env = Environment("robots") + + @env.template() + async def claim(): + yield {"prompt": "go", "bindings": {"robot": {"token": "slot-2"}}} + yield 1.0 + + handed = [] + + class Reader(Agent): + async def __call__(self, run: Run) -> None: + handed.append((run.bindings, get_current_trace_id())) + + job = await claim().run(Reader()) + + assert handed == [({"robot": {"token": "slot-2"}}, job.runs[0].trace_id)] + + +async def test_bindings_that_are_not_objects_fail_the_start() -> None: + env = Environment("robots") + + @env.template() + async def claim(): + yield {"prompt": "go", "bindings": {"robot": "slot-2"}} + yield 1.0 + + job = await claim().run(ScriptedAgent("x")) + + (run,) = job.runs + assert (run.trace.status, run.prompt) == ("error", None) + assert (run.trace.error or "").startswith( + "[starting task] TypeError: task start frame 'bindings' must map capability name -> object" + ) + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_the_agent_holds_its_connections_only_while_it_runs(tmp_path: Path) -> None: + env = Environment("searching") + env.workspace(tmp_path / "ws") + + @env.template() + async def look(): + yield "go" + yield 1.0 + + search = Connection( + name="search", + capability="shell", + url="https://search.example/v1", + headers={"Authorization": "Bearer secret"}, + ) + held = [] + + class Reader(Agent): + async def __call__(self, run: Run) -> None: + held.append(sorted(run.connections)) + + run = await rollout(look(), Reader(), runtime=LocalRuntime(env), connections=[search]) + + assert (held, run.connections, run.reward) == ([["search"]], {}, 1.0) + + +@pytest.mark.parametrize( + ("grade", "reward", "content", "status", "error"), + [ + pytest.param(0.5, 0.5, None, "completed", None, id="a-number"), + pytest.param( + {"score": 0.5, "content": "half right", "info": {"checked": 2}}, + 0.5, + "half right", + "completed", + None, + id="a-frame-with-a-score", + ), + pytest.param( + {"score": 0.0, "content": "the env broke", "isError": True}, + 0.0, + "the env broke", + "completed", + "the env broke", + id="a-frame-marked-as-an-error", + ), + pytest.param( + {"score": True}, + 0.0, + None, + "error", + "[grading] hud.clients.client.HudProtocolError: hud rpc error -32603: " + "tasks.grade: result must include a numeric 'score'", + id="a-boolean-score", + ), + pytest.param( + {"done": True}, + 0.0, + None, + "error", + "[grading] hud.clients.client.HudProtocolError: hud rpc error -32000: " + "task 'grade_it' graded with a dict missing a numeric 'score' (keys: ['done'])", + id="a-frame-without-a-score", + ), + pytest.param( + "high", + 0.0, + None, + "error", + "[grading] hud.clients.client.HudProtocolError: hud rpc error -32000: " + "task graded with str: yield a number, an object with a numeric .reward, " + "or a dict containing a numeric 'score'", + id="neither", + ), + ], +) +async def test_what_the_task_grades_with_is_the_runs_reward_or_its_error( + grade: Any, reward: float, content: str | None, status: str, error: str | None +) -> None: + env = Environment("grades") + + @env.template() + async def grade_it(): + yield "go" + yield grade + + job = await grade_it().run(ScriptedAgent("x")) + + (run,) = job.runs + assert (run.reward, run.grade.content, run.trace.status, run.trace.error) == ( + reward, + content, + status, + error, + ) + assert (run in job.errors) is (error is not None) + + +async def test_a_grade_breakdown_reaches_the_evaluation_without_its_grader_details() -> None: + env = Environment("grades") + + @env.template() + async def grade_it(): + yield "go" + yield { + "score": 0.5, + "subscores": [{"name": "tests", "value": 0.5, "info": {"stdout": "2 failed"}}], + } + + job = await grade_it().run(ScriptedAgent("x")) + + assert job.runs[0].evaluation["subscores"] == [ + {"name": "tests", "weight": 1.0, "value": 0.5, "children": None} + ] + + +async def test_an_agent_that_raises_before_answering_is_graded_on_no_answer() -> None: + env = Environment("checked") + + @env.template() + async def check(): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + class Raises(Agent): + async def __call__(self, run: Run) -> None: + raise RuntimeError("agent exploded") + + job = await check().run(Raises()) + + (run,) = job.runs + assert (run.reward, run.trace.status, run.trace.error, job.errors) == ( + 0.0, + "error", + "[agent loop] RuntimeError: agent exploded", + [], + ) + + +async def test_an_agent_that_raises_after_answering_keeps_its_grade() -> None: + env = Environment("checked") + + @env.template() + async def check(): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + class RaisesAfter(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "yes" + raise RuntimeError("agent exploded") + + job = await check().run(RaisesAfter()) + + (run,) = job.runs + assert (run.reward, run.trace.status, run.trace.error, job.reward) == ( + 1.0, + "error", + "[agent loop] RuntimeError: agent exploded", + 1.0, + ) + + +@pytest.mark.parametrize( + ("agent_timeout", "task_timeout"), + [(0.3, None), (30, 0.3)], + ids=["the-agents-own", "the-tasks-over-the-agents"], +) +async def test_the_agent_timeout_stops_the_agent_and_still_grades( + agent_timeout: float, task_timeout: float | None +) -> None: + env = Environment("checked") + + @env.template() + async def check(): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + class AnswersThenWaits(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "yes" + await asyncio.sleep(30) + + task = check() + if task_timeout is not None: + task.agent_config = {"timeout_seconds": task_timeout} + job = await task.run(AnswersThenWaits(AgentConfig(timeout_seconds=agent_timeout))) + + (run,) = job.runs + assert (run.reward, run.trace.status, run.trace.stop_reason, run.trace.error) == ( + 1.0, + "error", + "timeout", + "agent timed out after 0.3s", + ) + + +async def test_a_timeout_the_agent_raises_itself_is_an_ordinary_failure() -> None: + env = Environment("checked") + + @env.template() + async def check(): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + class ProviderTimesOut(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "yes" + raise TimeoutError("provider timed out") + + job = await check().run(ProviderTimesOut(AgentConfig(timeout_seconds=30))) + + (run,) = job.runs + assert (run.reward, run.trace.stop_reason, run.trace.error) == ( + 1.0, + None, + "[agent loop] TimeoutError: provider timed out", + ) + + +async def test_a_task_whose_start_raises_fails_before_the_agent_is_prompted() -> None: + env = Environment("broken") + + @env.template() + async def start_raises(): + raise RuntimeError("start exploded") + yield "never" + + job = await start_raises().run(ScriptedAgent("x")) + + (run,) = job.runs + assert (run.trace.status, run.prompt, job.errors) == ("error", None, [run]) + assert (run.trace.error or "").startswith("[starting task] ") + assert "start exploded" in (run.trace.error or "") + + +async def test_a_grader_that_raises_leaves_the_run_ungraded() -> None: + env = Environment("broken") + + @env.template() + async def grade_raises(): + yield "go" + raise RuntimeError("grader exploded") + yield 0.0 + + job = await grade_raises().run(ScriptedAgent("x")) + + (run,) = job.runs + assert (run.reward, run.trace.status, job.errors) == (0.0, "error", [run]) + assert (run.trace.error or "").startswith("[grading] ") + assert "grader exploded" in (run.trace.error or "") + + +async def test_cancelling_a_rollout_cancels_its_task_and_stops_its_env() -> None: + events = [] + env = Environment("cancelled") + + @env.shutdown + async def stop() -> None: + events.append("stopped") + + @env.template() + async def wait(): + try: + yield "go" + yield 1.0 + finally: + events.append("task closed") + + class Waits(Agent): + async def __call__(self, run: Run) -> None: + events.append("prompted") + await asyncio.sleep(30) + + pending = asyncio.create_task(wait().run(Waits())) + await eventually(lambda: events == ["prompted"]) + pending.cancel() + + with pytest.raises(asyncio.CancelledError): + await pending + await eventually(lambda: events == ["prompted", "task closed", "stopped"]) + + +@pytest.mark.parametrize( + ("template", "phase"), + [ + ("slow_start", "starting task"), + ("slow_agent", "agent loop"), + ("slow_grade", "grading"), + ], +) +async def test_the_rollout_timeout_ends_the_run_naming_its_phase_and_stops_the_env( + template: str, phase: str +) -> None: + events = [] + env = Environment("slow") + + @env.shutdown + async def stop() -> None: + events.append("stopped") + + @env.template() + async def slow_start(): + await asyncio.sleep(30) + yield "never" + yield 1.0 + + @env.template() + async def slow_agent(): + yield "take your time" + yield 1.0 + + @env.template() + async def slow_grade(): + yield "go" + await asyncio.sleep(30) + yield 1.0 + + class WaitsWhenAsked(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "done" + if run.prompt_text == "take your time": + await asyncio.sleep(30) + + job = await Task(env="slow", id=template).run( + WaitsWhenAsked(), runtime=LocalRuntime(env), rollout_timeout=0.3 + ) + + (run,) = job.runs + assert (run.reward, run.trace.status, run.trace.stop_reason, run.trace.error) == ( + 0.0, + "error", + "timeout", + f"rollout timed out after 0.3s during {phase}", + ) + assert job.errors == [run] + await eventually(lambda: events == ["stopped"]) + + +async def test_a_rollout_cancelled_on_an_env_served_elsewhere_closes_its_task_there() -> None: + events = [] + env = Environment("served") + + @env.template() + async def wait(): + try: + yield "go" + yield 1.0 + finally: + events.append("task closed") + + class Waits(Agent): + async def __call__(self, run: Run) -> None: + events.append("prompted") + await asyncio.sleep(30) + + async with LocalRuntime(env)(wait()) as served: + pending = asyncio.create_task(wait().run(Waits(), runtime=Runtime(served.url))) + await eventually(lambda: events == ["prompted"]) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + + await eventually(lambda: events == ["prompted", "task closed"]) + + +async def test_a_rollout_timed_out_on_an_env_served_elsewhere_closes_its_task_there() -> None: + events = [] + env = Environment("served") + + @env.template() + async def wait(): + try: + yield "go" + yield 1.0 + finally: + events.append("task closed") + + class Waits(Agent): + async def __call__(self, run: Run) -> None: + await asyncio.sleep(30) + + async with LocalRuntime(env)(wait()) as served: + job = await wait().run(Waits(), runtime=Runtime(served.url), rollout_timeout=0.3) + + await eventually(lambda: events == ["task closed"]) + assert (job.runs[0].trace.stop_reason, job.runs[0].runtime) == ("timeout", served.url) + + +async def test_the_rollout_timeout_ends_a_run_still_being_provisioned() -> None: + @asynccontextmanager + async def never_ready(task: Task) -> AsyncIterator[Runtime]: + await asyncio.sleep(30) + yield Runtime("tcp://127.0.0.1:1") + + task = Task(env="checked", id="check") + job = await task.run(ScriptedAgent("yes"), runtime=never_ready, rollout_timeout=0.3) + + (run,) = job.runs + assert (run.trace.status, run.trace.stop_reason, run.trace.error, run.runtime) == ( + "error", + "timeout", + "rollout timed out after 0.3s during provisioning", + None, + ) + + +async def test_a_rollout_timeout_in_teardown_keeps_the_grade_and_lets_teardown_finish() -> None: + env = Environment("checked") + + @env.template() + async def check(): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + release = asyncio.Event() + events = [] + + @asynccontextmanager + async def slow_to_stop(task: Task) -> AsyncIterator[Runtime]: + async with LocalRuntime(env)(task) as runtime: + yield runtime + await release.wait() + events.append("torn down") + + job = await check().run(ScriptedAgent("yes"), runtime=slow_to_stop, rollout_timeout=0.3) + release.set() + + (run,) = job.runs + assert (run.reward, run.trace.stop_reason, run.trace.error) == ( + 1.0, + "timeout", + "rollout timed out after 0.3s during actor cleanup", + ) + await eventually(lambda: events == ["torn down"]) + + +async def test_a_provider_that_fails_returns_a_run_with_what_it_reported() -> None: + @asynccontextmanager + async def fails_with_notes(task: Task) -> AsyncIterator[Runtime]: + error = EOFError("sandbox closed") + error.add_note("env output: ImportError: bugs") + raise error + yield Runtime("tcp://127.0.0.1:1") + + job = await Task(env="checked", id="check").run(ScriptedAgent("yes"), runtime=fails_with_notes) + + (run,) = job.runs + assert (run.reward, run.trace.status, run.prompt, run.runtime) == (0.0, "error", None, None) + assert run.trace.error == ( + "[provisioning] EOFError: sandbox closed\nenv output: ImportError: bugs" + ) diff --git a/tests/eval/test_runtimes.py b/tests/eval/test_runtimes.py new file mode 100644 index 000000000..b61df2b68 --- /dev/null +++ b/tests/eval/test_runtimes.py @@ -0,0 +1,465 @@ +"""What every runtime owes a rollout: it places the environment, carries the control +channel's frames to it and tears it down, whether the run ends, the env cannot serve, +or the rollout is cancelled; and it refuses a row it cannot provision before asking +anything of anyone. + +Each runtime serves a fixture environment from ``tests.fixtures.envs``. Those envs log +their lifecycle and start a daemon when ``HUD_FIXTURE_LIFECYCLE`` names a file, so a +runtime is clean once every env it started here has stopped and every process an env +ran in or started is gone. +""" + +from __future__ import annotations + +import asyncio +import os +import socket +import subprocess +import uuid +from collections import Counter +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.agents.base import Agent +from hud.eval import ( + DaytonaRuntime, + DockerRuntime, + HUDRuntime, + LocalRuntime, + ModalRuntime, + Runtime, + RuntimeConfig, + Shared, + SubprocessRuntime, + Task, +) +from hud.eval import runtime as runtime_module +from tests.fixtures.agents import ScriptedAgent +from tests.fixtures.envs import source +from tests.support import eventually +from tests.support.platform import SESSION, SESSIONS, host_on_runtime + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable + from pathlib import Path + + from hud.eval import Provider, Run + from tests.support import FakeServices, HudEnv + +CONTAINER = pytest.param("DockerRuntime", marks=[pytest.mark.e2e, pytest.mark.docker]) +MODAL = pytest.param("ModalRuntime", marks=[pytest.mark.e2e, pytest.mark.modal]) +#: Runtimes no placement contract runs on, and why. +NOT_PLACED = { + "DaytonaRuntime": "no lane has a Daytona account; its refusals are a table below", + "HostedRuntime": "the platform runs the whole rollout; test_hosted holds what it owes", +} +EVERY_RUNTIME = [ + "LocalRuntime", + "SubprocessRuntime", + "Shared", + "Runtime", + "HUDRuntime", + CONTAINER, + MODAL, +] + + +def test_every_runtime_hud_exports_is_a_row_or_says_why_not() -> None: + exported = {name for name in runtime_module.__all__ if name.endswith("Runtime")} | {"Shared"} + rows = {row if isinstance(row, str) else row.values[0] for row in EVERY_RUNTIME} + + assert rows | NOT_PLACED.keys() == exported + + +def gone(pid: int) -> bool: + """Whether process ``pid`` has exited (an unreaped zombie counts as exited).""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + status = subprocess.run( + ["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True, check=False + ) + return status.stdout.strip().startswith("Z") + + +def serves(url: str) -> bool: + host, port = url.removeprefix("tcp://").rsplit(":", 1) + try: + socket.create_connection((host, int(port)), timeout=1).close() + except OSError: + return False + return True + + +@asynccontextmanager +async def placed( + runtime: str, fixture: str, request: pytest.FixtureRequest, startup: int = 120 +) -> AsyncIterator[tuple[Provider, Callable[[], Awaitable[bool]]]]: + """Place fixture env ``fixture`` on ``runtime``; yield it and its cleanliness check.""" + path = source(fixture) + log = request.getfixturevalue("tmp_path") / "lifecycle.log" + request.getfixturevalue("hud_env").set(HUD_FIXTURE_LIFECYCLE=str(log)) + + async def nothing_left() -> bool: + events = [ + line.split()[:2] for line in (log.read_text() if log.exists() else "").splitlines() + ] + here = str(os.getpid()) + started = Counter(pid for event, pid in events if event == "start") + stopped = Counter(pid for event, pid in events if event == "stop") + processes = {int(pid) for event, pid in events if event != "stop" and pid != here} + return started[here] == stopped[here] and all(gone(pid) for pid in processes) + + match runtime: + case "LocalRuntime": + yield LocalRuntime(path, ready_timeout=startup), nothing_left + case "SubprocessRuntime": + yield SubprocessRuntime(path, ready_timeout=startup), nothing_left + case "Shared": + yield Shared(LocalRuntime(path, ready_timeout=startup), width=2), nothing_left + case "Runtime": + async with SubprocessRuntime(path)(Task(env=fixture, id="serve")) as served: + + async def still_serving() -> bool: + return serves(served.url) + + yield Runtime(served.url), still_serving + case "HUDRuntime": + services: FakeServices = request.getfixturevalue("services") + request.getfixturevalue("hud_env").set(HUD_API_KEY="k") + async with SubprocessRuntime(path)(Task(env=fixture, id="serve")) as served: + host_on_runtime(services, int(served.url.rsplit(":", 1)[1])) + + async def released() -> bool: + created = len(services.requests("runtime", "POST", SESSIONS)) + deleted = len(services.requests("runtime", "DELETE", SESSION)) + return created == deleted > 0 + + yield HUDRuntime(), released + case "DockerRuntime": + label = uuid.uuid4().hex + image = request.getfixturevalue("fixture_image")(fixture) + limits = RuntimeConfig.model_validate({"limits": {"startup_timeout_s": startup}}) + + async def removed() -> bool: + listing = await asyncio.create_subprocess_exec( + *("docker", "ps", "--all", "--quiet", "--filter", f"label=hud-test={label}"), + stdout=asyncio.subprocess.PIPE, + ) + listed, _ = await listing.communicate() + return listed.split() == [] + + labelled = ["--label", f"hud-test={label}"] + yield DockerRuntime(image, run_args=labelled, runtime_config=limits), removed + case "ModalRuntime": + import modal + + request.getfixturevalue("modal_account") + image = request.getfixturevalue("modal_image")(fixture) + limits = RuntimeConfig.model_validate({"limits": {"startup_timeout_s": startup}}) + inner = ModalRuntime(image=image, workdir="/hud", runtime_config=limits) + sandboxes = [] + + @asynccontextmanager + async def recorded(task: Task) -> AsyncIterator[Runtime]: + async with inner(task) as placed_runtime: + sandboxes.append(placed_runtime.params["instance_id"]) + yield placed_runtime + + async def terminated() -> bool: + for instance in sandboxes: + sandbox = await modal.Sandbox.from_id.aio(instance) + if await sandbox.poll.aio() is None: + return False + return True + + yield recorded, terminated + + +#: Over a WebSocket's default 1 MiB message limit, and under the 16 MiB frame limit. +LARGE = "x" * (4 * 1024 * 1024) + + +@pytest.mark.parametrize("runtime", EVERY_RUNTIME) +@pytest.mark.parametrize( + ("fixture", "template", "word", "reward", "status"), + [ + pytest.param("echo", "echo", "tangerine", 1.0, "completed", id="a-row-runs-to-its-grade"), + pytest.param("echo", "echo", LARGE, 1.0, "completed", id="a-frame-of-megabytes-crosses"), + pytest.param("faulty", "start_raises", None, 0.0, "error", id="a-start-that-fails"), + ], +) +async def test_a_runtime_places_the_env_carries_its_frames_and_leaves_nothing_behind( + fixture: str, + template: str, + word: str | None, + reward: float, + status: str, + runtime: str, + request: pytest.FixtureRequest, +) -> None: + row = Task(env=fixture, id=template, args={"word": word} if word else {}) + + async with placed(runtime, fixture, request) as (provider, nothing_left): + job = await row.run(ScriptedAgent(word or "x"), runtime=provider) + await eventually(nothing_left) + + (run,) = job.runs + assert (run.reward, run.trace.status) == (reward, status) + + +@pytest.mark.parametrize( + "runtime", ["LocalRuntime", "SubprocessRuntime", "Shared", CONTAINER, MODAL] +) +@pytest.mark.parametrize( + "fixture", + ["broken", "stalls"], + ids=["an-env-that-exits-at-once", "an-env-that-never-finishes-starting"], +) +async def test_an_env_that_cannot_serve_fails_the_run_before_the_prompt_and_is_cleaned_up( + fixture: str, runtime: str, request: pytest.FixtureRequest +) -> None: + async with placed(runtime, fixture, request, startup=5) as (provider, nothing_left): + job = await Task(env=fixture, id="never").run(ScriptedAgent("x"), runtime=provider) + await eventually(nothing_left) + + (run,) = job.runs + assert (run.trace.status, run.prompt, job.errors) == ("error", None, [run]) + assert (run.trace.error or "").startswith(("[provisioning] ", "[starting task] ")) + + +@pytest.mark.parametrize("runtime", EVERY_RUNTIME) +async def test_cancelling_a_rollout_propagates_and_leaves_nothing_behind( + runtime: str, request: pytest.FixtureRequest +) -> None: + prompted = asyncio.Event() + + class Waits(Agent): + async def __call__(self, run: Run) -> None: + prompted.set() + await asyncio.sleep(3600) + + async with placed(runtime, "echo", request) as (provider, nothing_left): + row = Task(env="echo", id="echo", args={"word": "tangerine"}) + pending = asyncio.create_task(row.run(Waits(), runtime=provider)) + await asyncio.wait_for(prompted.wait(), timeout=90) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + + await eventually(nothing_left, within=10) + + +@pytest.mark.parametrize("runtime", [CONTAINER, MODAL]) +@pytest.mark.parametrize(("link", "reward"), [(False, 1.0), (True, 0.0)], ids=["files", "a-link"]) +async def test_a_verifier_in_its_own_sandbox_receives_the_actors_session_files( + link: bool, reward: float, runtime: str, request: pytest.FixtureRequest +) -> None: + """The verifier's own limits give it its own sandbox, so the actor's files must move; + a session holding anything but files and directories does not.""" + row = Task.model_validate( + { + "env": "sessions", + "id": "solve", + "args": {"link": link}, + "verifier": { + "env": "sessions", + "id": "verify", + "runtime_config": {"limits": {"startup_timeout_s": 600}}, + }, + } + ) + + async with placed(runtime, "sessions", request) as (provider, nothing_left): + job = await row.run(ScriptedAgent("secret"), runtime=provider) + await eventually(nothing_left) + + (run,) = job.runs + assert run.reward == reward + assert ("runtime session contains" in (run.trace.error or "")) is link + + +# ─── what each runtime refuses to provision ──────────────────────────── + + +@pytest.mark.parametrize("runtime", [LocalRuntime, SubprocessRuntime]) +async def test_a_local_runtime_refuses_a_row_that_asks_for_its_own_runtime( + runtime: type[LocalRuntime | SubprocessRuntime], +) -> None: + row = Task(env="echo", id="echo", runtime_config={"image": "lab"}) + + job = await row.run(ScriptedAgent("x"), runtime=runtime(source("echo"))) + + assert job.runs[0].trace.error == ( + f"[provisioning] ValueError: {runtime.__name__} does not support task runtime_config" + ) + + +COMPOSE = {"compose": {"document": "compose.yaml"}} +SIMPLE = "services:\n main: {image: lab}\n" + + +@pytest.mark.parametrize( + ("runtime_config", "compose", "run_args", "message"), + [ + pytest.param( + {"image": "lab:1", "resources": {"gpu": {"type": "H100"}}}, + "", + [], + "cannot select GPUs by type", + id="a-gpu-type", + ), + pytest.param( + COMPOSE, + 'services:\n main: {image: lab, network_mode: "service:gateway"}\n' + " gateway: {image: gateway}\n", + ["--privileged"], + "run_args apply only to image environments", + id="run-args-for-a-compose-project", + ), + pytest.param( + COMPOSE, + "services:\n main: {image: lab, network_mode: 'service:relay'}\n" + " relay: {image: relay, network_mode: 'service:main'}\n", + [], + "network_mode service cycle includes 'main'", + id="a-network-cycle", + ), + pytest.param( + COMPOSE, + "include: [other.yaml]\n" + SIMPLE, + [], + "does not support Compose include or extends", + id="a-compose-include", + ), + pytest.param( + COMPOSE, + "services:\n main: {image: $HOST_ONLY}\n", + [], + "variable 'HOST_ONLY' is not set by the project .env", + id="a-variable-only-the-host-sets", + ), + ], +) +async def test_docker_refuses_what_it_cannot_provision_before_running_anything( + runtime_config: dict[str, Any], + compose: str, + run_args: list[str], + message: str, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(PATH="/nonexistent") + (tmp_path / ".env").write_text("") + (tmp_path / "compose.yaml").write_text(compose) + row = Task.model_validate( + {"env": "echo", "id": "echo", "runtime_config": runtime_config}, + context={"base_path": tmp_path}, + ) + + job = await row.run(ScriptedAgent("x"), runtime=DockerRuntime(run_args=run_args)) + + (run,) = job.runs + assert (run.trace.status, run.prompt) == ("error", None) + assert message in (run.trace.error or "") + + +@pytest.mark.parametrize( + ("runtime_config", "message"), + [ + pytest.param( + {"resources": {"gpu": {"type": ["A100", "H100"]}}}, + "alternative GPU types", + id="alternative-gpu-types", + ), + pytest.param({"resources": {"os": "windows"}}, "resources.os", id="an-os"), + pytest.param( + {**COMPOSE, "resources": {"gpu": {}}}, "cannot attach GPUs", id="a-gpu-for-compose" + ), + ], +) +async def test_modal_refuses_what_it_cannot_provision_before_creating_a_sandbox( + runtime_config: dict[str, Any], message: str, tmp_path: Path +) -> None: + pytest.importorskip("modal", reason="needs the modal extra") + (tmp_path / ".env").write_text("") + (tmp_path / "compose.yaml").write_text(SIMPLE) + row = Task.model_validate( + {"env": "echo", "id": "echo", "runtime_config": runtime_config}, + context={"base_path": tmp_path}, + ) + + job = await row.run(ScriptedAgent("x"), runtime=ModalRuntime("hud-lab")) + + assert message in (job.runs[0].trace.error or "") + + +async def test_modal_refuses_sandbox_secrets_for_a_compose_project(tmp_path: Path) -> None: + modal = pytest.importorskip("modal", reason="needs the modal extra") + (tmp_path / ".env").write_text("") + (tmp_path / "compose.yaml").write_text(SIMPLE) + row = Task.model_validate( + {"env": "echo", "id": "echo", "runtime_config": COMPOSE}, context={"base_path": tmp_path} + ) + runtime = ModalRuntime(sandbox_secrets=[modal.Secret.from_dict({"API": "x"})]) + + job = await row.run(ScriptedAgent("x"), runtime=runtime) + + assert "sandbox secrets require an image runtime" in (job.runs[0].trace.error or "") + + +@pytest.mark.parametrize( + ("snapshot", "runtime_config", "message"), + [ + pytest.param("snap", COMPOSE, "does not support runtime_config.compose", id="compose"), + pytest.param( + "snap", + {"limits": {"run_timeout_s": 60}}, + "does not support runtime_config.run_timeout_s", + id="a-run-limit", + ), + pytest.param( + None, + {"image": "lab", "resources": {"os": "linux"}}, + "does not support runtime_config.resources.os", + id="an-os", + ), + pytest.param( + None, + {"image": "lab", "resources": {"cpu": 1.5}}, + "needs a whole number of CPUs, got 1.5", + id="a-fraction-of-a-cpu", + ), + pytest.param( + None, None, "requires snapshot_name or runtime_config.image", id="nothing-to-boot" + ), + pytest.param( + "snap", + {"resources": {"cpu": 2}}, + "cannot resize an already-built snapshot", + id="resources-for-a-built-snapshot", + ), + ], +) +async def test_daytona_refuses_what_it_cannot_provision_before_creating_a_sandbox( + snapshot: str | None, + runtime_config: dict[str, Any] | None, + message: str, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + pytest.importorskip("daytona", reason="needs the daytona extra") + hud_env.set(DAYTONA_API_KEY="k", DAYTONA_API_URL="http://127.0.0.1:9") + (tmp_path / ".env").write_text("") + (tmp_path / "compose.yaml").write_text(SIMPLE) + row = Task.model_validate( + {"env": "echo", "id": "echo", "runtime_config": runtime_config}, + context={"base_path": tmp_path}, + ) + + job = await row.run(ScriptedAgent("x"), runtime=DaytonaRuntime(snapshot)) + + assert message in (job.runs[0].trace.error or "") diff --git a/tests/eval/test_shared.py b/tests/eval/test_shared.py new file mode 100644 index 000000000..7fc315262 --- /dev/null +++ b/tests/eval/test_shared.py @@ -0,0 +1,183 @@ +"""``Shared``: one substrate per environment and runtime configuration, leased to at most +``width`` rollouts at once for the life of its scope.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING + +from hud import Environment +from hud.agents.base import Agent +from hud.eval import LocalRuntime, Runtime, Shared, Task, Taskset, rollout +from tests.fixtures.agents import ScriptedAgent +from tests.support import eventually + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from hud.eval import Run + + +async def test_a_pool_boots_once_and_holds_at_most_width_rollouts_at_a_time() -> None: + events = [] + env = Environment("sim") + + @env.initialize + async def up() -> None: + events.append("up") + + @env.template() + async def step(n: int): + yield "go" + yield 1.0 + + live = [0] + seen = [] + + class Overlaps(Agent): + async def __call__(self, run: Run) -> None: + live[0] += 1 + await eventually(lambda: live[0] >= 2) + await asyncio.sleep(0.05) + seen.append(live[0]) + live[0] -= 1 + + pool = Shared(LocalRuntime(env), width=2) + job = await Taskset("sim", [step(n=1), step(n=2)]).run(Overlaps(), runtime=pool, group=2) + + assert [run.reward for run in job.runs] == [1.0] * 4 + assert (events, max(seen)) == (["up"], 2) + + +async def test_an_open_pool_stays_warm_across_calls_and_stops_when_its_scope_closes() -> None: + events = [] + env = Environment("sim") + + @env.initialize + async def up() -> None: + events.append("up") + + @env.shutdown + async def down() -> None: + events.append("down") + + @env.template() + async def step(): + yield "go" + yield 1.0 + + async with Shared(LocalRuntime(env), width=2) as pool: + await step().run(ScriptedAgent("x"), runtime=pool) + await step().run(ScriptedAgent("x"), runtime=pool) + assert events == ["up"] + + assert events == ["up", "down"] + + +async def test_each_environment_and_runtime_configuration_gets_its_own_substrate() -> None: + placed = [] + + def build(task: Task) -> Environment: + env = Environment(task.env) + + @env.template() + async def step(): + yield "go" + yield 1.0 + + return env + + @asynccontextmanager + async def sized(task: Task) -> AsyncIterator[Runtime]: + assert task.runtime_config is not None and task.runtime_config.resources is not None + placed.append((task.env, task.runtime_config.resources.cpu)) + unsized = task.model_copy(update={"runtime_config": None}) + async with LocalRuntime(build)(unsized) as runtime: + yield runtime + + rows = [ + Task(env="sim", id="step", slug="small", runtime_config={"resources": {"cpu": 1}}), + Task(env="sim", id="step", slug="small-again", runtime_config={"resources": {"cpu": 1}}), + Task(env="sim", id="step", slug="large", runtime_config={"resources": {"cpu": 4}}), + Task(env="lab", id="step", slug="elsewhere", runtime_config={"resources": {"cpu": 1}}), + ] + job = await Taskset("sim", rows).run(ScriptedAgent("x"), runtime=Shared(sized, width=4)) + + assert job.reward == 1.0 + assert sorted(placed) == [("lab", 1), ("sim", 1), ("sim", 4)] + + +async def test_a_failed_boot_fails_its_own_lease_and_the_next_lease_boots_again() -> None: + attempts = [] + env = Environment("sim") + + @env.template() + async def step(): + yield "go" + yield 1.0 + + @asynccontextmanager + async def flaky(task: Task) -> AsyncIterator[Runtime]: + attempts.append(task.slug) + if len(attempts) == 1: + raise RuntimeError("boot failed") + async with LocalRuntime(env)(task) as runtime: + yield runtime + + job = await step().run( + ScriptedAgent("x"), runtime=Shared(flaky, width=1), group=2, max_concurrent=1 + ) + + assert [(run.reward, run.trace.error) for run in job.runs] == [ + (0.0, "[provisioning] RuntimeError: boot failed"), + (1.0, None), + ] + assert len(attempts) == 2 + + +async def test_a_substrate_past_its_run_limit_is_replaced_on_the_next_lease() -> None: + events = [] + env = Environment("sim") + + @env.initialize + async def up() -> None: + events.append("up") + + @env.shutdown + async def down() -> None: + events.append("down") + + @env.template() + async def step(): + yield "go" + yield 1.0 + + @asynccontextmanager + async def limited(task: Task) -> AsyncIterator[Runtime]: + async with LocalRuntime(env)(task.model_copy(update={"runtime_config": None})) as runtime: + yield runtime + + row = Task(env="sim", id="step", runtime_config={"limits": {"run_timeout_s": 1}}) + async with Shared(limited, width=1) as pool: + await row.run(ScriptedAgent("x"), runtime=pool) + await asyncio.sleep(1.1) + await row.run(ScriptedAgent("x"), runtime=pool) + + assert events == ["up", "down", "up", "down"] + + +async def test_a_lease_outside_the_pools_scope_fails_the_run() -> None: + env = Environment("sim") + + @env.template() + async def step(): + yield "go" + yield 1.0 + + run = await rollout(step(), ScriptedAgent("x"), runtime=Shared(LocalRuntime(env), width=1)) + + assert (run.trace.error or "").startswith( + "[provisioning] RuntimeError: Shared substrates outlive single rollouts; lease inside " + "the scope" + ) diff --git a/tests/eval/test_sources.py b/tests/eval/test_sources.py new file mode 100644 index 000000000..c796e2594 --- /dev/null +++ b/tests/eval/test_sources.py @@ -0,0 +1,478 @@ +"""``LocalRuntime`` and ``SubprocessRuntime`` serve an environment from the source a user +has: a file, a directory, a package, a live instance, or a constructor.""" + +from __future__ import annotations + +import asyncio +import os +import signal +import subprocess +import textwrap +from typing import TYPE_CHECKING, Any + +import pytest + +from hud import Environment +from hud.agents.base import Agent +from hud.environment import load_environment +from hud.eval import LocalRuntime, SubprocessRuntime, Task, Taskset, rollout +from tests.fixtures.agents import ScriptedAgent +from tests.support import eventually + +if TYPE_CHECKING: + from pathlib import Path + + from hud.eval import Run + + +def write(path: Path, source: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(source)) + return path + + +@pytest.mark.parametrize( + ("source", "imports"), + [("counted/sums.py", 2), ("counted", 1)], + ids=["a-file-is-imported-for-each-rollout", "a-directory-is-imported-once"], +) +async def test_a_source_path_is_imported_to_serve_rollouts( + source: str, imports: int, tmp_path: Path +) -> None: + log = tmp_path / "imports.log" + write( + tmp_path / "counted" / "sums.py", + f""" + from pathlib import Path + + from hud import Environment + + with Path({str(log)!r}).open("a") as log: + log.write("import\\n") + + env = Environment("lab") + + + @env.template() + async def solve(): + yield "go" + yield 1.0 + """, + ) + + job = await Task(env="lab", id="solve").run( + ScriptedAgent("x"), runtime=LocalRuntime(tmp_path / source), group=2 + ) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + assert log.read_text().splitlines() == ["import"] * imports + + +async def test_a_package_entry_serves_the_env_its_relative_import_brings_in( + tmp_path: Path, +) -> None: + package = tmp_path / f"envs_{abs(hash(tmp_path)):x}" + write(package / "__init__.py", "") + write( + package / "sums.py", + """ + from hud import Environment + + env = Environment("lab") + + + @env.template() + async def solve(): + yield "go" + yield 1.0 + """, + ) + entry = write(package / "env.py", "from .sums import env\n") + + job = await Task(env="lab", id="solve").run(ScriptedAgent("x"), runtime=LocalRuntime(entry)) + + assert job.reward == 1.0 + + +async def test_a_second_copy_of_a_package_from_another_directory_is_refused( + tmp_path: Path, +) -> None: + name = f"envs_{abs(hash(tmp_path)):x}" + for copy in ("first", "second"): + write(tmp_path / copy / name / "__init__.py", "") + write( + tmp_path / copy / name / "env.py", + """ + from hud import Environment + + env = Environment("lab") + + + @env.template() + async def solve(): + yield "go" + yield 1.0 + """, + ) + row = Task(env="lab", id="solve") + + first = await row.run( + ScriptedAgent("x"), runtime=LocalRuntime(tmp_path / "first" / name / "env.py") + ) + second = await row.run( + ScriptedAgent("x"), runtime=LocalRuntime(tmp_path / "second" / name / "env.py") + ) + + assert first.reward == 1.0 + assert f"package {name!r} is already imported from a different source root" in ( + second.runs[0].trace.error or "" + ) + + +WHERE = """ +from pathlib import Path + +from hud import Environment + +env = Environment("{name}") + + +@env.template() +async def where(): + yield f"{name} serves from {{Path.cwd()}}" + yield 1.0 +""" + + +@pytest.mark.parametrize("serve", [LocalRuntime, SubprocessRuntime]) +async def test_one_source_serves_each_rows_env_unless_env_pins_one( + serve: type[LocalRuntime | SubprocessRuntime], tmp_path: Path +) -> None: + root = tmp_path / "project" + write(root / "alpha.py", WHERE.format(name="alpha")) + write(root / "beta.py", WHERE.format(name="beta")) + source = write( + root / "env.py", "from alpha import env as alpha\nfrom beta import env as beta\n" + ) + rows = [Task(env=name, id="where", slug=name) for name in ("alpha", "beta")] + prompts = [] + + class Reader(Agent): + async def __call__(self, run: Run) -> None: + prompts.append(run.prompt_text.split(" serves")[0]) + + await Taskset("mixed", rows).run(Reader(), runtime=serve(source), max_concurrent=1) + await Taskset("mixed", rows).run(Reader(), runtime=serve(source, env="beta"), max_concurrent=1) + + assert prompts == ["alpha", "beta", "beta", "beta"] + + +async def test_a_child_process_serves_from_its_sources_directory(tmp_path: Path) -> None: + source = write(tmp_path / "project" / "env.py", WHERE.format(name="lab")) + prompts = [] + + class Reader(Agent): + async def __call__(self, run: Run) -> None: + prompts.append(run.prompt_text) + + await Task(env="lab", id="where").run(Reader(), runtime=SubprocessRuntime(source)) + + assert prompts == [f"lab serves from {tmp_path / 'project'}"] + + +async def test_a_template_can_import_a_sibling_module_while_it_runs(tmp_path: Path) -> None: + write(tmp_path / "lazy_helper.py", "ANSWER = 'from the sibling'\n") + source = write( + tmp_path / "env.py", + """ + from hud import Environment + + env = Environment("lab") + + + @env.template() + async def solve(): + import lazy_helper + + answer = yield "go" + yield 1.0 if answer == lazy_helper.ANSWER else 0.0 + """, + ) + + job = await Task(env="lab", id="solve").run( + ScriptedAgent("from the sibling"), runtime=LocalRuntime(source) + ) + + assert job.reward == 1.0 + + +async def test_a_constructor_builds_an_env_for_each_rollout_and_they_run_at_once() -> None: + built = [] + started = asyncio.Event() + + def build(task: Task) -> Environment: + built.append(task.slug) + env = Environment("lab") + + @env.initialize + async def wait_for_the_others() -> None: + if len(built) == 3: + started.set() + await started.wait() + + @env.template() + async def solve(): + yield "go" + yield 1.0 + + return env + + row = Task(env="lab", id="solve") + job = await row.run(ScriptedAgent("x"), runtime=LocalRuntime(build, ready_timeout=5), group=3) + + assert ([run.reward for run in job.runs], built) == ([1.0] * 3, [row.slug] * 3) + + +def test_a_live_environment_serves_rollouts_from_one_event_loop_after_another() -> None: + env = Environment("lab") + + @env.template() + async def solve(): + yield "go" + yield 1.0 + + rewards = [asyncio.run(solve().run(ScriptedAgent("x"), group=2)).reward for _ in range(2)] + + assert rewards == [1.0, 1.0] + + +async def test_a_child_process_serves_a_live_env_from_the_file_its_templates_are_in( + tmp_path: Path, +) -> None: + source = write(tmp_path / "env.py", WHERE.format(name="lab")) + + job = await Task(env="lab", id="where").run( + ScriptedAgent("x"), runtime=SubprocessRuntime(load_environment(source)) + ) + + assert job.reward == 1.0 + + +@pytest.mark.filterwarnings("ignore:coroutine 'sleep' was never awaited:RuntimeWarning") +@pytest.mark.parametrize( + ("source", "row_env", "error"), + [ + pytest.param( + "from hud import Environment\n", + "other", + "ValueError", + id="a-source-without-the-rows-env", + ), + pytest.param( + "import asyncio\n\nasyncio.run(asyncio.sleep(0))\n", + "lab", + "the env source ran async code while being imported to place a rollout — guard " + 'top-level run calls with `if __name__ == "__main__":`', + id="a-source-that-runs-an-event-loop-at-import", + ), + ], +) +async def test_a_source_that_cannot_serve_the_row_fails_the_run( + source: str, row_env: str, error: str, tmp_path: Path +) -> None: + path = write(tmp_path / "env.py", source) + + run = await rollout( + Task(env=row_env, id="solve"), ScriptedAgent("x"), runtime=LocalRuntime(path) + ) + + assert (run.trace.status, run.prompt) == ("error", None) + assert error in (run.trace.error or "") + + +async def test_a_constructor_that_returns_something_else_fails_the_run() -> None: + def build(task: Task) -> Any: + return "lab" + + run = await rollout( + Task(env="lab", id="solve"), ScriptedAgent("x"), runtime=LocalRuntime(build) + ) + + assert ( + run.trace.error + == "[provisioning] TypeError: LocalRuntime: constructor returned 'lab', not an Environment" + ) + + +async def test_an_in_process_env_is_torn_down_while_a_peer_still_holds_a_connection() -> None: + env = Environment("lab") + + @env.template() + async def solve(): + yield "go" + yield 1.0 + + held = [] + + class HoldsAConnection(Agent): + async def __call__(self, run: Run) -> None: + assert run.runtime is not None + host, port = run.runtime.removeprefix("tcp://").rsplit(":", 1) + held.append(await asyncio.open_connection(host, int(port))) + + job = await asyncio.wait_for(solve().run(HoldsAConnection()), timeout=10) + + assert job.reward == 1.0 + ((_, writer),) = held + writer.close() + + +async def test_an_env_whose_initialize_raises_is_still_shut_down() -> None: + events = [] + env = Environment("lab") + + @env.initialize + async def explode() -> None: + raise RuntimeError("daemon failed to start") + + @env.shutdown + async def down() -> None: + events.append("shut down") + + @env.template() + async def solve(): + yield "go" + yield 1.0 + + job = await solve().run(ScriptedAgent("x")) + + assert "daemon failed to start" in (job.runs[0].trace.error or "") + assert events == ["shut down"] + + +async def test_a_missing_source_fails_the_run(tmp_path: Path) -> None: + run = await rollout( + Task(env="lab", id="solve"), + ScriptedAgent("x"), + runtime=SubprocessRuntime(tmp_path / "missing.py"), + ) + + assert run.trace.error == ( + f"[provisioning] FileNotFoundError: SubprocessRuntime: source not found: " + f"{tmp_path / 'missing.py'}" + ) + + +def gone(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + status = subprocess.run( + ["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True, check=False + ) + return status.stdout.strip().startswith("Z") + + +async def test_a_child_that_exits_before_serving_reports_its_output_and_takes_its_children( + tmp_path: Path, +) -> None: + source = write( + tmp_path / "env.py", + """ + import asyncio + import sys + from pathlib import Path + + from hud import Environment + + env = Environment("leaky") + + + @env.initialize + async def start(): + child = await asyncio.create_subprocess_exec("sleep", "120") + Path("child.pid").write_text(str(child.pid)) + print("daemon starting", file=sys.stderr, flush=True) + raise SystemExit(3) + """, + ) + + run = await rollout( + Task(env="leaky", id="solve"), ScriptedAgent("x"), runtime=SubprocessRuntime(source) + ) + + assert f"spawned env exited with code 3 before serving (source: {source}):\n" in ( + run.trace.error or "" + ) + assert "daemon starting" in (run.trace.error or "") + pid = int((tmp_path / "child.pid").read_text()) + try: + await eventually(lambda: gone(pid), within=5) + finally: + if not gone(pid): + os.kill(pid, signal.SIGKILL) + + +async def test_a_child_that_closes_its_output_before_serving_fails_the_run( + tmp_path: Path, +) -> None: + source = write( + tmp_path / "env.py", + """ + import os + import sys + import time + + print("closing stdout", file=sys.stderr, flush=True) + os.close(1) + time.sleep(60) + """, + ) + + run = await rollout( + Task(env="lab", id="solve"), ScriptedAgent("x"), runtime=SubprocessRuntime(source) + ) + + assert "spawned env closed stdout before serving" in (run.trace.error or "") + assert "closing stdout" in (run.trace.error or "") + + +async def test_a_child_streams_its_output_to_the_terminal( + tmp_path: Path, capfd: pytest.CaptureFixture[str] +) -> None: + source = write( + tmp_path / "env.py", + """ + import sys + + from hud import Environment + + env = Environment("lab") + + + @env.initialize + async def start(): + print("x" * 100_000, flush=True) + print("environment booted", flush=True) + print("y" * 100_000, file=sys.stderr, flush=True) + print("environment warning", file=sys.stderr, flush=True) + + + @env.template() + async def solve(): + print("solving", flush=True) + yield "go" + yield 1.0 + """, + ) + + job = await Task(env="lab", id="solve").run( + ScriptedAgent("x"), runtime=SubprocessRuntime(source) + ) + + out, err = capfd.readouterr() + assert job.reward == 1.0 + assert "x" * 100_000 + "\nenvironment booted\n" in out + assert "solving\n" in out + assert "y" * 100_000 + "\nenvironment warning\n" in err diff --git a/tests/eval/test_taskset_run.py b/tests/eval/test_taskset_run.py new file mode 100644 index 000000000..27819ab19 --- /dev/null +++ b/tests/eval/test_taskset_run.py @@ -0,0 +1,361 @@ +"""``Task.run`` and ``Taskset.run``: rows expand into rollouts that are placed, bounded +and collected into one ``Job``.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any + +import pytest + +from hud import Environment +from hud.agents.base import Agent +from hud.agents.types import AgentConfig +from hud.eval import ( + DockerRuntime, + Job, + LocalRuntime, + RuntimeConfig, + Shared, + Task, + Taskset, +) +from tests.fixtures.agents import ScriptedAgent +from tests.support import eventually + +if TYPE_CHECKING: + from hud.eval import Run + from tests.support import FakeServices, HudEnv + + +async def test_each_row_runs_group_times_in_task_major_order_under_one_job() -> None: + env = Environment("sums") + + @env.template() + async def add(a: int, b: int): + answer = yield f"{a}+{b}" + yield 1.0 if answer == str(a + b) else 0.0 + + class Adds(Agent): + async def __call__(self, run: Run) -> None: + a, b = run.prompt_text.split("+") + run.trace.content = str(int(a) + int(b)) + + first, second = add(a=1, b=2), add(a=3, b=4) + job = await Taskset("sums", [first, second]).run(Adds(), group=3) + + assert [run.slug for run in job.runs] == [first.slug] * 3 + [second.slug] * 3 + assert [run.reward for run in job.runs] == [1.0] * 6 + assert {run.job_id for run in job.runs} == {job.id} + assert [len(job.results[row.slug]) for row in (first, second)] == [3, 3] + groups = [{run.group_id for run in job.results[row.slug]} for row in (first, second)] + assert [len(group) for group in groups] == [1, 1] and groups[0] != groups[1] + + +@pytest.mark.parametrize(("max_concurrent", "peak"), [(None, 6), (2, 2)]) +async def test_every_rollout_gets_its_own_env_and_max_concurrent_bounds_them( + max_concurrent: int | None, peak: int +) -> None: + started = [] + + def build(task: Task) -> Environment: + env = Environment("sums") + + @env.initialize + async def up() -> None: + started.append(task.slug) + + @env.template() + async def add(a: int, b: int): + yield f"{a}+{b}" + yield 1.0 + + return env + + live = [0] + seen = [] + + class Overlaps(Agent): + async def __call__(self, run: Run) -> None: + live[0] += 1 + await eventually(lambda: live[0] >= peak) + await asyncio.sleep(0.05) + seen.append(live[0]) + live[0] -= 1 + + rows = [Task(env="sums", id="add", args={"a": a, "b": 1}) for a in (1, 2)] + await Taskset("sums", rows).run( + Overlaps(), runtime=LocalRuntime(build), group=3, max_concurrent=max_concurrent + ) + + assert (len(started), max(seen)) == (6, peak) + + +async def test_rollouts_on_one_live_environment_take_turns() -> None: + env = Environment("shared") + live = [] + + @env.template() + async def wait(): + live.append(1) + assert sum(live) == 1 + yield "go" + live.append(-1) + yield 1.0 + + class Pauses(Agent): + async def __call__(self, run: Run) -> None: + await asyncio.sleep(0.05) + + job = await wait().run(Pauses(), group=3) + + assert [run.reward for run in job.runs] == [1.0] * 3 + + +async def test_a_jobs_reward_averages_its_graded_runs_and_its_errors_are_the_rest() -> None: + env = Environment("mixed") + + @env.template() + async def check(): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + @env.template() + async def grade_raises(): + yield "go" + raise RuntimeError("grader exploded") + yield 0.0 + + class AnswersThenWaits(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "yes" + await asyncio.sleep(30) + + rows = [ + Task(env="mixed", id="check", slug="timed-out-after-answering"), + Task(env="mixed", id="grade_raises", slug="grader-raised"), + ] + agent = AnswersThenWaits(AgentConfig(timeout_seconds=0.3)) + job = await Taskset("mixed", rows).run(agent, runtime=LocalRuntime(env)) + + assert {run.slug: run.reward for run in job.runs} == { + "timed-out-after-answering": 1.0, + "grader-raised": 0.0, + } + assert ([run.slug for run in job.errors], job.reward) == (["grader-raised"], 1.0) + + +async def test_an_open_job_collects_the_runs_of_several_calls_with_its_group() -> None: + env = Environment("sums") + + @env.template() + async def add(a: int, b: int): + yield f"{a}+{b}" + yield 1.0 + + session = await Job.start("training", group=2) + + first = await add(a=1, b=2).run(ScriptedAgent("3"), job=session) + second = await add(a=3, b=4).run(ScriptedAgent("7"), job=session) + + assert first is second is session + assert [run.job_id for run in session.runs] == [session.id] * 4 + + +async def test_cancelling_a_run_cancels_every_rollout_and_stops_their_envs() -> None: + events = [] + + def build(task: Task) -> Environment: + env = Environment("sums") + + @env.shutdown + async def down() -> None: + events.append("stopped") + + @env.template() + async def add(a: int, b: int): + answer = yield f"{a}+{b}" + events.append(f"graded {answer}") + yield 1.0 + + return env + + prompted = [] + + class Waits(Agent): + async def __call__(self, run: Run) -> None: + prompted.append(run.prompt_text) + await asyncio.sleep(30) + + rows = [Task(env="sums", id="add", args={"a": a, "b": 1}) for a in (1, 2)] + pending = asyncio.create_task( + Taskset("sums", rows).run(Waits(), runtime=LocalRuntime(build), group=2) + ) + await eventually(lambda: len(prompted) == 4) + pending.cancel() + + with pytest.raises(asyncio.CancelledError): + await pending + await eventually(lambda: events == ["stopped"] * 4) + + +@pytest.mark.parametrize( + ("agent_timeout", "row_config", "provider_limits", "verifier_limits", "rollout_timeout"), + [ + pytest.param(None, None, None, None, 0, id="no-time-at-all"), + pytest.param(10, None, None, None, 10, id="an-agent-timeout-at-the-rollout-timeout"), + pytest.param( + 3600, + {"limits": {"run_timeout_s": 3600}}, + None, + None, + None, + id="an-agent-timeout-at-the-run-limit", + ), + pytest.param( + None, + {"limits": {"run_timeout_s": 3600}}, + None, + None, + 3600, + id="a-run-limit-at-the-rollout-timeout", + ), + pytest.param( + None, + None, + {"startup_timeout_s": 90}, + None, + 60, + id="a-providers-limit-past-the-rollout-timeout", + ), + pytest.param( + None, + None, + None, + {"startup_timeout_s": 90}, + 90, + id="a-verifiers-limit-at-the-rollout-timeout", + ), + ], +) +async def test_timeouts_that_contradict_each_other_raise_before_any_rollout( + agent_timeout: float | None, + row_config: dict[str, Any] | None, + provider_limits: dict[str, int] | None, + verifier_limits: dict[str, int] | None, + rollout_timeout: float | None, + services: FakeServices, + hud_env: HudEnv, +) -> None: + hud_env.set(PATH="/nonexistent") + task = Task.model_validate( + { + "env": "lab", + "id": "solve", + "agent_config": {"timeout_seconds": agent_timeout}, + "runtime_config": row_config, + "verifier": { + "env": "judge", + "id": "verify", + "runtime_config": {"limits": verifier_limits}, + }, + } + ) + provider = DockerRuntime( + "lab", runtime_config=RuntimeConfig.model_validate({"limits": provider_limits}) + ) + + with pytest.raises(ValueError, match="must be"): + await task.run(ScriptedAgent("x"), runtime=provider, rollout_timeout=rollout_timeout) + + assert services.requests() == [] + + +@pytest.mark.parametrize( + "runtime_config", + [None, {"image": "lab:1"}], + ids=["a-portable-row", "a-container-row-beside-a-portable-one"], +) +async def test_rows_nothing_places_need_a_runtime(runtime_config: dict[str, Any] | None) -> None: + rows = [ + Task(env="lab", id="solve", slug="portable"), + Task(env="lab", id="solve", slug="other", runtime_config=runtime_config), + ] + + with pytest.raises(ValueError, match=r"no placement: pass runtime= .*LocalRuntime"): + await Taskset("t", rows).run(ScriptedAgent("x")) + + +async def test_a_row_naming_an_image_is_placed_in_docker(hud_env: HudEnv) -> None: + hud_env.set(PATH="/nonexistent") + + job = await Task(env="lab", id="solve", runtime_config={"image": "lab:1"}).run( + ScriptedAgent("x") + ) + + assert job.runs[0].trace.error == ( + "[provisioning] FileNotFoundError: [Errno 2] No such file or directory: 'docker'" + ) + + +async def test_an_empty_taskset_needs_no_runtime() -> None: + job = await Taskset("empty", []).run(ScriptedAgent("x")) + + assert (job.runs, job.reward, job.errors) == ([], 0.0, []) + + +@pytest.mark.parametrize( + ("slugs", "group", "name"), + [ + (["only"], 1, "solve"), + (["only"], 3, "solve (3 times)"), + (["first", "second"], 1, "letters (2 tasks)"), + (["first", "second"], 2, "letters (2 tasks) (2 times)"), + ], +) +async def test_a_run_registers_one_job_named_for_what_it_runs( + slugs: list[str], group: int, name: str, services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + services.route("api", "POST", "/v2/trace/job/{id}/enter", json={}) + services.route("api", "POST", "/v2/trace/{id}/enter", json={}) + services.route("api", "POST", "/v2/trace/{id}/exit", json={}) + services.route("telemetry", "POST", "/trace/{id}/telemetry-upload", json={}) + env = Environment("letters") + + @env.template() + async def solve(): + yield "go" + yield 1.0 + + rows = [solve().model_copy(update={"slug": slug}) for slug in slugs] + job = await Taskset("letters", rows).run(ScriptedAgent("x"), group=group) + + (registered,) = services.requests("api", "POST", "/v2/trace/job/{id}/enter") + assert registered.path == f"/v2/trace/job/{job.id}/enter" + assert (registered.json["name"], registered.json["group"]) == (name, group) + + +async def test_a_pool_placement_boots_once_for_the_call_and_closes_after_it() -> None: + events = [] + env = Environment("pooled") + + @env.initialize + async def up() -> None: + events.append("up") + + @env.shutdown + async def down() -> None: + events.append("down") + + @env.template() + async def solve(n: int): + yield "go" + yield 1.0 + + rows = [solve(n=1), solve(n=2)] + job = await Taskset("pooled", rows).run( + ScriptedAgent("x"), runtime=Shared(LocalRuntime(env), width=4), group=2 + ) + + assert ([run.reward for run in job.runs], events) == ([1.0] * 4, ["up", "down"]) diff --git a/tests/eval/test_tasksets.py b/tests/eval/test_tasksets.py new file mode 100644 index 000000000..ca72c5660 --- /dev/null +++ b/tests/eval/test_tasksets.py @@ -0,0 +1,147 @@ +"""Tasksets as collections: the rows a source, a data file or the platform holds, how +each row is keyed, and what a sync would change.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from hud.eval import ComposeProject, RuntimeConfig, Task, Taskset + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import FakeServices, HudEnv + + +@pytest.mark.parametrize("suffix", [".json", ".jsonl"]) +def test_a_taskset_written_to_a_data_file_reads_back_row_for_row( + suffix: str, tmp_path: Path +) -> None: + project = tmp_path / "project" + project.mkdir() + (project / "compose.yaml").write_text("services:\n main:\n image: lab\n") + rows = [ + Task(env="lab", id="add", args={"a": 2, "b": 3}), + Task.model_validate( + { + "env": "lab", + "id": "add", + "args": {"a": 1, "b": 1}, + "slug": "one-plus-one", + "validation": [{"name": "answer", "arguments": {"expected": 2}}], + "agent_config": {"timeout_seconds": 30.0}, + "columns": {"difficulty": "easy"}, + "runtime_config": { + "image": "registry.test/trainer:1", + "resources": { + "cpu": 8, + "gpu": {"type": ["H100", "A100"], "count": 2}, + "tpu": {"type": "v5e", "topology": "2x4"}, + }, + "limits": {"startup_timeout_s": 600}, + }, + "verifier": {"env": "judge", "id": "verify", "args": {"expected": "2"}}, + } + ), + Task( + env="compose", + id="solve", + runtime_config=RuntimeConfig( + compose=ComposeProject(document=project / "compose.yaml", service_access=True) + ), + ), + ] + + written = Taskset("demo", rows).to_file(tmp_path / "out" / f"tasks{suffix}") + loaded = Taskset.from_file(written) + + assert loaded.name == "tasks" + assert [row.model_dump() for row in loaded] == [row.model_dump() for row in rows] + assert '"document": "../project/compose.yaml"' in written.read_text() + + +SOURCE = """ +from hud import Environment, Taskset + +env = Environment("lab") + + +@env.template() +async def solve(n: int): + yield "go" + yield 1.0 + + +single = solve(n=1) +several = [solve(n=2), solve(n=3)] +collected = Taskset("inner", [solve(n=4)]) +_private = solve(n=5) +""" + + +def test_a_python_source_holds_the_tasks_and_tasksets_it_defines(tmp_path: Path) -> None: + (tmp_path / "tasks.py").write_text(SOURCE) + + loaded = Taskset.from_file(tmp_path / "tasks.py") + + assert loaded.name == "tasks" + assert sorted(row.args["n"] for row in loaded) == [1, 2, 3, 4] + + +def test_a_directory_holds_the_tasks_of_every_source_in_it(tmp_path: Path) -> None: + root = tmp_path / "suite" + root.mkdir() + (root / "first.py").write_text(SOURCE) + (root / "second.py").write_text(SOURCE.replace("n=", "n=1")) + (root / "conftest.py").write_text("raise RuntimeError('not a task source')\n") + + loaded = Taskset.from_file(root) + + assert loaded.name == "suite" + assert len(loaded) == 8 + + +@pytest.mark.parametrize("reference", ["SheetBench-50", "eeeeeeee-0000-4000-8000-000000000001"]) +def test_a_platform_taskset_loads_by_name_or_id( + reference: str, services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + taskset_id = "eeeeeeee-0000-4000-8000-000000000001" + services.route( + "api", + "GET", + "/v2/tasksets/by-name/SheetBench-50", + json={"taskset_id": taskset_id, "name": "SheetBench-50"}, + ) + services.route( + "api", + "GET", + f"/v2/tasksets/{taskset_id}/export", + json={ + "name": "SheetBench-50", + "tasks": [ + {"name": "sum-a", "env": "sheets", "scenario": "sum", "args": {"cell": "A1"}}, + {"name": "sum-b", "env": "sheets", "scenario": "sum", "columns": {"hard": True}}, + ], + }, + ) + + loaded = Taskset.from_api(reference) + + assert (loaded.name, loaded.taskset_id) == ("SheetBench-50", taskset_id) + assert [(row.slug, row.env, row.id, row.args) for row in loaded] == [ + ("sum-a", "sheets", "sum", {"cell": "A1"}), + ("sum-b", "sheets", "sum", {}), + ] + + +def test_a_platform_taskset_that_does_not_exist_is_refused( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + services.route("api", "GET", "/v2/tasksets/by-name/{name}", status=404, json={}) + + with pytest.raises(ValueError, match="taskset not found: missing"): + Taskset.from_api("missing") diff --git a/tests/eval/test_verifier.py b/tests/eval/test_verifier.py new file mode 100644 index 000000000..550e2bc65 --- /dev/null +++ b/tests/eval/test_verifier.py @@ -0,0 +1,351 @@ +"""Verifier tasks: an agent-less second task whose grade is the run's grade of record. + +The actor's ``solve`` grades 0.25 and hands its answer on; the judge's ``verify`` pays +1.0 only when that answer was ``secret``. A reward of 1.0 proves the actor's result +reached the verifier. +""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING + +import pytest +from pydantic import BaseModel + +from hud import Environment +from hud.agents.base import Agent +from hud.eval import LocalRuntime, Runtime, RuntimeConfig, RuntimeLimits, Task +from tests.fixtures.agents import ScriptedAgent +from tests.support import eventually + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from hud.eval import Run + + +async def test_a_verifier_on_its_own_env_runs_after_the_actors_env_is_torn_down() -> None: + events = [] + actor = Environment("actor") + judge = Environment("judge") + + @actor.initialize + async def actor_up() -> None: + events.append("actor up") + + @actor.shutdown + async def actor_down() -> None: + events.append("actor down") + + @judge.initialize + async def judge_up() -> None: + events.append("judge up") + + @judge.shutdown + async def judge_down() -> None: + events.append("judge down") + + @actor.template() + async def solve(): + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} + + @judge.template() + async def verify(): + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 + + task = solve() + task.verifier = verify() + job = await task.run(ScriptedAgent("secret")) + + (run,) = job.runs + assert (run.reward, run.trace.status) == (1.0, "completed") + assert events == ["actor up", "actor down", "judge up", "judge down"] + + +async def test_a_verifier_on_the_actors_env_grades_on_the_same_substrate() -> None: + events = [] + env = Environment("reviewed") + + @env.initialize + async def up() -> None: + events.append("up") + + @env.shutdown + async def down() -> None: + events.append("down") + + @env.template() + async def solve(): + answer = yield "answer secret" + events.append("actor graded") + yield {"score": 0.25, "answer": answer} + + @env.template() + async def verify(): + result = yield "" + events.append("verifier graded") + yield 1.0 if result["answer"] == "secret" else 0.0 + + task = solve() + task.verifier = verify() + job = await task.run(ScriptedAgent("secret")) + + assert job.runs[0].reward == 1.0 + assert events == ["up", "actor graded", "verifier graded", "down"] + + +async def test_a_verifier_asking_for_its_own_runtime_runs_beside_an_actor_that_cannot_move( + caplog: pytest.LogCaptureFixture, +) -> None: + events = [] + env = Environment("reviewed") + + @env.initialize + async def up() -> None: + events.append("up") + + @env.template() + async def solve(): + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} + + @env.template() + async def verify(): + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 + + task = solve() + task.verifier = verify() + task.verifier.runtime_config = RuntimeConfig(limits=RuntimeLimits(run_timeout_s=20)) + async with LocalRuntime(env)(task.model_copy(update={"verifier": None})) as served: + job = await task.run(ScriptedAgent("secret"), runtime=Runtime(served.url)) + + assert (job.runs[0].reward, events) == (1.0, ["up"]) + assert "verifier runtime_config ignored: Runtime cannot transport the actor's session" in ( + caplog.text + ) + + +@pytest.mark.parametrize( + ("grading", "verdict", "reward", "error", "graded"), + [ + pytest.param( + "raise", + "check", + 1.0, + "[grading] hud.clients.client.HudProtocolError: hud rpc error -32000: " + "actor grade exploded", + True, + id="the-actors-grader-raised", + ), + pytest.param( + "scoreless", + "check", + 1.0, + "[grading] hud.clients.client.HudProtocolError: hud rpc error -32000: " + "task 'solve' graded with a dict missing a numeric 'score' (keys: ['answer'])", + True, + id="a-scoreless-actor-grade", + ), + pytest.param( + "score", + "raise", + 0.0, + "[verifying] hud.clients.client.HudProtocolError: hud rpc error -32000: " + "verifier exploded", + False, + id="the-verifier-raised", + ), + pytest.param( + "score", + "scoreless", + 0.0, + "[verifying] hud.clients.client.HudProtocolError: hud rpc error -32000: " + "task 'verify' graded with a dict missing a numeric 'score' (keys: ['verdict'])", + False, + id="a-scoreless-verdict", + ), + pytest.param("score", "zero", 0.0, None, True, id="a-zero-verdict"), + ], +) +async def test_the_verdict_is_the_grade_whichever_side_fails( + grading: str, verdict: str, reward: float, error: str | None, graded: bool +) -> None: + actor = Environment("actor") + judge = Environment("judge") + + @actor.template() + async def solve(grading: str): + answer = yield "answer secret" + if grading == "raise": + raise RuntimeError("actor grade exploded") + yield {"score": 0.25, "answer": answer} if grading == "score" else {"answer": answer} + + @judge.template() + async def verify(verdict: str): + result = yield "" + if verdict == "raise": + raise RuntimeError("verifier exploded") + if verdict == "scoreless": + yield {"verdict": "pass"} + else: + yield 1.0 if verdict == "check" and result["answer"] == "secret" else 0.0 + + task = solve(grading=grading) + task.verifier = verify(verdict=verdict) + job = await task.run(ScriptedAgent("secret")) + + (run,) = job.runs + assert (run.reward, run.trace.error, run in job.errors) == (reward, error, not graded) + + +async def test_an_agent_that_fails_after_answering_is_still_judged() -> None: + actor = Environment("actor") + judge = Environment("judge") + + @actor.template() + async def solve(): + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} + + @judge.template() + async def verify(): + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 + + class RaisesAfter(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "secret" + raise RuntimeError("agent exploded") + + task = solve() + task.verifier = verify() + job = await task.run(RaisesAfter()) + + (run,) = job.runs + assert (run.reward, run.trace.status, run.trace.error, job.errors) == ( + 1.0, + "error", + "[agent loop] RuntimeError: agent exploded", + [], + ) + + +async def test_a_verifier_that_cannot_be_placed_leaves_the_run_ungraded() -> None: + actor = Environment("actor") + + @actor.template() + async def solve(): + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} + + @asynccontextmanager + async def actor_only(task: Task) -> AsyncIterator[Runtime]: + if task.env != "actor": + raise RuntimeError("no judge here") + async with LocalRuntime(actor)(task) as runtime: + yield runtime + + task = solve() + task.verifier = Task(env="judge", id="verify") + job = await task.run(ScriptedAgent("secret"), runtime=actor_only) + + (run,) = job.runs + assert (run.reward, run.trace.error, job.errors) == ( + 0.0, + "[provisioning verifier] RuntimeError: no judge here", + [run], + ) + + +async def test_the_verifier_receives_the_actors_result_as_its_declared_type() -> None: + class ActorResult(BaseModel): + score: float + answer: str + + received = [] + env = Environment("reviewed") + + @env.template() + async def solve(): + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} + + @env.template(returns=ActorResult) + async def verify(): + result = yield "" + received.append(result.content) + yield 1.0 + + task = solve() + task.verifier = verify() + await task.run(ScriptedAgent("secret")) + + assert received == [ActorResult(score=0.25, answer="secret")] + + +@pytest.mark.parametrize( + ("held", "reward", "error", "events"), + [ + pytest.param( + "actor", + 0.0, + "rollout timed out after 0.3s during actor cleanup", + ["actor down"], + id="in-the-actors-teardown-the-verifier-never-starts", + ), + pytest.param( + "judge", + 1.0, + "rollout timed out after 0.3s during cleanup", + ["actor down", "judge up", "judge down"], + id="in-the-verifiers-teardown-the-verdict-stands", + ), + ], +) +async def test_a_rollout_timeout_in_teardown_returns_and_lets_teardown_finish( + held: str, reward: float, error: str, events: list[str] +) -> None: + seen = [] + release = asyncio.Event() + actor = Environment("actor") + judge = Environment("judge") + + @actor.shutdown + async def actor_down() -> None: + if held == "actor": + await release.wait() + seen.append("actor down") + + @judge.initialize + async def judge_up() -> None: + seen.append("judge up") + + @judge.shutdown + async def judge_down() -> None: + if held == "judge": + await release.wait() + seen.append("judge down") + + @actor.template() + async def solve(): + answer = yield "answer secret" + yield {"score": 0.25, "answer": answer} + + @judge.template() + async def verify(): + result = yield "" + yield 1.0 if result["answer"] == "secret" else 0.0 + + task = solve() + task.verifier = verify() + job = await task.run(ScriptedAgent("secret"), rollout_timeout=0.3) + release.set() + + (run,) = job.runs + assert (run.reward, run.trace.stop_reason, run.trace.error) == (reward, "timeout", error) + await eventually(lambda: seen == events) diff --git a/tests/fixtures/Dockerfile b/tests/fixtures/Dockerfile new file mode 100644 index 000000000..c0235a4e5 --- /dev/null +++ b/tests/fixtures/Dockerfile @@ -0,0 +1,10 @@ +# The fixture environments as containers: this checkout's SDK and one fixture env, +# served the way a deployed environment image serves its own. +FROM python:3.12-slim +COPY *.whl /tmp/ +RUN pip install --no-cache-dir /tmp/*.whl && rm /tmp/*.whl +COPY envs/ /hud/ +ARG FIXTURE +ENV FIXTURE=${FIXTURE} +EXPOSE 8765 +CMD ["sh", "-c", "exec hud serve \"/hud/$FIXTURE.py\" --host 0.0.0.0 --port 8765"] diff --git a/hud/integrations/harbor/tests/__init__.py b/tests/fixtures/__init__.py similarity index 100% rename from hud/integrations/harbor/tests/__init__.py rename to tests/fixtures/__init__.py diff --git a/tests/fixtures/agents.py b/tests/fixtures/agents.py new file mode 100644 index 000000000..0c52359e4 --- /dev/null +++ b/tests/fixtures/agents.py @@ -0,0 +1,53 @@ +"""Agents with fixed answers: an agent whose model is a function of the prompt.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from hud.agents.base import Agent + +if TYPE_CHECKING: + from collections.abc import Callable + + from hud.eval import Run + + +class ScriptedAgent(Agent): + """Answers every run with ``answer``, or ``answer(prompt)`` when it is callable. + + ``delay`` waits before answering; ``fail_before`` raises instead of answering + and ``fail_after`` raises after the answer is recorded. ``linger`` keeps the + agent running that long after answering, so a deadline can land on it. + """ + + def __init__( + self, + answer: str | Callable[[str], str] = "", + *, + delay: float = 0.0, + fail_before: BaseException | None = None, + fail_after: BaseException | None = None, + linger: float = 0.0, + ) -> None: + super().__init__() + self._answer: Callable[[str], str] = ( + (lambda _prompt: answer) if isinstance(answer, str) else answer + ) + self._delay = delay + self._fail_before = fail_before + self._fail_after = fail_after + self._linger = linger + self.prompts: list[str] = [] + + async def __call__(self, run: Run) -> None: + self.prompts.append(run.prompt_text) + if self._delay: + await asyncio.sleep(self._delay) + if self._fail_before is not None: + raise self._fail_before + run.trace.content = self._answer(run.prompt_text) + if self._linger: + await asyncio.sleep(self._linger) + if self._fail_after is not None: + raise self._fail_after diff --git a/tests/fixtures/environments.py b/tests/fixtures/environments.py new file mode 100644 index 000000000..24906329d --- /dev/null +++ b/tests/fixtures/environments.py @@ -0,0 +1,224 @@ +"""Environments built in the test process: ``lab`` (one template per way a task starts +or grades), the ``actor`` and ``judge`` verifier pair, and ``workspace_env`` (a real +workspace over SSH); ``served`` connects a client to one and ``run_task`` rolls an +agent out on one. Environments loaded from a file are in ``tests/fixtures/envs``.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +from hud import Environment +from hud.clients import connect +from hud.eval import LocalRuntime, Task, rollout +from tests.support.machine import write_files + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Mapping + from pathlib import Path + + from hud.agents.base import Agent + from hud.capabilities import Capability + from hud.clients import HudClient + from hud.eval import Run + + +SUMS_SOURCE = """ +from hud import Environment + +env = Environment("lab") + + +@env.template() +async def add(a: int, b: int): + answer = yield f"add {a} {b}" + yield 1.0 if answer == str(a + b) else 0.0 +""" + + +def solve(prompt: str) -> str: + """The right answer to an ``add`` prompt.""" + _, a, b = prompt.split() + return str(int(a) + int(b)) + + +def lab(events: list[str] | None = None) -> Environment: + """The ``lab`` environment: one template per way a task can start or grade.""" + log = events if events is not None else [] + env = Environment("lab") + + @env.initialize + async def initialize() -> None: + log.append("initialize") + + @env.shutdown + async def shutdown() -> None: + log.append("shutdown") + + @env.template() + async def add(a: int, b: int): + log.append(f"start add {a} {b}") + try: + answer = yield f"add {a} {b}" + log.append(f"grade {answer}") + yield 1.0 if answer == str(a + b) else 0.0 + finally: + log.append("end add") + + @env.template() + async def chat(messages: list[dict[str, Any]]): + yield messages + yield 1.0 + + @env.template() + async def silent(): + yield None + yield 1.0 + + @env.template() + async def claim(published: Any): + yield {"prompt": "go", "bindings": published} + yield 1.0 + + @env.template() + async def grade_raises(): + yield "go" + raise RuntimeError("grader exploded") + yield 0.0 + + @env.template() + async def frame(result: dict[str, Any]): + yield "go" + yield result + + @env.template() + async def hang_grading(): + try: + yield "go" + await asyncio.Event().wait() + yield 1.0 + finally: + log.append("grading cancelled") + + @env.template() + async def hang_start(): + try: + await asyncio.Event().wait() + yield "never" + yield 0.0 + finally: + log.append("start cancelled") + + @env.template() + async def large(criteria: str): + log.append("large started") + yield f"{len(criteria)}" + yield 1.0 + + return env + + +def minted(events: list[str] | None = None) -> Task: + """A row minted by a live ``lab`` environment, which places it by default.""" + env = lab(events) + + @env.template() + async def total(a: int, b: int): + answer = yield f"add {a} {b}" + yield 1.0 if answer == str(a + b) else 0.0 + + return total(a=2, b=3) + + +def actor(env: Environment | None = None, *, grade: str = "score") -> Environment: + """Declare the actor side of a verifier pair: ``solve`` grades 0.25 and carries the answer. + + It goes on ``env``, or on a new ``actor`` environment. ``grade="raise"`` makes + its grading raise and ``grade="scoreless"`` yields a frame without a score. + """ + env = env or Environment("actor") + + @env.template() + async def solve(): + answer = yield "answer secret" + if grade == "raise": + raise RuntimeError("actor grade exploded") + yield {"score": 0.25, "answer": answer} if grade == "score" else {"answer": answer} + + return env + + +def judge(env: Environment | None = None, *, verdict: str = "check") -> Environment: + """Declare the judge side: ``verify`` pays 1.0 when the actor's answer was ``secret``. + + It goes on ``env``, or on a new ``judge`` environment. ``verdict`` picks a + failure instead: ``"raise"``, ``"scoreless"`` or ``"zero"``. + """ + env = env or Environment("judge") + + @env.template() + async def verify(): + result = yield "" + if verdict == "raise": + raise RuntimeError("verifier exploded") + if verdict == "scoreless": + yield {"verdict": "pass"} + elif verdict == "zero": + yield 0.0 + else: + yield 1.0 if result["answer"] == "secret" else 0.0 + + return env + + +PROMPT = "Do the task." + + +def workspace_env( + root: Path, + *, + files: Mapping[str, bytes | str] | None = None, + passes: Callable[[Path], bool] = lambda _root: True, + capabilities: tuple[Capability, ...] = (), + name: str = "ws", + settle: float = 0.0, + **workspace: Any, +) -> Environment: + """An environment serving ``root`` over SSH with one template, ``task``. + + ``files`` seed the workspace before each run; the task scores 1.0 when + ``passes(root)`` holds ``settle`` seconds after the agent finishes, long + enough for anything the agent left running to show. ``workspace`` goes to + :meth:`Environment.workspace` (``env=`` for the shell environment). + """ + env = Environment(name, capabilities=capabilities) + env.workspace(root, guest_path=str(root), **workspace) + + @env.initialize + async def seed() -> None: + write_files(root, files or {}) + + @env.template() + async def task(prompt: str = PROMPT): + yield prompt + await asyncio.sleep(settle) + yield 1.0 if passes(root) else 0.0 + + return env + + +@asynccontextmanager +async def served(env: Environment, *, row: Task | None = None) -> AsyncIterator[HudClient]: + """Serve ``env`` on loopback and yield a control-channel client connected to it.""" + task = row or Task(env=env.name, id="scenario") + async with LocalRuntime(env)(task) as runtime, connect(runtime) as client: + yield client + + +async def run_task(env: Environment, agent: Agent, *, prompt: str = PROMPT, **options: Any) -> Run: + """Roll ``agent`` out on ``env``'s ``task`` in this process.""" + args = {} if prompt == PROMPT else {"prompt": prompt} + return await rollout( + Task(env=env.name, id="task", args=args), agent, runtime=LocalRuntime(env), **options + ) diff --git a/tests/fixtures/envs/__init__.py b/tests/fixtures/envs/__init__.py new file mode 100644 index 000000000..6ce408a4d --- /dev/null +++ b/tests/fixtures/envs/__init__.py @@ -0,0 +1,81 @@ +"""Fixture environments: each reward is 1 only when the path it names worked. + +Every module is a real environment source, so any placement can serve it: in +this process, in a child process, in a container, or attached by address. +Scenarios load them by path with ``source``. Modules with a ``tasks`` list also +load the way a user's tasks file does, through ``hud eval`` or ``Taskset.from_file``. + +``capable_model`` answers a fixture the way a capable model would, from what the +agent actually sent. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import TYPE_CHECKING + +from tests.support import call, say + +if TYPE_CHECKING: + from collections.abc import Callable + + from tests.support import ModelRequest, Turn + +HERE = Path(__file__).parent + + +def source(name: str) -> Path: + """The path of fixture environment ``name``.""" + path = HERE / f"{name}.py" + assert path.is_file(), path + return path + + +def repeat_word(request: ModelRequest) -> Turn: + """Answer ``echo`` and ``verifier``: the word the prompt asks for.""" + match = re.search(r"Reply with: (\S+)", request.prompt) + return say(match.group(1) if match else "no word in the prompt") + + +def collect_digits() -> Callable[[ModelRequest], Turn]: + """Answer ``tools``: call next_digit until it says done, then send every digit. + + Responses chains requests and sends only the newest tool result, so the policy + keeps the digits it has seen, as the model's own context would. + """ + digits: list[str] = [] + + def policy(request: ModelRequest) -> Turn: + if request.tool_results: + latest = re.search(r"\b(\d|done)\b", request.tool_results[-1]) + assert latest is not None, request.tool_results[-1] + if latest.group(1) == "done": + return say("".join(digits)) + digits.append(latest.group(1)) + (tool,) = [name for name in request.tools if name.endswith("next_digit")] + return call(tool) + + return policy + + +def read_token(request: ModelRequest) -> Turn: + """Answer ``secret``: cat the file the prompt names, then send the token it printed.""" + if not request.tool_results: + path = re.search(r"Read the file (\S+) and", request.prompt) + assert path is not None, request.prompt + return call("bash", command=f"cat {path.group(1)}") + token = re.search(r"^[0-9a-f]{16}$", request.tool_results[-1], re.MULTILINE) + return say(token.group(0) if token else "no token in the tool result") + + +def capable_model(fixture: str) -> Callable[[ModelRequest], Turn]: + """A scripted model that takes fixture environment ``fixture`` to its reward.""" + match fixture: + case "echo" | "verifier": + return repeat_word + case "secret": + return read_token + case "tools": + return collect_digits() + raise ValueError(f"no scripted model answers {fixture!r}") diff --git a/tests/fixtures/envs/broken.py b/tests/fixtures/envs/broken.py new file mode 100644 index 000000000..4eaa4918b --- /dev/null +++ b/tests/fixtures/envs/broken.py @@ -0,0 +1,3 @@ +"""A source that fails as it loads: a container serving it exits before it serves.""" + +raise ImportError("broken fixture: no module named bugs") diff --git a/tests/fixtures/envs/echo.py b/tests/fixtures/envs/echo.py new file mode 100644 index 000000000..71ba0db4a --- /dev/null +++ b/tests/fixtures/envs/echo.py @@ -0,0 +1,54 @@ +"""The agent repeats a word: a reward of 1 proves the prompt and the answer both arrived.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +from hud import Environment + +env = Environment("echo") + + +def record(event: str) -> None: + """Append ``event`` and this process's id to the lifecycle log the test names, if any.""" + log = os.environ.get("HUD_FIXTURE_LIFECYCLE") + if log: + with Path(log).open("a") as out: + out.write(f"{event} {os.getpid()}\n") + + +DAEMONS: list[asyncio.subprocess.Process] = [] + + +@env.initialize +async def record_start() -> None: + """Log the start, and start a daemon the way an env starts a browser or a display.""" + record("start") + if os.environ.get("HUD_FIXTURE_LIFECYCLE"): + DAEMONS.append(await asyncio.create_subprocess_exec("sleep", "600")) + record(f"daemon {DAEMONS[-1].pid}") + + +@env.shutdown +async def record_stop() -> None: + for daemon in DAEMONS: + daemon.kill() + await daemon.wait() + record("stop") + + +@env.template() +async def echo(word: str): + answer = yield f"Reply with: {word}" + yield 1.0 if answer == word else 0.0 + + +@env.template(returns=int) +async def count(word: str): + answer = yield f"How many letters are in {word}?" + yield 1.0 if answer.content == len(word) else 0.0 + + +tasks = [echo(word="tangerine")] diff --git a/tests/fixtures/envs/faulty.py b/tests/fixtures/envs/faulty.py new file mode 100644 index 000000000..b9cf08b88 --- /dev/null +++ b/tests/fixtures/envs/faulty.py @@ -0,0 +1,71 @@ +"""One template per point a task can fail: its start, its grade, or a grade that never comes.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +from hud import Environment + +env = Environment("faulty") + + +def record(event: str) -> None: + """Append ``event`` and this process's id to the lifecycle log the test names, if any.""" + log = os.environ.get("HUD_FIXTURE_LIFECYCLE") + if log: + with Path(log).open("a") as out: + out.write(f"{event} {os.getpid()}\n") + + +DAEMONS: list[asyncio.subprocess.Process] = [] + + +@env.initialize +async def record_start() -> None: + """Log the start, and start a daemon the way an env starts a browser or a display.""" + record("start") + if os.environ.get("HUD_FIXTURE_LIFECYCLE"): + DAEMONS.append(await asyncio.create_subprocess_exec("sleep", "600")) + record(f"daemon {DAEMONS[-1].pid}") + + +@env.shutdown +async def record_stop() -> None: + for daemon in DAEMONS: + daemon.kill() + await daemon.wait() + record("stop") + + +@env.template() +async def start_raises(): + raise RuntimeError("start exploded") + yield "never" + + +@env.template() +async def grade_raises(): + yield "go" + raise RuntimeError("grader exploded") + yield 0.0 + + +@env.template() +async def scoreless(): + yield "go" + yield {"done": True} + + +@env.template() +async def bool_score(): + yield "go" + yield {"score": True} + + +@env.template() +async def hang_grading(): + yield "go" + await asyncio.Event().wait() + yield 1.0 diff --git a/tests/fixtures/envs/picture.py b/tests/fixtures/envs/picture.py new file mode 100644 index 000000000..2cf63d27d --- /dev/null +++ b/tests/fixtures/envs/picture.py @@ -0,0 +1,53 @@ +"""A tool that shows a colour: only a model that saw the image can name it. + +The prompt never says which colour the task chose, so with a real model a reward +of 1 proves the tool's image reached it. +""" + +from __future__ import annotations + +import base64 +import io +from contextlib import AsyncExitStack + +import mcp.types as mcp_types +from fastmcp import FastMCP +from PIL import Image + +from hud import Environment +from hud.capabilities import Capability +from tests.support import serve_asgi + +server = FastMCP(name="picture") +shown: list[str] = [] + + +@server.tool +def look() -> list[mcp_types.ContentBlock]: + """Show the picture.""" + buffer = io.BytesIO() + Image.new("RGB", (64, 64), shown[-1]).save(buffer, format="PNG") + data = base64.b64encode(buffer.getvalue()).decode() + return [mcp_types.ImageContent(type="image", data=data, mimeType="image/png")] + + +env = Environment("picture") +stack = AsyncExitStack() + + +@env.initialize +async def serve_picture() -> None: + port = await stack.enter_async_context(serve_asgi(server.http_app(path="/mcp"))) + env.add_capability(Capability.mcp(name="picture", url=f"http://127.0.0.1:{port}/mcp")) + + +@env.shutdown +async def stop_picture() -> None: + await stack.aclose() + + +@env.template() +async def name_the_colour(colour: str): + shown.append(colour) + answer = yield "Call the look tool, then reply with the name of the colour it shows." + yield 1.0 if colour in answer.lower() else 0.0 diff --git a/tests/fixtures/envs/screen.py b/tests/fixtures/envs/screen.py new file mode 100644 index 000000000..ba1bfc15c --- /dev/null +++ b/tests/fixtures/envs/screen.py @@ -0,0 +1,49 @@ +"""A screen with a target: a reward of 1 proves the agent's click landed on it. + +A click within ``SLACK`` pixels of the target counts, so a real model that rounds +its coordinates still lands. +""" + +from __future__ import annotations + +from contextlib import AsyncExitStack + +from hud import Environment +from hud.capabilities import Capability +from tests.support import fake_screen + +WIDTH, HEIGHT = 200, 100 +TARGET = (100, 50) +SLACK = 2 + +env = Environment("screen") +stack = AsyncExitStack() +screens = [] + + +@env.initialize +async def show_screen() -> None: + screen = await stack.enter_async_context(fake_screen(width=WIDTH, height=HEIGHT)) + screens.append(screen) + env.add_capability(Capability.rfb(name="screen", url=screen.url)) + + +@env.shutdown +async def hide_screen() -> None: + await stack.aclose() + screens.clear() + + +@env.template() +async def click_target(): + (screen,) = screens + already = len(screen.pointer_events()) + yield ( + f"Use the computer to left-click the pixel at {TARGET[0]},{TARGET[1]} on the " + f"{WIDTH}x{HEIGHT} screen. Take the action; do not describe it." + ) + pressed = [ + (event.x, event.y) for event in screen.pointer_events()[already:] if event.buttons & 1 + ] + landed = all(abs(x - TARGET[0]) <= SLACK and abs(y - TARGET[1]) <= SLACK for x, y in pressed) + yield 1.0 if pressed and landed else 0.0 diff --git a/tests/fixtures/envs/secret.py b/tests/fixtures/envs/secret.py new file mode 100644 index 000000000..443326ab6 --- /dev/null +++ b/tests/fixtures/envs/secret.py @@ -0,0 +1,32 @@ +"""A token only the workspace holds: the agent can answer only by reading it through a shell.""" + +from __future__ import annotations + +import os +import secrets +import shutil +import tempfile +from pathlib import Path + +from hud import Environment + +ROOT = Path(tempfile.gettempdir()) / f"hud-fixture-secret-{os.getpid()}" + +env = Environment("secret") +env.workspace(ROOT, guest_path=str(ROOT)) + + +@env.shutdown +async def remove_workspace() -> None: + shutil.rmtree(ROOT, ignore_errors=True) + + +@env.template(id="read-secret") +async def read_secret(): + token = secrets.token_hex(8) + (ROOT / "secret.txt").write_text(token, encoding="utf-8") + answer = yield f"Read the file {ROOT / 'secret.txt'} and reply with its contents only." + yield 1.0 if (answer or "").strip() == token else 0.0 + + +tasks = [read_secret()] diff --git a/tests/fixtures/envs/servers.py b/tests/fixtures/envs/servers.py new file mode 100644 index 000000000..77f4c6cb1 --- /dev/null +++ b/tests/fixtures/envs/servers.py @@ -0,0 +1,56 @@ +"""Two MCP servers with a tool of the same name: a reward of 1 proves each call reached its own. + +With more than one server, the agent qualifies tool names by server, so the +model must tell ``database__lookup`` from ``search__lookup``. +""" + +from __future__ import annotations + +import secrets +from contextlib import AsyncExitStack + +from fastmcp import FastMCP + +from hud import Environment +from hud.capabilities import Capability +from tests.support import serve_asgi + +answers: dict[str, str] = {} + + +def server(name: str) -> FastMCP: + mcp = FastMCP(name) + + @mcp.tool + def lookup(key: str) -> str: + """Look a key up.""" + return answers[f"{name}:{key}"] + + return mcp + + +env = Environment("servers") +stack = AsyncExitStack() + + +@env.initialize +async def serve_tools() -> None: + for name in ("database", "search"): + port = await stack.enter_async_context(serve_asgi(server(name).http_app(path="/mcp"))) + env.add_capability(Capability.mcp(name=name, url=f"http://127.0.0.1:{port}/mcp")) + + +@env.shutdown +async def stop_tools() -> None: + await stack.aclose() + + +@env.template() +async def look_up_both(): + answers.update({"database:alpha": secrets.token_hex(4), "search:beta": secrets.token_hex(4)}) + answer = yield ( + "Look up the key 'alpha' in the database and the key 'beta' in the search index, " + "then reply with both results." + ) + found = answers["database:alpha"] in answer and answers["search:beta"] in answer + yield 1.0 if found else 0.0 diff --git a/tests/fixtures/envs/sessions.py b/tests/fixtures/envs/sessions.py new file mode 100644 index 000000000..5417a523b --- /dev/null +++ b/tests/fixtures/envs/sessions.py @@ -0,0 +1,36 @@ +"""An actor that leaves its answer in its session's directory, and a verifier that reads it there. + +Placed in two containers, a reward of 1 proves the runtime moved the actor's +session files into the verifier's container. +""" + +from __future__ import annotations + +from pathlib import Path + +from hud import Environment +from hud.environment.env import current_session_id + +env = Environment("sessions") + + +def session() -> Path: + return Path("/runtime/sessions") / str(current_session_id.get()) + + +@env.template() +async def solve(link: bool = False): + answer = yield "Reply with: secret" + directory = session() + directory.mkdir(parents=True, exist_ok=True) + (directory / "work.txt").write_text(str(answer)) + if link: + (directory / "escape").symlink_to("/etc/passwd") + yield {"score": 0.0, "answer": answer} + + +@env.template() +async def verify(): + yield "" + work = session() / "work.txt" + yield 1.0 if work.exists() and work.read_text() == "secret" else 0.0 diff --git a/tests/fixtures/envs/stalls.py b/tests/fixtures/envs/stalls.py new file mode 100644 index 000000000..bb5ef5f84 --- /dev/null +++ b/tests/fixtures/envs/stalls.py @@ -0,0 +1,50 @@ +"""An environment that never finishes starting: its initialize hook waits forever.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +from hud import Environment + +env = Environment("stalls") + + +def record(event: str) -> None: + """Append ``event`` and this process's id to the lifecycle log the test names, if any.""" + log = os.environ.get("HUD_FIXTURE_LIFECYCLE") + if log: + with Path(log).open("a") as out: + out.write(f"{event} {os.getpid()}\n") + + +DAEMONS: list[asyncio.subprocess.Process] = [] + + +@env.initialize +async def record_start() -> None: + """Log the start, and start a daemon the way an env starts a browser or a display.""" + record("start") + if os.environ.get("HUD_FIXTURE_LIFECYCLE"): + DAEMONS.append(await asyncio.create_subprocess_exec("sleep", "600")) + record(f"daemon {DAEMONS[-1].pid}") + + +@env.shutdown +async def record_stop() -> None: + for daemon in DAEMONS: + daemon.kill() + await daemon.wait() + record("stop") + + +@env.initialize +async def wait_forever() -> None: + await asyncio.Event().wait() + + +@env.template() +async def never(): + yield "never" + yield 0.0 diff --git a/tests/fixtures/envs/tools.py b/tests/fixtures/envs/tools.py new file mode 100644 index 000000000..5a3590114 --- /dev/null +++ b/tests/fixtures/envs/tools.py @@ -0,0 +1,56 @@ +"""A code revealed one digit per tool call: a reward of 1 proves state was kept across calls.""" + +from __future__ import annotations + +import re +import secrets +from contextlib import AsyncExitStack + +from fastmcp import FastMCP + +from hud import Environment +from hud.capabilities import Capability +from tests.support import serve_asgi + +server = FastMCP(name="digits") +code: list[str] = [] +revealed: list[str] = [] + + +@server.tool +def next_digit() -> str: + """Return the next digit of the code, or "done" once every digit is revealed.""" + if len(revealed) == len(code): + return "done" + revealed.append(code[len(revealed)]) + return revealed[-1] + + +env = Environment("tools") +stack = AsyncExitStack() + + +@env.initialize +async def serve_tools() -> None: + port = await stack.enter_async_context(serve_asgi(server.http_app(path="/mcp"))) + env.add_capability(Capability.mcp(name="digits", url=f"http://127.0.0.1:{port}/mcp")) + + +@env.shutdown +async def stop_tools() -> None: + await stack.aclose() + + +@env.template() +async def collect_code(length: int = 4): + code[:] = [str(secrets.randbelow(10)) for _ in range(length)] + revealed.clear() + answer = yield ( + 'Call next_digit until it returns "done", then reply with every digit it returned, ' + "in order, as one string." + ) + # A real model may wrap the code in prose; the digits it gives are what count. + yield 1.0 if re.sub(r"\D", "", answer) == "".join(code) and revealed == code else 0.0 + + +tasks = [collect_code()] diff --git a/tests/fixtures/envs/verifier.py b/tests/fixtures/envs/verifier.py new file mode 100644 index 000000000..55d2c77df --- /dev/null +++ b/tests/fixtures/envs/verifier.py @@ -0,0 +1,24 @@ +"""An actor that grades itself 0 and a verifier that reads its answer: 1 means the verifier won.""" + +from __future__ import annotations + +from hud import Environment + +actor = Environment("actor") +judge = Environment("judge") + + +@actor.template(id="relay") +async def relay(word: str): + answer = yield f"Reply with: {word}" + yield {"score": 0.0, "answer": answer} + + +@judge.template(id="check-relay") +async def check_relay(expected: str): + actor_result = yield "" + yield 1.0 if (actor_result["answer"] or "").strip() == expected else 0.0 + + +tasks = [relay(word="lighthouse")] +tasks[0].verifier = check_relay(expected="lighthouse") diff --git a/tests/fixtures/envs/workspace.py b/tests/fixtures/envs/workspace.py new file mode 100644 index 000000000..a546620b9 --- /dev/null +++ b/tests/fixtures/envs/workspace.py @@ -0,0 +1,30 @@ +"""A file to fix in a real workspace: a reward of 1 proves the agent's own tools edited it.""" + +from __future__ import annotations + +import shutil +import tempfile +from pathlib import Path + +from hud import Environment + +ROOT = Path(tempfile.mkdtemp(prefix="hud-fixture-workspace-")) + +env = Environment("workspace") +env.workspace(ROOT, guest_path=str(ROOT)) + + +@env.shutdown +async def remove_workspace() -> None: + shutil.rmtree(ROOT, ignore_errors=True) + + +@env.template() +async def fix_the_file(): + notes = ROOT / "notes.txt" + notes.write_text("one\ntwo\nthree\n") + yield ( + f"The file {notes} has a line that reads 'two'. Change that line to '2' " + "and leave the other lines as they are." + ) + yield 1.0 if notes.read_text() == "one\n2\nthree\n" else 0.0 diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/__init__.py b/tests/graders/__init__.py similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/__init__.py rename to tests/graders/__init__.py diff --git a/tests/graders/test_graders.py b/tests/graders/test_graders.py new file mode 100644 index 000000000..82be7ddb8 --- /dev/null +++ b/tests/graders/test_graders.py @@ -0,0 +1,430 @@ +"""Graders as task authors use them: a template grades the agent's answer with ``combine``, +the bash grader, the LLM judge or a grader of its own, and the run's reward and +evaluation show the result.""" + +from __future__ import annotations + +import time +import warnings +from typing import TYPE_CHECKING, Any + +import pytest + +from hud import Environment +from hud.graders import ( + BashGrader, + EvaluationResult, + Grader, + LLMJudgeGrader, + SubScore, + combine, + combine_all, + combine_any, + exact_match, +) +from tests.fixtures.agents import ScriptedAgent +from tests.support import say + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import HudEnv, ModelRequest, Models, Turn + +ANSWER = "Paris is the capital." + + +def leaf(name: str, weight: float, value: float) -> dict[str, Any]: + return {"name": name, "weight": weight, "value": value, "children": None} + + +async def test_repeated_graders_are_named_apart_and_weighted_into_the_reward() -> None: + env = Environment("graders") + + @env.template() + async def check(): + yield "go" + yield await combine( + BashGrader.grade(weight=0.3, command="true"), + BashGrader.grade(weight=0.3, command="true"), + SubScore(name="BashGrader-1", value=0.0, weight=0.4), + ) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert run.reward == pytest.approx(0.6) + assert run.evaluation["subscores"] == [ + leaf("BashGrader-2", 0.3, 1.0), + leaf("BashGrader-3", 0.3, 1.0), + leaf("BashGrader-1", 0.4, 0.0), + ] + + +async def test_any_and_all_combine_their_children_into_one_weighted_subscore() -> None: + env = Environment("graders") + + @env.template() + async def check(): + answer = yield "Name the capital of France." + either = combine_any( + weight=0.5, subscores=[SubScore(name="a", value=0.0), SubScore(name="b", value=1.0)] + ) + both = combine_all( + weight=0.25, + subscores=[SubScore(name="lint", value=1.0), SubScore(name="types", value=0.0)], + name="checks", + ) + exact = SubScore(name="format", value=exact_match(answer, ANSWER), weight=0.25) + yield await combine(either, both, exact) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert run.reward == pytest.approx(0.75) + assert run.evaluation["subscores"] == [ + { + "name": "any", + "weight": 0.5, + "value": 1.0, + "children": [leaf("a", 1.0, 0.0), leaf("b", 1.0, 1.0)], + }, + { + "name": "checks", + "weight": 0.25, + "value": 0.0, + "children": [leaf("lint", 1.0, 1.0), leaf("types", 1.0, 0.0)], + }, + leaf("format", 0.25, 1.0), + ] + + +@pytest.mark.parametrize( + ("correct", "reward"), [(1.0, 0.8), (0.0, -0.2)], ids=["passing", "failing-goes-negative"] +) +async def test_a_negative_weight_is_a_penalty_taken_off_the_reward( + correct: float, reward: float +) -> None: + env = Environment("graders") + + @env.template() + async def check(): + yield "go" + yield await combine( + SubScore(name="right", value=correct, weight=1.0), + SubScore(name="penalty", value=1.0, weight=-0.2), + ) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert run.reward == pytest.approx(reward) + assert [s["weight"] for s in run.evaluation["subscores"]] == [1.0, -0.2] + + +async def test_weights_not_summing_to_one_are_normalized_with_a_warning_and_penalties_kept() -> ( + None +): + env = Environment("graders") + + @env.template() + async def check(): + yield "go" + yield await combine( + SubScore(name="a", value=1.0, weight=0.5), + SubScore(name="b", value=1.0, weight=0.3), + SubScore(name="c", value=0.0, weight=0.3), + SubScore(name="penalty", value=1.0, weight=-0.1), + ) + + with pytest.warns(UserWarning, match="grader weights sum to 1.1000, not 1.0"): + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert run.reward == pytest.approx(0.8 / 1.1 - 0.1) + assert [s["weight"] for s in run.evaluation["subscores"]] == pytest.approx( + [0.5 / 1.1, 0.3 / 1.1, 0.3 / 1.1, -0.1] + ) + + +async def test_subscores_that_do_not_match_the_reward_warn() -> None: + env = Environment("graders") + + @env.template() + async def check(): + yield "go" + yield EvaluationResult(reward=1.0, subscores=[SubScore(name="a", value=0.5, weight=1.0)]) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert run.reward == 1.0 + assert any(issubclass(warning.category, UserWarning) for warning in caught) + + +async def test_a_custom_grader_returns_a_number_a_pair_or_a_subscore_and_keeps_its_inputs() -> None: + class Scaled(Grader): + name = "scaled" + + @classmethod + async def compute_score(cls, **kwargs: Any) -> Any: + return 0.75 + + class Explained(Grader): + name = "explained" + + @classmethod + async def compute_score(cls, **kwargs: Any) -> Any: + return 0.5, {"source": "pair"} + + class Detailed(Grader): + name = "detailed" + + @classmethod + async def compute_score(cls, **kwargs: Any) -> Any: + return SubScore(name="ignored", value=1.0, info={"reason": "did it"}) + + env = Environment("graders") + + @env.template() + async def check(): + yield "go" + yield await combine( + Scaled.grade(weight=0.25, payload=object()), + Explained.grade(weight=0.25, level=2), + Detailed.grade(weight=0.25, name="renamed"), + SubScore.model_validate( + {"name": "legacy", "value": 1.0, "weight": 0.25, "metadata": {"from": "metadata"}} + ), + ) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert run.reward == pytest.approx(0.8125) + assert {s["name"]: s["info"] for s in run.grade.raw["subscores"]} == { + "scaled": {"_parameters": {"payload": ""}}, + "explained": {"source": "pair", "_parameters": {"level": 2}}, + "renamed": {"reason": "did it", "_parameters": {}}, + "legacy": {"from": "metadata"}, + } + + +async def test_graders_passed_to_combine_run_at_the_same_time(tmp_path: Path) -> None: + env = Environment("graders") + wait = "for _ in $(seq 50); do [ -e {other} ] && exit 0; sleep 0.1; done; exit 1" + + @env.template() + async def check(): + yield "go" + yield await combine( + BashGrader.grade( + weight=0.5, command=f"touch {tmp_path}/a; " + wait.format(other=tmp_path / "b") + ), + BashGrader.grade( + weight=0.5, command=f"touch {tmp_path}/b; " + wait.format(other=tmp_path / "a") + ), + ) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert run.reward == 1.0 + + +@pytest.mark.parametrize( + ("command", "limit", "value", "info"), + [ + ("echo oops >&2; false", None, 0.0, {"exit_code": 1, "stdout": "", "stderr": "oops\n"}), + ( + "echo started; sleep 30 & exit 0", + None, + 1.0, + {"exit_code": 0, "stdout": "started\n", "stderr": ""}, + ), + ( + "echo progress; echo stuck >&2; sleep 30", + 1, + 0.0, + {"exit_code": None, "stdout": "progress\n", "stderr": "stuck\n", "timed_out": True}, + ), + ("yes hud | head -c 500000", None, 1.0, {"exit_code": 0, "stdout": "hud\n" * 125_000}), + ], + ids=[ + "a-failing-command", + "a-background-child-does-not-hold-the-grade", + "an-overrun-keeps-its-output", + "large-output-does-not-block", + ], +) +async def test_the_bash_grader_scores_by_exit_code_and_reports_what_ran( + command: str, limit: float | None, value: float, info: dict[str, Any] +) -> None: + env = Environment("graders") + + @env.template() + async def check(): + yield "go" + yield await combine(BashGrader.grade(weight=1.0, command=command, timeout_seconds=limit)) + + started = time.monotonic() + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + (subscore,) = run.grade.raw["subscores"] + assert run.reward == value + assert {key: subscore["info"][key] for key in info} == info + assert time.monotonic() - started < 15 + + +async def test_the_bash_grader_runs_in_the_directory_it_is_given(tmp_path: Path) -> None: + (tmp_path / "marker.txt").write_text("here") + env = Environment("graders") + + @env.template() + async def check(): + yield "go" + yield await combine( + BashGrader.grade(weight=1.0, command="cat marker.txt", cwd=str(tmp_path)) + ) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert (run.reward, run.grade.raw["subscores"][0]["info"]["stdout"]) == (1.0, "here") + + +VERDICTS = { + "json": '{"criterion_status": "MET", "explanation": "plainly stated"}', + "fenced": '```json\n{"criterion_status": "UNMET", "explanation": "no source"}\n```', + "prose": "Verdict: UNMET, although the word MET appears here.", + "prose-met": "The criterion is MET.", +} + + +@pytest.mark.parametrize( + ("criteria", "verdicts", "reward", "children"), + [ + ( + [["names Paris", 1.0], ["cites a source", 2.0], ["invents facts", -1.0]], + {"names Paris": "json", "cites a source": "fenced", "invents facts": "prose"}, + 1 / 3, + [ + ("names Paris", 1.0, 1.0, "plainly stated"), + ("cites a source", 2.0, 0.0, "no source"), + ("invents facts", -1.0, 0.0, VERDICTS["prose"]), + ], + ), + ( + [["invents facts", -1.0]], + {"invents facts": "prose-met"}, + 0.0, + [("invents facts", -1.0, 1.0, "The criterion is MET.")], + ), + ( + [["invents facts", -1.0]], + {"invents facts": "prose"}, + 1.0, + [("invents facts", -1.0, 0.0, VERDICTS["prose"])], + ), + ], + ids=["json-fenced-and-prose-verdicts", "a-penalty-that-is-met", "a-penalty-that-is-not-met"], +) +async def test_the_judge_asks_once_per_criterion_and_weighs_each_verdict( + criteria: list[list[Any]], + verdicts: dict[str, str], + reward: float, + children: list[tuple[str, float, float, str]], + models: Models, + hud_env: HudEnv, +) -> None: + hud_env.set(HUD_API_KEY="k") + + def judge(request: ModelRequest) -> Turn: + prompt = request.body["messages"][1]["content"] + criterion = prompt.split("\n", 1)[1].split("\n", 1)[0] + return say(VERDICTS[verdicts[criterion]]) + + models.respond(judge) + env = Environment("graders") + + @env.template() + async def check(): + answer = yield "Name the capital of France." + judged = LLMJudgeGrader.grade( + weight=1.0, + answer=answer, + criteria=[(requirement, weight) for requirement, weight in criteria], + question="Name the capital of France.", + ) + yield await combine(judged) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + (judge,) = run.grade.raw["subscores"] + assert run.reward == pytest.approx(reward) + assert [ + (c["name"], c["weight"], c["value"], c["info"]["reason"]) for c in judge["children"] + ] == children + assert [request.credential for request in models.requests()] == ["k"] * len(criteria) + + +async def test_a_judge_with_no_criteria_asks_no_model(models: Models, hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="k") + env = Environment("graders") + + @env.template() + async def check(): + answer = yield "go" + yield await combine(LLMJudgeGrader.grade(weight=1.0, answer=answer, criteria=[])) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert (run.reward, models.requests()) == (0.0, []) + + +async def test_a_judge_without_a_hud_api_key_fails_the_grade(models: Models) -> None: + env = Environment("graders") + + @env.template() + async def check(): + answer = yield "go" + yield await combine(LLMJudgeGrader.grade(weight=1.0, answer=answer, criteria=["x"])) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert run.trace.status == "error" + assert "HUD_API_KEY is required for HUD gateway clients" in (run.trace.error or "") + assert models.requests() == [] + + +@pytest.mark.parametrize( + ("weights", "refusal"), + [ + ([-1.0], "subscores must include at least one positive weight"), + ([], "subscores must not be empty"), + ], + ids=["only-penalties", "nothing-to-combine"], +) +async def test_a_grade_with_nothing_positive_to_weigh_fails( + weights: list[float], refusal: str +) -> None: + env = Environment("graders") + + @env.template() + async def check(): + yield "go" + yield await combine( + *(SubScore(name=f"s{i}", value=1.0, weight=w) for i, w in enumerate(weights)) + ) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert (run.trace.error or "").startswith("[grading] ") + assert (run.trace.error or "").endswith(refusal) + + +async def test_combining_something_that_is_not_a_subscore_fails_the_grade() -> None: + env = Environment("graders") + + @env.template() + async def check(): + yield "go" + mixed: list[Any] = [SubScore(name="a", value=1.0), 0.5] + yield await combine(*mixed) + + (run,) = (await check().run(ScriptedAgent(ANSWER))).runs + + assert (run.trace.error or "").endswith("Expected SubScore or Awaitable[SubScore], got float") diff --git a/hud/telemetry/tests/__init__.py b/tests/integrations/__init__.py similarity index 100% rename from hud/telemetry/tests/__init__.py rename to tests/integrations/__init__.py diff --git a/hud/tests/__init__.py b/tests/integrations/harbor/__init__.py similarity index 100% rename from hud/tests/__init__.py rename to tests/integrations/harbor/__init__.py diff --git a/tests/integrations/harbor/cases.py b/tests/integrations/harbor/cases.py new file mode 100644 index 000000000..b3456410f --- /dev/null +++ b/tests/integrations/harbor/cases.py @@ -0,0 +1,36 @@ +"""Harbor case directories: a tree of files a case starts from, and what it expects.""" + +from __future__ import annotations + +import shutil +import tomllib +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from pathlib import Path + + +def read_expected(case: Path) -> dict[str, Any]: + """The ``expected.toml`` of a case directory, or an empty table when it has none.""" + spec = case / "expected.toml" + return tomllib.loads(spec.read_text("utf-8")) if spec.is_file() else {} + + +def assemble_case(case: Path, into: Path, *ignore: str) -> None: + """Copy the case directory ``case`` to ``into``, symlinks kept as links. + + ``base = ""`` in ``expected.toml`` names a sibling case to start from: + it is assembled first and ``case``'s files replace its files, so a variant + holds only what it changes. Neither ``expected.toml`` nor a name matching an + ``ignore`` pattern is copied. + """ + base = read_expected(case).get("base") + if base is not None: + assemble_case(case.parent / base, into, *ignore) + shutil.copytree( + case, + into, + symlinks=True, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns("expected.toml", "__pycache__", *ignore), + ) diff --git a/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/Containerfile b/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/Containerfile new file mode 100644 index 000000000..0a93e44d3 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/Containerfile @@ -0,0 +1 @@ +FROM python:3.12 diff --git a/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/docker-compose.yaml b/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/docker-compose.yaml new file mode 100644 index 000000000..f5063eeed --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/docker-compose.yaml @@ -0,0 +1,7 @@ +services: + main: + build: + context: . + dockerfile: Containerfile + args: + FLAVOR: compose diff --git a/tests/integrations/harbor/cases/compose-main-build/containerfile/instruction.md b/tests/integrations/harbor/cases/compose-main-build/containerfile/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/containerfile/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/hud/utils/tests/__init__.py b/tests/integrations/harbor/cases/compose-main-build/containerfile/task.toml similarity index 100% rename from hud/utils/tests/__init__.py rename to tests/integrations/harbor/cases/compose-main-build/containerfile/task.toml diff --git a/tests/integrations/harbor/cases/compose-main-build/containerfile/tests/test.sh b/tests/integrations/harbor/cases/compose-main-build/containerfile/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/containerfile/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/compose-main-build/expected.toml b/tests/integrations/harbor/cases/compose-main-build/expected.toml new file mode 100644 index 000000000..2e5695357 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/expected.toml @@ -0,0 +1,8 @@ +# An authored Compose main build becomes the base image main builds on. +[task.containerfile] +"compose.services.hud-base.build" = { context = "./compose-project/environment", dockerfile = "Containerfile", args = { FLAVOR = "compose" } } +"compose.services.main.build.additional_contexts" = { hud-base = "service:hud-base" } + +[task.image-and-dockerfile] +"compose.services.hud-base.build" = { context = "./compose-project/environment" } +"compose.services.main.build.additional_contexts" = { hud-base = "service:hud-base" } diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/Dockerfile b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/docker-compose.yaml b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/docker-compose.yaml new file mode 100644 index 000000000..8a3d24f45 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + image: authored/main:latest diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/instruction.md b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/task.toml b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/tests/test.sh b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/compose-variables/expected.toml b/tests/integrations/harbor/cases/compose-variables/expected.toml new file mode 100644 index 000000000..891577496 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/expected.toml @@ -0,0 +1,5 @@ +# Compose variables and limits reach the main service and the row's placement. +[task.task-a] +"compose.services.main.environment" = { FROM_DOTENV = "dotenv", FROM_ENV = "from-task" } +"compose.services.main.deploy" = { resources = { limits = { cpus = "2", memory = "4096M" } } } +placement = { resources = { cpu = 2.0, memory_mb = 4096 } } diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/environment/.env b/tests/integrations/harbor/cases/compose-variables/task-a/environment/.env new file mode 100644 index 000000000..465da46bd --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/environment/.env @@ -0,0 +1,3 @@ +DOTENV_ONLY=dotenv +SIDECAR_TAG=7-alpine +LITERAL=from-dotenv diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/compose-variables/task-a/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/environment/docker-compose.yaml b/tests/integrations/harbor/cases/compose-variables/task-a/environment/docker-compose.yaml new file mode 100644 index 000000000..1e566f3c4 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/environment/docker-compose.yaml @@ -0,0 +1,16 @@ +services: + main: + build: + context: ${CONTEXT_DIR} + image: ${MAIN_IMAGE_NAME} + environment: + FROM_ENV: ${LITERAL} + FROM_DOTENV: ${DOTENV_ONLY} + deploy: + resources: + limits: + cpus: ${CPUS} + memory: ${MEMORY} + redis: + image: redis:${SIDECAR_TAG} + expose: [6379] diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/instruction.md b/tests/integrations/harbor/cases/compose-variables/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/task.toml b/tests/integrations/harbor/cases/compose-variables/task-a/task.toml new file mode 100644 index 000000000..8c3a7cda0 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/task.toml @@ -0,0 +1,7 @@ +[environment] +cpus = 2 +memory_mb = 4096 + +[environment.env] +LITERAL = "from-task" +TEMPLATED = "${HARBOR_JUDGE_KEY}" diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/tests/test.sh b/tests/integrations/harbor/cases/compose-variables/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/build-pmars/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/build-pmars/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/build-pmars/task.toml b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/task.toml new file mode 100644 index 000000000..8eb24600d --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/task.toml @@ -0,0 +1,18 @@ +artifacts = [ + "/tmp/result", + { source = "/app/outputs", destination = "results/outputs", exclude = ["*.tmp", "cache"] }, +] + +[task] +description = "Build pMARS." +keywords = ["c", "build"] + +[metadata] +category = "systems" +difficulty = "medium" + +[agent] +timeout_sec = 45 + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/cases/dataset-grouping/build-pmars/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/environment/Dockerfile new file mode 100644 index 000000000..c3f2c07ff --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM alpine:3.20 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/task.toml b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/task.toml new file mode 100644 index 000000000..c900fd66a --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/task.toml @@ -0,0 +1,2 @@ +[metadata] +category = "machine-learning" diff --git a/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/task.toml b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/task.toml new file mode 100644 index 000000000..5f9766ded --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/task.toml @@ -0,0 +1,5 @@ +[metadata] +category = "systems" + +[verifier] +environment_mode = "shared" diff --git a/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/task.toml b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/task.toml new file mode 100644 index 000000000..070c3446e --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/task.toml @@ -0,0 +1,2 @@ +[metadata] +category = "games" diff --git a/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/expected.toml b/tests/integrations/harbor/cases/dataset-grouping/expected.toml new file mode 100644 index 000000000..0a1737874 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/expected.toml @@ -0,0 +1,14 @@ +# Tasks with the same environment share one project; a task keeps its artifacts and timeouts. +projects = [ + ["build-pmars", "chess-best-move"], + ["caffe-cifar-10", "sam-cell-seg"], + ["cancel-async-tasks"], +] + +[task.build-pmars] +"task.artifacts" = [ + { service = "main", source = "/tmp/result" }, + { service = "main", source = "/app/outputs", destination = "results/outputs", exclude = ["*.tmp", "cache"] }, +] +"task.verifier_timeout" = 30.0 +agent_config = { timeout_seconds = 45.0 } diff --git a/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/environment/Dockerfile new file mode 100644 index 000000000..c3f2c07ff --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM alpine:3.20 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/task.toml b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/task.toml new file mode 100644 index 000000000..c900fd66a --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/task.toml @@ -0,0 +1,2 @@ +[metadata] +category = "machine-learning" diff --git a/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/env-templates/templated/environment/Dockerfile b/tests/integrations/harbor/cases/env-templates/templated/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/templated/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/env-templates/templated/instruction.md b/tests/integrations/harbor/cases/env-templates/templated/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/templated/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/env-templates/templated/task.toml b/tests/integrations/harbor/cases/env-templates/templated/task.toml new file mode 100644 index 000000000..4690c7964 --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/templated/task.toml @@ -0,0 +1,6 @@ +[environment.env] +JUDGE_KEY = "${HARBOR_JUDGE_KEY}" +MODEL = "${HARBOR_JUDGE_MODEL:-gpt-4o}" + +[verifier.env] +VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" diff --git a/tests/integrations/harbor/cases/env-templates/templated/tests/test.sh b/tests/integrations/harbor/cases/env-templates/templated/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/templated/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-absolute-destination/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-absolute-destination/instruction.md b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-absolute-destination/task.toml b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/task.toml new file mode 100644 index 000000000..3f81952f9 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/task.toml @@ -0,0 +1 @@ +artifacts = [{ source = "/output", destination = "/tmp/out" }] diff --git a/tests/integrations/harbor/cases/findings/artifact-absolute-destination/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-backslash-destination/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-backslash-destination/instruction.md b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-backslash-destination/task.toml b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/task.toml new file mode 100644 index 000000000..280c896b4 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/task.toml @@ -0,0 +1 @@ +artifacts = [{ source = "/output", destination = "a\\b" }] diff --git a/tests/integrations/harbor/cases/findings/artifact-backslash-destination/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/instruction.md b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/task.toml b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/task.toml new file mode 100644 index 000000000..9d60c4c7f --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/task.toml @@ -0,0 +1 @@ +artifacts = [{ source = "/output", destination = "../out" }] diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-dotdot/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot/instruction.md b/tests/integrations/harbor/cases/findings/artifact-dotdot/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot/task.toml b/tests/integrations/harbor/cases/findings/artifact-dotdot/task.toml new file mode 100644 index 000000000..b3b91ef1e --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot/task.toml @@ -0,0 +1 @@ +artifacts = ["/workspace/../secret"] diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-dotdot/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-double-slash/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-double-slash/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-double-slash/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-double-slash/instruction.md b/tests/integrations/harbor/cases/findings/artifact-double-slash/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-double-slash/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-double-slash/task.toml b/tests/integrations/harbor/cases/findings/artifact-double-slash/task.toml new file mode 100644 index 000000000..f08ceddb7 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-double-slash/task.toml @@ -0,0 +1 @@ +artifacts = ["//"] diff --git a/tests/integrations/harbor/cases/findings/artifact-double-slash/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-double-slash/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-double-slash/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-manifest-destination/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-manifest-destination/instruction.md b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-manifest-destination/task.toml b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/task.toml new file mode 100644 index 000000000..cf19a6e99 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/task.toml @@ -0,0 +1 @@ +artifacts = [{ source = "/output", destination = "manifest.json" }] diff --git a/tests/integrations/harbor/cases/findings/artifact-manifest-destination/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-root/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-root/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-root/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-root/instruction.md b/tests/integrations/harbor/cases/findings/artifact-root/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-root/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-root/task.toml b/tests/integrations/harbor/cases/findings/artifact-root/task.toml new file mode 100644 index 000000000..3132b7263 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-root/task.toml @@ -0,0 +1 @@ +artifacts = ["/"] diff --git a/tests/integrations/harbor/cases/findings/artifact-root/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-root/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-root/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/bad-config/environment/Dockerfile b/tests/integrations/harbor/cases/findings/bad-config/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-config/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/bad-config/instruction.md b/tests/integrations/harbor/cases/findings/bad-config/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-config/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/bad-config/task.toml b/tests/integrations/harbor/cases/findings/bad-config/task.toml new file mode 100644 index 000000000..4a3e19fbd --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-config/task.toml @@ -0,0 +1,3 @@ +[environment] +cpus = -1 +network_mode = "sometimes" diff --git a/tests/integrations/harbor/cases/findings/bad-config/tests/test.sh b/tests/integrations/harbor/cases/findings/bad-config/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-config/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/bad-toml/environment/Dockerfile b/tests/integrations/harbor/cases/findings/bad-toml/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-toml/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/bad-toml/instruction.md b/tests/integrations/harbor/cases/findings/bad-toml/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-toml/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/bad-toml/task.toml b/tests/integrations/harbor/cases/findings/bad-toml/task.toml new file mode 100644 index 000000000..8758ff120 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-toml/task.toml @@ -0,0 +1 @@ +not = [valid diff --git a/tests/integrations/harbor/cases/findings/bad-toml/tests/test.sh b/tests/integrations/harbor/cases/findings/bad-toml/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-toml/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/compose-extends/environment/Dockerfile b/tests/integrations/harbor/cases/findings/compose-extends/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/compose-extends/environment/base.yaml b/tests/integrations/harbor/cases/findings/compose-extends/environment/base.yaml new file mode 100644 index 000000000..2ca784f72 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/environment/base.yaml @@ -0,0 +1,3 @@ +services: + web: + image: nginx diff --git a/tests/integrations/harbor/cases/findings/compose-extends/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/compose-extends/environment/docker-compose.yaml new file mode 100644 index 000000000..782f39ca6 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/environment/docker-compose.yaml @@ -0,0 +1,4 @@ +services: + main: {} + web: + extends: {file: base.yaml, service: web} diff --git a/tests/integrations/harbor/cases/findings/compose-extends/instruction.md b/tests/integrations/harbor/cases/findings/compose-extends/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/compose-extends/task.toml b/tests/integrations/harbor/cases/findings/compose-extends/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/compose-extends/tests/test.sh b/tests/integrations/harbor/cases/findings/compose-extends/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/compose-include/environment/Dockerfile b/tests/integrations/harbor/cases/findings/compose-include/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/compose-include/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/compose-include/environment/docker-compose.yaml new file mode 100644 index 000000000..01b92e914 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +include: [other.yaml] +services: + main: {} diff --git a/tests/integrations/harbor/cases/findings/compose-include/environment/other.yaml b/tests/integrations/harbor/cases/findings/compose-include/environment/other.yaml new file mode 100644 index 000000000..ad189ddbe --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/environment/other.yaml @@ -0,0 +1 @@ +services: {} diff --git a/tests/integrations/harbor/cases/findings/compose-include/instruction.md b/tests/integrations/harbor/cases/findings/compose-include/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/compose-include/task.toml b/tests/integrations/harbor/cases/findings/compose-include/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/compose-include/tests/test.sh b/tests/integrations/harbor/cases/findings/compose-include/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/Dockerfile b/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/docker-compose.yaml new file mode 100644 index 000000000..340f75d7a --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/docker-compose.yaml @@ -0,0 +1 @@ +- main diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/instruction.md b/tests/integrations/harbor/cases/findings/compose-not-mapping/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-not-mapping/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/task.toml b/tests/integrations/harbor/cases/findings/compose-not-mapping/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/tests/test.sh b/tests/integrations/harbor/cases/findings/compose-not-mapping/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-not-mapping/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/environment/Dockerfile b/tests/integrations/harbor/cases/findings/compose-path-escape/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-path-escape/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/compose-path-escape/environment/docker-compose.yaml new file mode 100644 index 000000000..2c04472ef --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-path-escape/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + env_file: ../../secret.env diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/instruction.md b/tests/integrations/harbor/cases/findings/compose-path-escape/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-path-escape/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/task.toml b/tests/integrations/harbor/cases/findings/compose-path-escape/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/tests/test.sh b/tests/integrations/harbor/cases/findings/compose-path-escape/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-path-escape/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/expected.toml b/tests/integrations/harbor/cases/findings/expected.toml new file mode 100644 index 000000000..7467d840a --- /dev/null +++ b/tests/integrations/harbor/cases/findings/expected.toml @@ -0,0 +1,40 @@ +# Every task but ``supported`` is refused with the findings it exists to raise. +findings = [ + ["artifact-absolute-destination", "harbor.invalid.task_config"], + ["artifact-backslash-destination", "harbor.invalid.task_config"], + ["artifact-dotdot", "harbor.invalid.task_config"], + ["artifact-dotdot-destination", "harbor.invalid.task_config"], + ["artifact-double-slash", "harbor.invalid.task_config"], + ["artifact-manifest-destination", "harbor.invalid.task_config"], + ["artifact-root", "harbor.invalid.task_config"], + ["bad-config", "harbor.invalid.task_config"], + ["bad-config", "harbor.invalid.task_config"], + ["bad-toml", "harbor.invalid.task_config_toml"], + ["compose-extends", "harbor.unsupported.compose_include_extends"], + ["compose-include", "harbor.unsupported.compose_include_extends"], + ["compose-not-mapping", "harbor.invalid.compose"], + ["compose-path-escape", "harbor.invalid.compose_project_path"], + ["healthcheck-duration", "harbor.invalid.healthcheck"], + ["healthcheck-test", "harbor.invalid.healthcheck"], + ["host-variable", "harbor.unsupported.host_compose_variable"], + ["main-build-escape", "harbor.invalid.compose_project_path"], + ["main-build-escape", "harbor.invalid.compose_main_build_escape"], + ["main-build-escape", "harbor.invalid.missing_compose_main_dockerfile"], + ["main-build-path-type", "harbor.invalid.compose_main_build_path"], + ["mcp-servers", "harbor.invalid.duplicate_mcp_name"], + ["mcp-servers", "harbor.invalid.reserved_mcp_name"], + ["mcp-servers", "harbor.invalid.reserved_mcp_name"], + ["mcp-servers", "harbor.invalid.mcp_url"], + ["missing-instruction", "harbor.invalid.missing_instruction"], + ["missing-main-dockerfile", "harbor.invalid.missing_compose_main_dockerfile"], + ["missing-tests", "harbor.invalid.missing_tests"], + ["missing-verifier-dockerfile", "harbor.invalid.missing_verifier_dockerfile"], + ["multi-step", "harbor.unsupported.multi_step"], + ["no-recipe", "harbor.invalid.environment_recipe"], + ["non-utf8-toml", "harbor.invalid.task_config_toml"], + ["reserved-ports", "harbor.invalid.reserved_main_port"], + ["reserved-ports", "harbor.invalid.reserved_main_port"], + ["reserved-ports", "harbor.invalid.reserved_main_port"], + ["reserved-services", "harbor.invalid.reserved_compose_service"], + ["sidecar-without-recipe", "harbor.invalid.sidecar_recipe"], +] diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/Dockerfile b/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/docker-compose.yaml new file mode 100644 index 000000000..b89312b7f --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: + healthcheck: + test: [CMD, 'true'] + interval: 2 seconds diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/instruction.md b/tests/integrations/harbor/cases/findings/healthcheck-duration/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-duration/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/task.toml b/tests/integrations/harbor/cases/findings/healthcheck-duration/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/tests/test.sh b/tests/integrations/harbor/cases/findings/healthcheck-duration/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-duration/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/environment/Dockerfile b/tests/integrations/harbor/cases/findings/healthcheck-test/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-test/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/healthcheck-test/environment/docker-compose.yaml new file mode 100644 index 000000000..3d5410ce8 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-test/environment/docker-compose.yaml @@ -0,0 +1,4 @@ +services: + main: + healthcheck: + test: [CMD-SHELL, a, b] diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/instruction.md b/tests/integrations/harbor/cases/findings/healthcheck-test/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-test/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/task.toml b/tests/integrations/harbor/cases/findings/healthcheck-test/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/tests/test.sh b/tests/integrations/harbor/cases/findings/healthcheck-test/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-test/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/host-variable/environment/Dockerfile b/tests/integrations/harbor/cases/findings/host-variable/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/host-variable/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/host-variable/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/host-variable/environment/docker-compose.yaml new file mode 100644 index 000000000..33bdab834 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/host-variable/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + image: ${HARBOR_JUDGE_KEY} diff --git a/tests/integrations/harbor/cases/findings/host-variable/instruction.md b/tests/integrations/harbor/cases/findings/host-variable/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/host-variable/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/host-variable/task.toml b/tests/integrations/harbor/cases/findings/host-variable/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/host-variable/tests/test.sh b/tests/integrations/harbor/cases/findings/host-variable/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/host-variable/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/environment/Dockerfile b/tests/integrations/harbor/cases/findings/main-build-escape/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-escape/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/main-build-escape/environment/docker-compose.yaml new file mode 100644 index 000000000..56c44581f --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-escape/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + build: ../.. diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/instruction.md b/tests/integrations/harbor/cases/findings/main-build-escape/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-escape/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/task.toml b/tests/integrations/harbor/cases/findings/main-build-escape/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/tests/test.sh b/tests/integrations/harbor/cases/findings/main-build-escape/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-escape/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/environment/Dockerfile b/tests/integrations/harbor/cases/findings/main-build-path-type/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-path-type/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/main-build-path-type/environment/docker-compose.yaml new file mode 100644 index 000000000..90223d5ea --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-path-type/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: + build: + context: . + dockerfile: [Dockerfile] diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/instruction.md b/tests/integrations/harbor/cases/findings/main-build-path-type/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-path-type/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/task.toml b/tests/integrations/harbor/cases/findings/main-build-path-type/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/tests/test.sh b/tests/integrations/harbor/cases/findings/main-build-path-type/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-path-type/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/mcp-servers/environment/Dockerfile b/tests/integrations/harbor/cases/findings/mcp-servers/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/mcp-servers/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/mcp-servers/instruction.md b/tests/integrations/harbor/cases/findings/mcp-servers/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/mcp-servers/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/mcp-servers/task.toml b/tests/integrations/harbor/cases/findings/mcp-servers/task.toml new file mode 100644 index 000000000..a72ac7bd5 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/mcp-servers/task.toml @@ -0,0 +1,18 @@ +[[environment.mcp_servers]] +name = "tools" +transport = "sse" +url = "http://tools:8000/sse" + +[[environment.mcp_servers]] +name = "tools" +transport = "streamable-http" + +[[environment.mcp_servers]] +name = "shell" +transport = "stdio" +command = "shell-mcp" + +[[environment.mcp_servers]] +name = "filetracking" +transport = "stdio" +command = "ft-mcp" diff --git a/tests/integrations/harbor/cases/findings/mcp-servers/tests/test.sh b/tests/integrations/harbor/cases/findings/mcp-servers/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/mcp-servers/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/missing-instruction/environment/Dockerfile b/tests/integrations/harbor/cases/findings/missing-instruction/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-instruction/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/missing-instruction/task.toml b/tests/integrations/harbor/cases/findings/missing-instruction/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/missing-instruction/tests/test.sh b/tests/integrations/harbor/cases/findings/missing-instruction/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-instruction/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/missing-main-dockerfile/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/environment/docker-compose.yaml new file mode 100644 index 000000000..31b70f11b --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: + build: + context: . + dockerfile: Missing diff --git a/tests/integrations/harbor/cases/findings/missing-main-dockerfile/instruction.md b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/missing-main-dockerfile/task.toml b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/missing-main-dockerfile/tests/test.sh b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/missing-tests/environment/Dockerfile b/tests/integrations/harbor/cases/findings/missing-tests/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-tests/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/missing-tests/instruction.md b/tests/integrations/harbor/cases/findings/missing-tests/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-tests/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/missing-tests/task.toml b/tests/integrations/harbor/cases/findings/missing-tests/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/environment/Dockerfile b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/instruction.md b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/task.toml b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/task.toml new file mode 100644 index 000000000..08c3f86d5 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/task.toml @@ -0,0 +1,2 @@ +[verifier] +environment_mode = "separate" diff --git a/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/tests/test.sh b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/multi-step/environment/Dockerfile b/tests/integrations/harbor/cases/findings/multi-step/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/multi-step/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/multi-step/steps/plan/instruction.md b/tests/integrations/harbor/cases/findings/multi-step/steps/plan/instruction.md new file mode 100644 index 000000000..a12d2e375 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/multi-step/steps/plan/instruction.md @@ -0,0 +1 @@ +# plan diff --git a/tests/integrations/harbor/cases/findings/multi-step/steps/plan/tests/test.sh b/tests/integrations/harbor/cases/findings/multi-step/steps/plan/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/multi-step/steps/plan/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/multi-step/task.toml b/tests/integrations/harbor/cases/findings/multi-step/task.toml new file mode 100644 index 000000000..cf49ef638 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/multi-step/task.toml @@ -0,0 +1,2 @@ +[[steps]] +name = "plan" diff --git a/tests/integrations/harbor/cases/findings/no-recipe/instruction.md b/tests/integrations/harbor/cases/findings/no-recipe/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/no-recipe/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/no-recipe/task.toml b/tests/integrations/harbor/cases/findings/no-recipe/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/no-recipe/tests/test.sh b/tests/integrations/harbor/cases/findings/no-recipe/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/no-recipe/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/non-utf8-toml/environment/Dockerfile b/tests/integrations/harbor/cases/findings/non-utf8-toml/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/non-utf8-toml/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/non-utf8-toml/instruction.md b/tests/integrations/harbor/cases/findings/non-utf8-toml/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/non-utf8-toml/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/non-utf8-toml/task.toml b/tests/integrations/harbor/cases/findings/non-utf8-toml/task.toml new file mode 100644 index 000000000..234e56f06 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/non-utf8-toml/task.toml @@ -0,0 +1 @@ +name = "ÿ" diff --git a/tests/integrations/harbor/cases/findings/non-utf8-toml/tests/test.sh b/tests/integrations/harbor/cases/findings/non-utf8-toml/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/non-utf8-toml/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/environment/Dockerfile b/tests/integrations/harbor/cases/findings/reserved-ports/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-ports/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/reserved-ports/environment/docker-compose.yaml new file mode 100644 index 000000000..8c58c8e16 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-ports/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + expose: [3128, 3129, 8765, 8080] diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/instruction.md b/tests/integrations/harbor/cases/findings/reserved-ports/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-ports/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/task.toml b/tests/integrations/harbor/cases/findings/reserved-ports/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/tests/test.sh b/tests/integrations/harbor/cases/findings/reserved-ports/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-ports/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/reserved-services/environment/Dockerfile b/tests/integrations/harbor/cases/findings/reserved-services/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-services/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/reserved-services/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/reserved-services/environment/docker-compose.yaml new file mode 100644 index 000000000..d555138bb --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-services/environment/docker-compose.yaml @@ -0,0 +1,6 @@ +services: + main: {} + hud-base: + image: busybox + hud-verifier: + image: busybox diff --git a/tests/integrations/harbor/cases/findings/reserved-services/instruction.md b/tests/integrations/harbor/cases/findings/reserved-services/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-services/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/reserved-services/task.toml b/tests/integrations/harbor/cases/findings/reserved-services/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/reserved-services/tests/test.sh b/tests/integrations/harbor/cases/findings/reserved-services/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-services/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/Dockerfile b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/docker-compose.yaml new file mode 100644 index 000000000..3cb0b7224 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: {} + worker: {} diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/instruction.md b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/task.toml b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/tests/test.sh b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/supported/environment/Dockerfile b/tests/integrations/harbor/cases/findings/supported/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/findings/supported/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/supported/instruction.md b/tests/integrations/harbor/cases/findings/supported/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/supported/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/supported/task.toml b/tests/integrations/harbor/cases/findings/supported/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/supported/tests/test.sh b/tests/integrations/harbor/cases/findings/supported/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/supported/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/image-failures/broken/environment/Dockerfile b/tests/integrations/harbor/cases/image-failures/broken/environment/Dockerfile new file mode 100644 index 000000000..7f48f97f2 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/broken/environment/Dockerfile @@ -0,0 +1 @@ +FROM registry.invalid/missing diff --git a/tests/integrations/harbor/cases/image-failures/broken/instruction.md b/tests/integrations/harbor/cases/image-failures/broken/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/broken/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/image-failures/broken/task.toml b/tests/integrations/harbor/cases/image-failures/broken/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/image-failures/broken/tests/test.sh b/tests/integrations/harbor/cases/image-failures/broken/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/broken/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/image-failures/expected.toml b/tests/integrations/harbor/cases/image-failures/expected.toml new file mode 100644 index 000000000..0b79671e8 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/expected.toml @@ -0,0 +1,18 @@ +# An image Docker cannot resolve is reported on its task, naming what failed. +findings = [ + ["broken", "harbor.invalid.image"], + ["missing-sidecar-image", "harbor.invalid.image"], + ["verifier-on-arm", "harbor.invalid.image"], +] + +# Words each task's finding messages contain. Linux daemons name only the +# registry they could not reach, so the finding names the service. +[mentions] +broken = ["registry.invalid/missing"] +missing-sidecar-image = ["cache", "registry.invalid"] +verifier-on-arm = ["--platform linux/amd64"] + +# Words the messages contain exactly when the Docker daemon is not amd64: a +# verifier that cannot build there says how to adapt for one platform. +[mentions_off_amd64] +verifier-on-arm = ["DOCKER_DEFAULT_PLATFORM=linux/amd64"] diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/Dockerfile b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/docker-compose.yaml b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/docker-compose.yaml new file mode 100644 index 000000000..83f5ef20f --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + cache: + image: registry.invalid/cache:1 diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/instruction.md b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/task.toml b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/tests/test.sh b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/environment/Dockerfile b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/environment/Dockerfile new file mode 100644 index 000000000..d4f30923b --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM --platform=linux/amd64 busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/instruction.md b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/task.toml b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/task.toml new file mode 100644 index 000000000..08c3f86d5 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/task.toml @@ -0,0 +1,2 @@ +[verifier] +environment_mode = "separate" diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/Dockerfile b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/Dockerfile new file mode 100644 index 000000000..602db9aa7 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-alpine +COPY . /tests +RUN exit 1 diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/test.sh b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/image-failures/working/environment/Dockerfile b/tests/integrations/harbor/cases/image-failures/working/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/working/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/image-failures/working/instruction.md b/tests/integrations/harbor/cases/image-failures/working/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/working/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/image-failures/working/task.toml b/tests/integrations/harbor/cases/image-failures/working/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/image-failures/working/tests/test.sh b/tests/integrations/harbor/cases/image-failures/working/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/working/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/plain-image/task-a/environment/.dockerignore b/tests/integrations/harbor/cases/plain-image/task-a/environment/.dockerignore new file mode 100644 index 000000000..cdb20a6c1 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/environment/.dockerignore @@ -0,0 +1,2 @@ +hud/ +Dockerfile diff --git a/tests/integrations/harbor/cases/plain-image/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/plain-image/task-a/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/plain-image/task-a/environment/docker-compose.yml b/tests/integrations/harbor/cases/plain-image/task-a/environment/docker-compose.yml new file mode 100644 index 000000000..b8e29e994 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/environment/docker-compose.yml @@ -0,0 +1 @@ +{"api_gateway": {"interval": "30s"}} diff --git a/tests/integrations/harbor/cases/plain-image/task-a/environment/ignored.txt b/tests/integrations/harbor/cases/plain-image/task-a/environment/ignored.txt new file mode 100644 index 000000000..4eea88a85 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/environment/ignored.txt @@ -0,0 +1 @@ +unchanged diff --git a/tests/integrations/harbor/cases/plain-image/task-a/instruction.md b/tests/integrations/harbor/cases/plain-image/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/plain-image/task-a/task.toml b/tests/integrations/harbor/cases/plain-image/task-a/task.toml new file mode 100644 index 000000000..d1ffb5ceb --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/task.toml @@ -0,0 +1,5 @@ +[metadata] +category = "systems" + +[verifier] +timeout_sec = 120 diff --git a/tests/integrations/harbor/cases/plain-image/task-a/tests/test.sh b/tests/integrations/harbor/cases/plain-image/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/prebuilt-image/expected.toml b/tests/integrations/harbor/cases/prebuilt-image/expected.toml new file mode 100644 index 000000000..00c583282 --- /dev/null +++ b/tests/integrations/harbor/cases/prebuilt-image/expected.toml @@ -0,0 +1,5 @@ +# A prebuilt image is the base of main and of the verifier, without a build of its own. +[task.task-a] +services = ["main"] +"compose.services.main.build.args.BASE_IMAGE" = "golang:1.23-alpine" +"compose.services.main.build.args.VERIFIER_IMAGE" = "golang:1.23-alpine" diff --git a/tests/integrations/harbor/cases/prebuilt-image/task-a/instruction.md b/tests/integrations/harbor/cases/prebuilt-image/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/prebuilt-image/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/prebuilt-image/task-a/task.toml b/tests/integrations/harbor/cases/prebuilt-image/task-a/task.toml new file mode 100644 index 000000000..10d007ada --- /dev/null +++ b/tests/integrations/harbor/cases/prebuilt-image/task-a/task.toml @@ -0,0 +1,2 @@ +[environment] +docker_image = "golang:1.23-alpine" diff --git a/tests/integrations/harbor/cases/prebuilt-image/task-a/tests/test.sh b/tests/integrations/harbor/cases/prebuilt-image/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/prebuilt-image/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/resources-and-budgets/budget/environment/Dockerfile b/tests/integrations/harbor/cases/resources-and-budgets/budget/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/budget/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/resources-and-budgets/budget/instruction.md b/tests/integrations/harbor/cases/resources-and-budgets/budget/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/budget/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/resources-and-budgets/budget/task.toml b/tests/integrations/harbor/cases/resources-and-budgets/budget/task.toml new file mode 100644 index 000000000..ce24d8eae --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/budget/task.toml @@ -0,0 +1,5 @@ +[environment] +build_timeout_sec = 900.5 + +[agent] +timeout_sec = 28800 diff --git a/tests/integrations/harbor/cases/resources-and-budgets/budget/tests/test.sh b/tests/integrations/harbor/cases/resources-and-budgets/budget/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/budget/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/environment/Dockerfile b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/instruction.md b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/task.toml b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/task.toml new file mode 100644 index 000000000..e2e50397d --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/task.toml @@ -0,0 +1,2 @@ +[environment] +gpus = 0 diff --git a/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/tests/test.sh b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/resources-and-budgets/expected.toml b/tests/integrations/harbor/cases/resources-and-budgets/expected.toml new file mode 100644 index 000000000..6f9a16afe --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/expected.toml @@ -0,0 +1,13 @@ +# Each task carries its own resources, limits and agent budget. +[task.gpu] +placement = { resources = { cpu = 4.0, memory_mb = 8192, storage_mb = 32768, gpu = { count = 2, type = ["H100"] } } } + +[task.windows-tpu] +placement = { resources = { os = "windows", tpu = { type = "v5", topology = "2x2" } } } + +[task.budget] +placement = { limits = { startup_timeout_s = 901 } } +agent_config = { timeout_seconds = 28800.0 } + +[task.cpu-only] +placement = {} diff --git a/tests/integrations/harbor/cases/resources-and-budgets/gpu/environment/Dockerfile b/tests/integrations/harbor/cases/resources-and-budgets/gpu/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/gpu/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/resources-and-budgets/gpu/instruction.md b/tests/integrations/harbor/cases/resources-and-budgets/gpu/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/gpu/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/resources-and-budgets/gpu/task.toml b/tests/integrations/harbor/cases/resources-and-budgets/gpu/task.toml new file mode 100644 index 000000000..eef4cfae7 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/gpu/task.toml @@ -0,0 +1,6 @@ +[environment] +cpus = 4 +memory_mb = 8192 +storage_mb = 32768 +gpus = 2 +gpu_types = ["H100"] diff --git a/tests/integrations/harbor/cases/resources-and-budgets/gpu/tests/test.sh b/tests/integrations/harbor/cases/resources-and-budgets/gpu/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/gpu/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/environment/Dockerfile b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/instruction.md b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/task.toml b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/task.toml new file mode 100644 index 000000000..682c1503d --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/task.toml @@ -0,0 +1,3 @@ +[environment] +os = "windows" +tpu = { type = "v5", topology = "2x2" } diff --git a/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/tests/test.sh b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/separate-verifier/expected.toml b/tests/integrations/harbor/cases/separate-verifier/expected.toml new file mode 100644 index 000000000..0e34b010f --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/expected.toml @@ -0,0 +1,13 @@ +# A separate verifier runs as its own row with its own placement. +[task.placement] +placement = { resources = { cpu = 2.0, memory_mb = 2048 }, limits = { startup_timeout_s = 600 } } +"verifier.runtime_config" = { resources = { cpu = 4.0, memory_mb = 1024, gpu = { count = 1, type = ["T4"] } }, limits = { startup_timeout_s = 1200 } } +service_access = true +"task.collect" = [{ service = "redis", command = "redis-cli save", timeout_sec = 10.0 }] + +# A verifier placed like its actor, or not placed at all, carries no placement of its own. +[task.plain] +"verifier.runtime_config" = false + +[task.same-placement] +"verifier.runtime_config" = false diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/environment/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/placement/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/environment/docker-compose.yaml b/tests/integrations/harbor/cases/separate-verifier/placement/environment/docker-compose.yaml new file mode 100644 index 000000000..086670a2d --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: {} + redis: + image: redis:7-alpine + expose: [6379] diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/instruction.md b/tests/integrations/harbor/cases/separate-verifier/placement/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/task.toml b/tests/integrations/harbor/cases/separate-verifier/placement/task.toml new file mode 100644 index 000000000..6b07a4817 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/task.toml @@ -0,0 +1,22 @@ +artifacts = ["/tmp/agent.patch"] + +[environment] +cpus = 2 +memory_mb = 2048 +build_timeout_sec = 600 + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[verifier.environment] +cpus = 4 +memory_mb = 1024 +build_timeout_sec = 1200 +gpus = 1 +gpu_types = ["T4"] + +[[verifier.collect]] +service = "redis" +command = "redis-cli save" +timeout_sec = 10 diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/tests/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/placement/tests/Dockerfile new file mode 100644 index 000000000..2b6adf3c5 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-alpine +COPY . /tests diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/tests/test.sh b/tests/integrations/harbor/cases/separate-verifier/placement/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/environment/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/plain/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/environment/docker-compose.yaml b/tests/integrations/harbor/cases/separate-verifier/plain/environment/docker-compose.yaml new file mode 100644 index 000000000..086670a2d --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: {} + redis: + image: redis:7-alpine + expose: [6379] diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/instruction.md b/tests/integrations/harbor/cases/separate-verifier/plain/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/task.toml b/tests/integrations/harbor/cases/separate-verifier/plain/task.toml new file mode 100644 index 000000000..08c3f86d5 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/task.toml @@ -0,0 +1,2 @@ +[verifier] +environment_mode = "separate" diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/tests/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/plain/tests/Dockerfile new file mode 100644 index 000000000..2b6adf3c5 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-alpine +COPY . /tests diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/tests/test.sh b/tests/integrations/harbor/cases/separate-verifier/plain/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/environment/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/same-placement/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/instruction.md b/tests/integrations/harbor/cases/separate-verifier/same-placement/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/task.toml b/tests/integrations/harbor/cases/separate-verifier/same-placement/task.toml new file mode 100644 index 000000000..a2e78698b --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/task.toml @@ -0,0 +1,10 @@ +[environment] +cpus = 4 +memory_mb = 14336 + +[verifier] +environment_mode = "separate" + +[verifier.environment] +cpus = 4 +memory_mb = 14336 diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/Dockerfile new file mode 100644 index 000000000..2b6adf3c5 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-alpine +COPY . /tests diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/test.sh b/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/expected.toml b/tests/integrations/harbor/cases/shared-collect-sidecar/expected.toml new file mode 100644 index 000000000..d37ad6b9e --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/expected.toml @@ -0,0 +1,6 @@ +# A task that collects from a sidecar gets service access. +[task.task-a] +service_access = true +"compose.services.main.build.target" = "service-access" +"task.collect" = [{ service = "redis", command = "redis-cli save", timeout_sec = 600.0 }] +"task.artifacts" = [{ service = "redis", source = "/data/dump.rdb" }] diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/docker-compose.yaml b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/docker-compose.yaml new file mode 100644 index 000000000..086670a2d --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: {} + redis: + image: redis:7-alpine + expose: [6379] diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/instruction.md b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/task.toml b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/task.toml new file mode 100644 index 000000000..becc8f63c --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/task.toml @@ -0,0 +1,5 @@ +artifacts = [{ source = "/data/dump.rdb", service = "redis" }] + +[[verifier.collect]] +service = "redis" +command = "redis-cli save" diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/tests/test.sh b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/sidecars/expected.toml b/tests/integrations/harbor/cases/sidecars/expected.toml new file mode 100644 index 000000000..cd6aece80 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/expected.toml @@ -0,0 +1,13 @@ +# Sidecars keep their settings and their order around main. +[task.implicit-main] +services = ["cache", "hud-base", "main", "redis", "seed", "worker"] +"compose.services.redis.depends_on" = { main = { condition = "service_started", restart = true }, seed = { condition = "service_completed_successfully" } } +"compose.services.redis.command" = ["redis-server", "--save", ""] +"compose.services.redis.environment" = { SIDE = "car" } +"compose.services.redis.healthcheck" = { interval = "2s", test = ["CMD", "redis-cli", "ping"] } +"compose.services.cache.build" = "./compose-project/environment/cache" +"compose.services.seed.build" = "./compose-project/environment/seed" + +[task.main-process] +"compose.services.main.environment" = { FROM_COMPOSE = "yes" } +"compose.services.main.expose" = ["8080"] diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/environment/Dockerfile b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/environment/cache/Dockerfile b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/cache/Dockerfile new file mode 100644 index 000000000..3ad78127e --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/cache/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +EXPOSE 11211/tcp 11211/udp diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/environment/docker-compose.yaml b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/docker-compose.yaml new file mode 100644 index 000000000..c70230697 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/docker-compose.yaml @@ -0,0 +1,21 @@ +services: + redis: + image: redis:7-alpine + depends_on: + main: + condition: service_healthy + restart: true + seed: + condition: service_completed_successfully + command: [redis-server, --save, ""] + environment: {SIDE: car} + expose: [6379] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + worker: + image: busybox:1.37 + cache: + build: ./cache + seed: + build: ./seed diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/environment/seed/Dockerfile b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/seed/Dockerfile new file mode 100644 index 000000000..4f0cb0660 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/seed/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +CMD ["true"] diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/instruction.md b/tests/integrations/harbor/cases/sidecars/implicit-main/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/task.toml b/tests/integrations/harbor/cases/sidecars/implicit-main/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/tests/test.sh b/tests/integrations/harbor/cases/sidecars/implicit-main/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/sidecars/main-process/environment/Dockerfile b/tests/integrations/harbor/cases/sidecars/main-process/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/main-process/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/sidecars/main-process/environment/docker-compose.yaml b/tests/integrations/harbor/cases/sidecars/main-process/environment/docker-compose.yaml new file mode 100644 index 000000000..000fb6084 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/main-process/environment/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + main: + user: 1001:1002 + entrypoint: [/compose-init] + working_dir: /compose-work + environment: {FROM_COMPOSE: "yes"} + expose: [8080] + healthcheck: + test: [CMD-SHELL, curl -f http://localhost:8080/health] + interval: 2s + timeout: 3s + retries: 5 + start_period: 1s diff --git a/tests/integrations/harbor/cases/sidecars/main-process/instruction.md b/tests/integrations/harbor/cases/sidecars/main-process/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/main-process/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/sidecars/main-process/task.toml b/tests/integrations/harbor/cases/sidecars/main-process/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/sidecars/main-process/tests/test.sh b/tests/integrations/harbor/cases/sidecars/main-process/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/main-process/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/expected.toml b/tests/integrations/harbor/cases/source-builds-and-volumes/expected.toml new file mode 100644 index 000000000..1ea8c32ba --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/expected.toml @@ -0,0 +1,13 @@ +# Source builds, volumes and env files are rebased into the project. +[task.task-a] +"compose.services.main.volumes" = [ + "./compose-project/environment/main-data:/mounts/0", + { type = "bind", source = "./compose-project/environment/readonly", target = "/mounts/1", read_only = true }, + "outputs:/mounts/2", + "./compose-project/tests:/controller/tests:ro", +] +"compose.services.main.env_file" = "./compose-project/environment/main.env" +"compose.services.database.build" = { context = "./compose-project/environment/database" } +"compose.services.database.env_file" = "./compose-project/environment/database/db.env" +"compose.services.database.volumes" = ["./compose-project/environment/database/data:/var/lib/postgresql/data"] +"compose.volumes" = { outputs = {} } diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/Dockerfile b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/Dockerfile new file mode 100644 index 000000000..9a1a0f96e --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/Dockerfile @@ -0,0 +1 @@ +FROM postgres:16 diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/data/.keep b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/data/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/db.env b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/db.env new file mode 100644 index 000000000..cb5b149b8 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/db.env @@ -0,0 +1 @@ +POSTGRES_DB=test diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/docker-compose.yaml b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/docker-compose.yaml new file mode 100644 index 000000000..f2644a3c8 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/docker-compose.yaml @@ -0,0 +1,18 @@ +services: + main: + env_file: ./main.env + volumes: + - ./main-data:/var/lib/main + - type: bind + source: ./readonly + target: /etc/authored + read_only: true + - outputs:/app/outputs + database: + build: + context: ./database + env_file: ./database/db.env + volumes: [./database/data:/var/lib/postgresql/data] + expose: [5432] +volumes: + outputs: {} diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main-data/.keep b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main-data/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main.env b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main.env new file mode 100644 index 000000000..bed5dccd5 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main.env @@ -0,0 +1 @@ +MAIN=true diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/readonly/.keep b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/readonly/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/instruction.md b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/task.toml b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/tests/test.sh b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/symlinks/expected.toml b/tests/integrations/harbor/cases/symlinks/expected.toml new file mode 100644 index 000000000..a5d8b609b --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/expected.toml @@ -0,0 +1,3 @@ +# Symlinks in a task stay links in its project. +[task.task-a] +links = { "compose-project/environment/Dockerfile.link" = "Dockerfile", "compose-project/tests/task-a/host-file" = "/etc/hostname" } diff --git a/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile new file mode 100644 index 000000000..08b8580ee --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile.link b/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile.link new file mode 120000 index 000000000..1d1fe94df --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile.link @@ -0,0 +1 @@ +Dockerfile \ No newline at end of file diff --git a/tests/integrations/harbor/cases/symlinks/task-a/instruction.md b/tests/integrations/harbor/cases/symlinks/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/symlinks/task-a/task.toml b/tests/integrations/harbor/cases/symlinks/task-a/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/symlinks/task-a/tests/host-file b/tests/integrations/harbor/cases/symlinks/task-a/tests/host-file new file mode 120000 index 000000000..48980ad58 --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/tests/host-file @@ -0,0 +1 @@ +/etc/hostname \ No newline at end of file diff --git a/tests/integrations/harbor/cases/symlinks/task-a/tests/test.sh b/tests/integrations/harbor/cases/symlinks/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/sources/answer-file/expected.toml b/tests/integrations/harbor/sources/answer-file/expected.toml new file mode 100644 index 000000000..932a90a7d --- /dev/null +++ b/tests/integrations/harbor/sources/answer-file/expected.toml @@ -0,0 +1,8 @@ +# The instruction names where to answer. +base = "demo" +answer_file = "/app/out.txt" + +[folders.solve-4080368a] +task = "solve" +args = { n = 2, prompt = "what's next?" } +prompt = "solve 2: what's next?" diff --git a/tests/integrations/harbor/sources/bound-under-another-name/expected.toml b/tests/integrations/harbor/sources/bound-under-another-name/expected.toml new file mode 100644 index 000000000..0d9e3d92b --- /dev/null +++ b/tests/integrations/harbor/sources/bound-under-another-name/expected.toml @@ -0,0 +1,13 @@ +# A source may bind its environment under any name. +base = "demo" +export = "tasks.py" + +[folders.solve-4080368a] +task = "solve" +args = { n = 2, prompt = "what's next?" } +prompt = "solve 2: what's next?" + +[[folders.solve-4080368a.grade]] +answer = "2" +exit = 0 +reward = "1.0" diff --git a/tests/integrations/harbor/sources/bound-under-another-name/tasks.py b/tests/integrations/harbor/sources/bound-under-another-name/tasks.py new file mode 100644 index 000000000..1607fcc94 --- /dev/null +++ b/tests/integrations/harbor/sources/bound-under-another-name/tasks.py @@ -0,0 +1,31 @@ +import asyncio +import os +from pathlib import Path + +from hud import Environment + +bench = Environment("demo") +markers = os.environ.get("EXPORT_TEST_MARKERS") + + +@bench.initialize +async def started() -> None: + if markers: + await asyncio.to_thread(Path(markers, "initialized").touch) + + +@bench.shutdown +async def stopped() -> None: + if markers: + await asyncio.to_thread(Path(markers, "stopped").touch) + + +@bench.template() +async def solve(n: int = 1, prompt: str = ""): + answer = yield f"solve {n}: {prompt}" + if answer == "crash": + raise RuntimeError("the grader is broken") + yield 1.0 if answer == str(n) else 0.25 + + +tasks = [solve(n=2, prompt="what's next?")] diff --git a/tests/integrations/harbor/sources/cdp-capability/env.py b/tests/integrations/harbor/sources/cdp-capability/env.py new file mode 100644 index 000000000..f08580f91 --- /dev/null +++ b/tests/integrations/harbor/sources/cdp-capability/env.py @@ -0,0 +1,33 @@ +import asyncio +import os +from pathlib import Path + +from hud import Environment +from hud.capabilities import Capability + +env = Environment("demo") +env.add_capability(Capability.cdp(url="ws://127.0.0.1:9222")) +markers = os.environ.get("EXPORT_TEST_MARKERS") + + +@env.initialize +async def started() -> None: + if markers: + await asyncio.to_thread(Path(markers, "initialized").touch) + + +@env.shutdown +async def stopped() -> None: + if markers: + await asyncio.to_thread(Path(markers, "stopped").touch) + + +@env.template() +async def solve(n: int = 1, prompt: str = ""): + answer = yield f"solve {n}: {prompt}" + if answer == "crash": + raise RuntimeError("the grader is broken") + yield 1.0 if answer == str(n) else 0.25 + + +tasks = [solve(n=2, prompt="what's next?")] diff --git a/tests/integrations/harbor/sources/cdp-capability/expected.toml b/tests/integrations/harbor/sources/cdp-capability/expected.toml new file mode 100644 index 000000000..02ce62e02 --- /dev/null +++ b/tests/integrations/harbor/sources/cdp-capability/expected.toml @@ -0,0 +1,3 @@ +# Harbor cannot carry a CDP capability. +base = "demo" +refusal = { type = "ValueError", message = "declares non-Harbor capabilities ['cdp", started = true } diff --git a/tests/integrations/harbor/sources/colliding-slugs/expected.toml b/tests/integrations/harbor/sources/colliding-slugs/expected.toml new file mode 100644 index 000000000..103540669 --- /dev/null +++ b/tests/integrations/harbor/sources/colliding-slugs/expected.toml @@ -0,0 +1,4 @@ +# Two slugs naming one directory; neither folder is written. +base = "demo" +export = "tasks.jsonl" +refusal = { type = "ValueError", message = "both name the export directory 'suite-fix'", started = true } diff --git a/tests/integrations/harbor/sources/colliding-slugs/tasks.jsonl b/tests/integrations/harbor/sources/colliding-slugs/tasks.jsonl new file mode 100644 index 000000000..07aee1d33 --- /dev/null +++ b/tests/integrations/harbor/sources/colliding-slugs/tasks.jsonl @@ -0,0 +1,2 @@ +{"env": "demo", "id": "solve", "args": {"n": 1}, "slug": "suite/fix"} +{"env": "demo", "id": "solve", "args": {"n": 2}, "slug": "suite-fix"} diff --git a/tests/integrations/harbor/sources/demo/Dockerfile b/tests/integrations/harbor/sources/demo/Dockerfile new file mode 100644 index 000000000..3774d440b --- /dev/null +++ b/tests/integrations/harbor/sources/demo/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +COPY *.whl /tmp/ +RUN pip install --no-cache-dir /tmp/*.whl +WORKDIR /workspace +COPY . ./ diff --git a/tests/integrations/harbor/sources/demo/env.py b/tests/integrations/harbor/sources/demo/env.py new file mode 100644 index 000000000..c5b725369 --- /dev/null +++ b/tests/integrations/harbor/sources/demo/env.py @@ -0,0 +1,31 @@ +import asyncio +import os +from pathlib import Path + +from hud import Environment + +env = Environment("demo") +markers = os.environ.get("EXPORT_TEST_MARKERS") + + +@env.initialize +async def started() -> None: + if markers: + await asyncio.to_thread(Path(markers, "initialized").touch) + + +@env.shutdown +async def stopped() -> None: + if markers: + await asyncio.to_thread(Path(markers, "stopped").touch) + + +@env.template() +async def solve(n: int = 1, prompt: str = ""): + answer = yield f"solve {n}: {prompt}" + if answer == "crash": + raise RuntimeError("the grader is broken") + yield 1.0 if answer == str(n) else 0.25 + + +tasks = [solve(n=2, prompt="what's next?")] diff --git a/tests/integrations/harbor/sources/demo/expected.toml b/tests/integrations/harbor/sources/demo/expected.toml new file mode 100644 index 000000000..6e73ac359 --- /dev/null +++ b/tests/integrations/harbor/sources/demo/expected.toml @@ -0,0 +1,20 @@ +# A python source exports its one task, which grades its answer in the built image. +[folders.solve-4080368a] +task = "solve" +args = { n = 2, prompt = "what's next?" } +prompt = "solve 2: what's next?" + +[[folders.solve-4080368a.grade]] +answer = "2" +exit = 0 +reward = "1.0" + +[[folders.solve-4080368a.grade]] +answer = "wrong" +exit = 0 +reward = "0.25" + +# A grader that raises fails test.sh and leaves no reward. +[[folders.solve-4080368a.grade]] +answer = "crash" +exit = 1 diff --git a/tests/integrations/harbor/sources/hud-dockerfile/Dockerfile.hud b/tests/integrations/harbor/sources/hud-dockerfile/Dockerfile.hud new file mode 100644 index 000000000..ad8037368 --- /dev/null +++ b/tests/integrations/harbor/sources/hud-dockerfile/Dockerfile.hud @@ -0,0 +1,2 @@ +FROM python:3.12-slim AS hud +RUN pip install hud diff --git a/tests/integrations/harbor/sources/hud-dockerfile/expected.toml b/tests/integrations/harbor/sources/hud-dockerfile/expected.toml new file mode 100644 index 000000000..87faf3f77 --- /dev/null +++ b/tests/integrations/harbor/sources/hud-dockerfile/expected.toml @@ -0,0 +1,7 @@ +# A Dockerfile.hud is exported in place of the plain Dockerfile. +base = "demo" + +[folders.solve-4080368a] +task = "solve" +args = { n = 2, prompt = "what's next?" } +prompt = "solve 2: what's next?" diff --git a/tests/integrations/harbor/sources/json-nested-slug/expected.toml b/tests/integrations/harbor/sources/json-nested-slug/expected.toml new file mode 100644 index 000000000..8e6d53e5a --- /dev/null +++ b/tests/integrations/harbor/sources/json-nested-slug/expected.toml @@ -0,0 +1,8 @@ +# A JSON taskset row's nested slug names its folder. +base = "demo" +export = "tasks.json" + +[folders.suite-fix] +task = "solve" +args = { n = 3 } +prompt = "solve 3: " diff --git a/tests/integrations/harbor/sources/json-nested-slug/tasks.json b/tests/integrations/harbor/sources/json-nested-slug/tasks.json new file mode 100644 index 000000000..29cc9f343 --- /dev/null +++ b/tests/integrations/harbor/sources/json-nested-slug/tasks.json @@ -0,0 +1 @@ +[{"env": "demo", "id": "solve", "args": {"n": 3}, "slug": "suite/fix"}] diff --git a/tests/integrations/harbor/sources/jsonl/expected.toml b/tests/integrations/harbor/sources/jsonl/expected.toml new file mode 100644 index 000000000..d28b6f1b6 --- /dev/null +++ b/tests/integrations/harbor/sources/jsonl/expected.toml @@ -0,0 +1,18 @@ +# Each JSONL taskset row becomes its own folder. +base = "demo" +export = "tasks.jsonl" + +[folders.four] +task = "solve" +args = { n = 4 } +prompt = "solve 4: " + +[[folders.four.grade]] +answer = "4" +exit = 0 +reward = "1.0" + +[folders.five] +task = "solve" +args = { n = 5 } +prompt = "solve 5: " diff --git a/tests/integrations/harbor/sources/jsonl/tasks.jsonl b/tests/integrations/harbor/sources/jsonl/tasks.jsonl new file mode 100644 index 000000000..c81848bb5 --- /dev/null +++ b/tests/integrations/harbor/sources/jsonl/tasks.jsonl @@ -0,0 +1,2 @@ +{"env": "demo", "id": "solve", "args": {"n": 4}, "slug": "four"} +{"env": "demo", "id": "solve", "args": {"n": 5}, "slug": "five"} diff --git a/tests/integrations/harbor/sources/main-thread/env.py b/tests/integrations/harbor/sources/main-thread/env.py new file mode 100644 index 000000000..6d7351e16 --- /dev/null +++ b/tests/integrations/harbor/sources/main-thread/env.py @@ -0,0 +1,34 @@ +import asyncio +import os +import threading +from pathlib import Path + +from hud import Environment + +assert threading.current_thread() is threading.main_thread() + +env = Environment("demo") +markers = os.environ.get("EXPORT_TEST_MARKERS") + + +@env.initialize +async def started() -> None: + if markers: + await asyncio.to_thread(Path(markers, "initialized").touch) + + +@env.shutdown +async def stopped() -> None: + if markers: + await asyncio.to_thread(Path(markers, "stopped").touch) + + +@env.template() +async def solve(n: int = 1, prompt: str = ""): + answer = yield f"solve {n}: {prompt}" + if answer == "crash": + raise RuntimeError("the grader is broken") + yield 1.0 if answer == str(n) else 0.25 + + +tasks = [solve(n=2, prompt="what's next?")] diff --git a/tests/integrations/harbor/sources/main-thread/expected.toml b/tests/integrations/harbor/sources/main-thread/expected.toml new file mode 100644 index 000000000..d2b342a83 --- /dev/null +++ b/tests/integrations/harbor/sources/main-thread/expected.toml @@ -0,0 +1,7 @@ +# Export imports a python source on the main thread. +base = "demo" + +[folders.solve-4080368a] +task = "solve" +args = { n = 2, prompt = "what's next?" } +prompt = "solve 2: what's next?" diff --git a/tests/integrations/harbor/sources/no-dockerfile/env.py b/tests/integrations/harbor/sources/no-dockerfile/env.py new file mode 100644 index 000000000..c5b725369 --- /dev/null +++ b/tests/integrations/harbor/sources/no-dockerfile/env.py @@ -0,0 +1,31 @@ +import asyncio +import os +from pathlib import Path + +from hud import Environment + +env = Environment("demo") +markers = os.environ.get("EXPORT_TEST_MARKERS") + + +@env.initialize +async def started() -> None: + if markers: + await asyncio.to_thread(Path(markers, "initialized").touch) + + +@env.shutdown +async def stopped() -> None: + if markers: + await asyncio.to_thread(Path(markers, "stopped").touch) + + +@env.template() +async def solve(n: int = 1, prompt: str = ""): + answer = yield f"solve {n}: {prompt}" + if answer == "crash": + raise RuntimeError("the grader is broken") + yield 1.0 if answer == str(n) else 0.25 + + +tasks = [solve(n=2, prompt="what's next?")] diff --git a/tests/integrations/harbor/sources/no-dockerfile/expected.toml b/tests/integrations/harbor/sources/no-dockerfile/expected.toml new file mode 100644 index 000000000..b7d358ce5 --- /dev/null +++ b/tests/integrations/harbor/sources/no-dockerfile/expected.toml @@ -0,0 +1,2 @@ +# Without a Dockerfile there is no image to rebuild, so nothing is started. +refusal = { type = "FileNotFoundError", message = "no Dockerfile(.hud) next to", started = false } diff --git a/tests/integrations/harbor/sources/non-root-multiline-cmd/Dockerfile b/tests/integrations/harbor/sources/non-root-multiline-cmd/Dockerfile new file mode 100644 index 000000000..7499fb0fb --- /dev/null +++ b/tests/integrations/harbor/sources/non-root-multiline-cmd/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.12-slim +RUN useradd -m app +USER app +CMD ["python", \ + "-m", \ + "app"] diff --git a/tests/integrations/harbor/sources/non-root-multiline-cmd/expected.toml b/tests/integrations/harbor/sources/non-root-multiline-cmd/expected.toml new file mode 100644 index 000000000..f2d305ecc --- /dev/null +++ b/tests/integrations/harbor/sources/non-root-multiline-cmd/expected.toml @@ -0,0 +1,7 @@ +# The exported image is the source image; HUD's entrypoint replaces its user and CMD. +base = "demo" + +[folders.solve-4080368a] +task = "solve" +args = { n = 2, prompt = "what's next?" } +prompt = "solve 2: what's next?" diff --git a/tests/integrations/harbor/sources/restrictive-dockerignore/.dockerignore b/tests/integrations/harbor/sources/restrictive-dockerignore/.dockerignore new file mode 100644 index 000000000..72e8ffc0d --- /dev/null +++ b/tests/integrations/harbor/sources/restrictive-dockerignore/.dockerignore @@ -0,0 +1 @@ +* diff --git a/tests/integrations/harbor/sources/restrictive-dockerignore/expected.toml b/tests/integrations/harbor/sources/restrictive-dockerignore/expected.toml new file mode 100644 index 000000000..3c89c64f1 --- /dev/null +++ b/tests/integrations/harbor/sources/restrictive-dockerignore/expected.toml @@ -0,0 +1,7 @@ +# A restrictive .dockerignore still admits the entrypoint; other source files are copied. +base = "demo" + +[folders.solve-4080368a] +task = "solve" +args = { n = 2, prompt = "what's next?" } +prompt = "solve 2: what's next?" diff --git a/tests/integrations/harbor/sources/restrictive-dockerignore/package.json b/tests/integrations/harbor/sources/restrictive-dockerignore/package.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tests/integrations/harbor/sources/restrictive-dockerignore/package.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tests/integrations/harbor/sources/ruff.toml b/tests/integrations/harbor/sources/ruff.toml new file mode 100644 index 000000000..acd3f4114 --- /dev/null +++ b/tests/integrations/harbor/sources/ruff.toml @@ -0,0 +1,5 @@ +# Each directory here is a user's source tree that export reads, not a package. +extend = "../../../../pyproject.toml" + +[lint.extend-per-file-ignores] +"*.py" = ["INP001"] diff --git a/tests/integrations/harbor/sources/setup-breaks-in-image/Dockerfile b/tests/integrations/harbor/sources/setup-breaks-in-image/Dockerfile new file mode 100644 index 000000000..d816a882e --- /dev/null +++ b/tests/integrations/harbor/sources/setup-breaks-in-image/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.12-slim +COPY *.whl /tmp/ +RUN pip install --no-cache-dir /tmp/*.whl +WORKDIR /workspace +COPY . ./ +ENV BREAK_SETUP=1 diff --git a/tests/integrations/harbor/sources/setup-breaks-in-image/env.py b/tests/integrations/harbor/sources/setup-breaks-in-image/env.py new file mode 100644 index 000000000..132fa1776 --- /dev/null +++ b/tests/integrations/harbor/sources/setup-breaks-in-image/env.py @@ -0,0 +1,33 @@ +import asyncio +import os +from pathlib import Path + +from hud import Environment + +env = Environment("demo") +markers = os.environ.get("EXPORT_TEST_MARKERS") + + +@env.initialize +async def started() -> None: + if markers: + await asyncio.to_thread(Path(markers, "initialized").touch) + + +@env.shutdown +async def stopped() -> None: + if markers: + await asyncio.to_thread(Path(markers, "stopped").touch) + + +@env.template() +async def solve(n: int = 1, prompt: str = ""): + if os.environ.get("BREAK_SETUP"): + raise RuntimeError("setup broke") + answer = yield f"solve {n}: {prompt}" + if answer == "crash": + raise RuntimeError("the grader is broken") + yield 1.0 if answer == str(n) else 0.25 + + +tasks = [solve(n=2, prompt="what's next?")] diff --git a/tests/integrations/harbor/sources/setup-breaks-in-image/expected.toml b/tests/integrations/harbor/sources/setup-breaks-in-image/expected.toml new file mode 100644 index 000000000..b5f617bf9 --- /dev/null +++ b/tests/integrations/harbor/sources/setup-breaks-in-image/expected.toml @@ -0,0 +1,17 @@ +# Export starts the task to read its prompt, so only the image breaks setup. An image +# whose task setup fails never runs the agent's command; the same image, its setup +# working, does. +base = "demo" + +[folders.solve-4080368a] +task = "solve" +args = { n = 2, prompt = "what's next?" } +prompt = "solve 2: what's next?" + +[[folders.solve-4080368a.run]] +env = {} +succeeds = false + +[[folders.solve-4080368a.run]] +env = { BREAK_SETUP = "" } +succeeds = true diff --git a/tests/integrations/harbor/sources/undefined-task/expected.toml b/tests/integrations/harbor/sources/undefined-task/expected.toml new file mode 100644 index 000000000..0e26c9774 --- /dev/null +++ b/tests/integrations/harbor/sources/undefined-task/expected.toml @@ -0,0 +1,4 @@ +# A row naming a task the environment does not define. +base = "demo" +export = "tasks.json" +refusal = { type = "TypeError", message = "needs a local env defining task 'missing'", started = true } diff --git a/tests/integrations/harbor/sources/undefined-task/tasks.json b/tests/integrations/harbor/sources/undefined-task/tasks.json new file mode 100644 index 000000000..8c81ed2aa --- /dev/null +++ b/tests/integrations/harbor/sources/undefined-task/tasks.json @@ -0,0 +1 @@ +[{"env": "demo", "id": "missing", "args": {}}] diff --git a/tests/integrations/harbor/sources/unusable-slug/expected.toml b/tests/integrations/harbor/sources/unusable-slug/expected.toml new file mode 100644 index 000000000..697acb2e3 --- /dev/null +++ b/tests/integrations/harbor/sources/unusable-slug/expected.toml @@ -0,0 +1,4 @@ +# A slug without a usable directory name. +base = "demo" +export = "tasks.json" +refusal = { type = "ValueError", message = "task slug '..' does not form a usable directory name", started = true } diff --git a/tests/integrations/harbor/sources/unusable-slug/tasks.json b/tests/integrations/harbor/sources/unusable-slug/tasks.json new file mode 100644 index 000000000..f50491caf --- /dev/null +++ b/tests/integrations/harbor/sources/unusable-slug/tasks.json @@ -0,0 +1 @@ +[{"env": "demo", "id": "solve", "args": {}, "slug": ".."}] diff --git a/tests/integrations/harbor/tasks/agent-answer-separate/expected.toml b/tests/integrations/harbor/tasks/agent-answer-separate/expected.toml new file mode 100644 index 000000000..a1ae3d61e --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer-separate/expected.toml @@ -0,0 +1 @@ +base = "agent-answer" diff --git a/tests/integrations/harbor/tasks/agent-answer-separate/task.toml b/tests/integrations/harbor/tasks/agent-answer-separate/task.toml new file mode 100644 index 000000000..999372f26 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer-separate/task.toml @@ -0,0 +1,3 @@ +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/agent-answer-separate/tests/Dockerfile b/tests/integrations/harbor/tasks/agent-answer-separate/tests/Dockerfile new file mode 100644 index 000000000..3d6ba54af --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer-separate/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +COPY . /tests diff --git a/tests/integrations/harbor/tasks/agent-answer/environment/Dockerfile b/tests/integrations/harbor/tasks/agent-answer/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/agent-answer/expected.toml b/tests/integrations/harbor/tasks/agent-answer/expected.toml new file mode 100644 index 000000000..466d882db --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/expected.toml @@ -0,0 +1 @@ +# The oracle answers with the solution's stdout. diff --git a/tests/integrations/harbor/tasks/agent-answer/instruction.md b/tests/integrations/harbor/tasks/agent-answer/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/agent-answer/solution/solve.sh b/tests/integrations/harbor/tasks/agent-answer/solution/solve.sh new file mode 100644 index 000000000..420a14799 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo forty-two diff --git a/tests/integrations/harbor/tasks/agent-answer/task.toml b/tests/integrations/harbor/tasks/agent-answer/task.toml new file mode 100644 index 000000000..0db3dcc22 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/task.toml @@ -0,0 +1,2 @@ +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/agent-answer/tests/test.sh b/tests/integrations/harbor/tasks/agent-answer/tests/test.sh new file mode 100644 index 000000000..551bb4079 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p /logs/verifier +if [ "$(cat /logs/agent_answer.txt)" = "forty-two" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/environment/Dockerfile b/tests/integrations/harbor/tasks/agent-lifecycle/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/environment/Dockerfile rename to tests/integrations/harbor/tasks/agent-lifecycle/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/instruction.md b/tests/integrations/harbor/tasks/agent-lifecycle/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/instruction.md rename to tests/integrations/harbor/tasks/agent-lifecycle/instruction.md diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/solution/solve.sh b/tests/integrations/harbor/tasks/agent-lifecycle/solution/solve.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/solution/solve.sh rename to tests/integrations/harbor/tasks/agent-lifecycle/solution/solve.sh diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/task.toml b/tests/integrations/harbor/tasks/agent-lifecycle/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/task.toml rename to tests/integrations/harbor/tasks/agent-lifecycle/task.toml diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/tests/test.sh b/tests/integrations/harbor/tasks/agent-lifecycle/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/tests/test.sh rename to tests/integrations/harbor/tasks/agent-lifecycle/tests/test.sh diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/expected.toml b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/expected.toml new file mode 100644 index 000000000..3daea823c --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/expected.toml @@ -0,0 +1,4 @@ +base = "sidecar-reachability" + +reward = 0.0 +error = "artifact /app/linked/tests has a symbolic link in its path" diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/solution/solve.sh new file mode 100644 index 000000000..a086c3b59 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +ln -s /verifier /app/linked diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/task.toml b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/task.toml new file mode 100644 index 000000000..4030594c3 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/task.toml @@ -0,0 +1,8 @@ +artifacts = [{ source = "/app/linked/tests", service = "main" }] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/artifact-exclude/expected.toml b/tests/integrations/harbor/tasks/artifact-exclude/expected.toml new file mode 100644 index 000000000..f9f1d4d05 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/expected.toml @@ -0,0 +1 @@ +base = "sidecar-reachability" diff --git a/tests/integrations/harbor/tasks/artifact-exclude/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-exclude/solution/solve.sh new file mode 100644 index 000000000..faa5dddaf --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/solution/solve.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +mkdir -p /app/outputs/cache/nested /app/outputs/logs +echo keep > /app/outputs/keep.txt +echo junk > /app/outputs/junk.tmp +echo junk > /app/outputs/logs/nested.tmp +echo junk > /app/outputs/cache/nested/blob +ln -s /etc/passwd /app/outputs/cache/link +ln -s missing /app/outputs/logs/dangling.tmp diff --git a/tests/integrations/harbor/tasks/artifact-exclude/task.toml b/tests/integrations/harbor/tasks/artifact-exclude/task.toml new file mode 100644 index 000000000..ff6a6b891 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/task.toml @@ -0,0 +1,10 @@ +artifacts = [ + { source = "/app/outputs", destination = "results", exclude = ["*.tmp", "cache"] }, +] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/artifact-exclude/tests/test.sh b/tests/integrations/harbor/tasks/artifact-exclude/tests/test.sh new file mode 100644 index 000000000..334ce2ecc --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/tests/test.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -u +mkdir -p /logs/verifier +if [ "$(cat /app/outputs/keep.txt 2>/dev/null)" = "keep" ] \ + && [ -d /app/outputs/logs ] \ + && [ ! -e /app/outputs/junk.tmp ] \ + && [ ! -e /app/outputs/logs/nested.tmp ] \ + && [ ! -L /app/outputs/logs/dangling.tmp ] \ + && [ ! -e /app/outputs/cache ]; then + echo 1 > /logs/verifier/reward.txt +else + echo "excluded artifact entries leaked into the verifier" >&2 + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/expected.toml b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/expected.toml new file mode 100644 index 000000000..f02ac5597 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/expected.toml @@ -0,0 +1,4 @@ +base = "sidecar-reachability" + +reward = 0.0 +error = "artifact /link contains a symbolic link" diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/task.toml b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/task.toml new file mode 100644 index 000000000..106f24e9c --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/task.toml @@ -0,0 +1,13 @@ +artifacts = [{ source = "/link", service = "workspace", exclude = ["main.html"] }] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +service = "workspace" +command = "ln -sfn /app /link" +timeout_sec = 10 diff --git a/tests/integrations/harbor/tasks/artifact-symlink/expected.toml b/tests/integrations/harbor/tasks/artifact-symlink/expected.toml new file mode 100644 index 000000000..74debb75c --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/expected.toml @@ -0,0 +1,4 @@ +base = "sidecar-reachability" + +reward = 0.0 +error = "artifact /app/main.html is a symbolic link" diff --git a/tests/integrations/harbor/tasks/artifact-symlink/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-symlink/solution/solve.sh new file mode 100644 index 000000000..20285b41c --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +ln -s /verifier/tests /app/main.html diff --git a/tests/integrations/harbor/tasks/artifact-symlink/task.toml b/tests/integrations/harbor/tasks/artifact-symlink/task.toml new file mode 100644 index 000000000..7f2081489 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/task.toml @@ -0,0 +1,8 @@ +artifacts = [{ source = "/app/main.html", service = "main" }] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/artifact-volume/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/artifact-volume/environment/docker-compose.yaml new file mode 100644 index 000000000..45a0a6a8a --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/environment/docker-compose.yaml @@ -0,0 +1,6 @@ +services: + main: + volumes: + - outputs:/app/outputs +volumes: + outputs: {} diff --git a/tests/integrations/harbor/tasks/artifact-volume/expected.toml b/tests/integrations/harbor/tasks/artifact-volume/expected.toml new file mode 100644 index 000000000..f9f1d4d05 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/expected.toml @@ -0,0 +1 @@ +base = "sidecar-reachability" diff --git a/tests/integrations/harbor/tasks/artifact-volume/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-volume/solution/solve.sh new file mode 100644 index 000000000..dcd2bf533 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +echo from-volume > /app/outputs/result.txt diff --git a/tests/integrations/harbor/tasks/artifact-volume/task.toml b/tests/integrations/harbor/tasks/artifact-volume/task.toml new file mode 100644 index 000000000..e0fe398d6 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/task.toml @@ -0,0 +1,8 @@ +artifacts = [{ source = "/app/outputs/result.txt", service = "main" }] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/artifact-volume/tests/test.sh b/tests/integrations/harbor/tasks/artifact-volume/tests/test.sh new file mode 100644 index 000000000..92d1a8b4d --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p /logs/verifier +if [ "$(cat /app/outputs/result.txt 2>/dev/null)" = "from-volume" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/bad-agent-user/expected.toml b/tests/integrations/harbor/tasks/bad-agent-user/expected.toml new file mode 100644 index 000000000..50a4b816c --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/expected.toml @@ -0,0 +1,3 @@ +# The controller refuses to serve. +base = "agent-answer" +startup_error = "Harbor user 'ghost' does not exist" diff --git a/tests/integrations/harbor/tasks/bad-agent-user/solution/solve.sh b/tests/integrations/harbor/tasks/bad-agent-user/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/bad-agent-user/task.toml b/tests/integrations/harbor/tasks/bad-agent-user/task.toml new file mode 100644 index 000000000..75c219938 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/task.toml @@ -0,0 +1,5 @@ +[agent] +user = "ghost" + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/bad-agent-user/tests/test.sh b/tests/integrations/harbor/tasks/bad-agent-user/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/expected.toml b/tests/integrations/harbor/tasks/bad-verifier-user/expected.toml new file mode 100644 index 000000000..1d4323eca --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/expected.toml @@ -0,0 +1,3 @@ +base = "agent-answer" +reward = 0.0 +error = "Harbor user 'ghost' does not exist in this image" diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/solution/solve.sh b/tests/integrations/harbor/tasks/bad-verifier-user/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/task.toml b/tests/integrations/harbor/tasks/bad-verifier-user/task.toml new file mode 100644 index 000000000..10327e812 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/task.toml @@ -0,0 +1,3 @@ +[verifier] +user = "ghost" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/tests/test.sh b/tests/integrations/harbor/tasks/bad-verifier-user/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/expected.toml b/tests/integrations/harbor/tasks/collect-hook-fails/expected.toml new file mode 100644 index 000000000..00f99bc12 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/expected.toml @@ -0,0 +1,3 @@ +base = "agent-answer" +reward = 0.0 +error = "collect hook on 'main' failed: broken" diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/solution/solve.sh b/tests/integrations/harbor/tasks/collect-hook-fails/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/task.toml b/tests/integrations/harbor/tasks/collect-hook-fails/task.toml new file mode 100644 index 000000000..c5a6f0b42 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/task.toml @@ -0,0 +1,7 @@ +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +command = "echo broken >&2; exit 3" +timeout_sec = 1 diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/tests/Dockerfile b/tests/integrations/harbor/tasks/collect-hook-fails/tests/Dockerfile new file mode 100644 index 000000000..3d6ba54af --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +COPY . /tests diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/tests/test.sh b/tests/integrations/harbor/tasks/collect-hook-fails/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/collect-hook-times-out/expected.toml b/tests/integrations/harbor/tasks/collect-hook-times-out/expected.toml new file mode 100644 index 000000000..0bf3d8670 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-times-out/expected.toml @@ -0,0 +1,3 @@ +base = "collect-hook-fails" +reward = 0.0 +error = "collect hook on 'main' timed out after 1s" diff --git a/tests/integrations/harbor/tasks/collect-hook-times-out/task.toml b/tests/integrations/harbor/tasks/collect-hook-times-out/task.toml new file mode 100644 index 000000000..f450638a5 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-times-out/task.toml @@ -0,0 +1,7 @@ +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +command = "sleep 30" +timeout_sec = 1 diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/environment/Dockerfile b/tests/integrations/harbor/tasks/entrypoint-exits/environment/Dockerfile new file mode 100644 index 000000000..e58726553 --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.11-slim +WORKDIR /workspace +ENTRYPOINT ["sh", "-c", "exit 7"] diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/expected.toml b/tests/integrations/harbor/tasks/entrypoint-exits/expected.toml new file mode 100644 index 000000000..b4183321c --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/expected.toml @@ -0,0 +1,3 @@ +base = "agent-answer" +reward = 0.0 +error = "Harbor environment entrypoint exited with status 7" diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/solution/solve.sh b/tests/integrations/harbor/tasks/entrypoint-exits/solution/solve.sh new file mode 100644 index 000000000..5cba02a37 --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 1 diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/tests/test.sh b/tests/integrations/harbor/tasks/entrypoint-exits/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/env-template-missing/expected.toml b/tests/integrations/harbor/tasks/env-template-missing/expected.toml new file mode 100644 index 000000000..6af79c2f2 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/expected.toml @@ -0,0 +1,3 @@ +# The controller refuses to serve. +base = "agent-answer" +startup_error = "HARBOR_MISSING_KEY" diff --git a/tests/integrations/harbor/tasks/env-template-missing/solution/solve.sh b/tests/integrations/harbor/tasks/env-template-missing/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/env-template-missing/task.toml b/tests/integrations/harbor/tasks/env-template-missing/task.toml new file mode 100644 index 000000000..97337976c --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/task.toml @@ -0,0 +1,5 @@ +[environment.env] +API_KEY = "${HARBOR_MISSING_KEY}" + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/env-template-missing/tests/test.sh b/tests/integrations/harbor/tasks/env-template-missing/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/env-templates/expected.toml b/tests/integrations/harbor/tasks/env-templates/expected.toml new file mode 100644 index 000000000..4a71c83ec --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/expected.toml @@ -0,0 +1,5 @@ +# An empty runtime value falls back to the template default. +base = "agent-answer" +[runtime.env_vars] +HARBOR_GREETING = "" +HARBOR_JUDGE_KEY = "judge-secret" diff --git a/tests/integrations/harbor/tasks/env-templates/solution/solve.sh b/tests/integrations/harbor/tasks/env-templates/solution/solve.sh new file mode 100644 index 000000000..f6695a2ad --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +[ "$JUDGE_KEY" = "judge-secret" ] && [ "$GREETING" = "hello" ] diff --git a/tests/integrations/harbor/tasks/env-templates/task.toml b/tests/integrations/harbor/tasks/env-templates/task.toml new file mode 100644 index 000000000..8c7fb1a83 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/task.toml @@ -0,0 +1,11 @@ +[environment.env] +GREETING = "${HARBOR_GREETING:-hello}" +JUDGE_KEY = "${HARBOR_JUDGE_KEY}" +EMBEDDED = "Bearer ${HARBOR_JUDGE_KEY}" + +[verifier] +timeout_sec = 120 + +[verifier.env] +VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" +EMPTY_DEFAULT = "${HARBOR_UNSET:-}" diff --git a/tests/integrations/harbor/tasks/env-templates/tests/test.sh b/tests/integrations/harbor/tasks/env-templates/tests/test.sh new file mode 100644 index 000000000..6335fad86 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/tests/test.sh @@ -0,0 +1,8 @@ +#!/bin/bash +fail() { echo "unexpected $1"; echo "0.0" > /logs/verifier/reward.txt; exit 0; } +[ "$GREETING" = "hello" ] || fail "GREETING=$GREETING" +[ "$JUDGE_KEY" = "judge-secret" ] || fail "JUDGE_KEY=$JUDGE_KEY" +[ "$EMBEDDED" = 'Bearer ${HARBOR_JUDGE_KEY}' ] || fail "EMBEDDED=$EMBEDDED" +[ "$VERIFIER_KEY" = "judge-secret" ] || fail "VERIFIER_KEY=$VERIFIER_KEY" +[ "${EMPTY_DEFAULT-unset}" = "" ] || fail "EMPTY_DEFAULT=${EMPTY_DEFAULT-unset}" +echo "1.0" > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/environment/Dockerfile b/tests/integrations/harbor/tasks/fedora-bootstrap/environment/Dockerfile new file mode 100644 index 000000000..9f8bc3592 --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM fedora:42 +WORKDIR /app diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/instruction.md b/tests/integrations/harbor/tasks/fedora-bootstrap/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/solution/solve.sh b/tests/integrations/harbor/tasks/fedora-bootstrap/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/task.toml b/tests/integrations/harbor/tasks/fedora-bootstrap/task.toml new file mode 100644 index 000000000..a724393dc --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/task.toml @@ -0,0 +1,5 @@ +[task] +name = "fedora-bootstrap" + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/tests/test.sh b/tests/integrations/harbor/tasks/fedora-bootstrap/tests/test.sh new file mode 100644 index 000000000..3772af98f --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/expected.toml b/tests/integrations/harbor/tasks/healthcheck-fails/expected.toml new file mode 100644 index 000000000..aaded40ef --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/expected.toml @@ -0,0 +1,3 @@ +base = "agent-answer" +reward = 0.0 +error = "Harbor environment healthcheck failed after 3 attempts: not ready" diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/solution/solve.sh b/tests/integrations/harbor/tasks/healthcheck-fails/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/task.toml b/tests/integrations/harbor/tasks/healthcheck-fails/task.toml new file mode 100644 index 000000000..1509363d2 --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/task.toml @@ -0,0 +1,8 @@ +[environment.healthcheck] +command = "echo not ready >&2; false" +interval_sec = 0.1 +timeout_sec = 5 +retries = 3 + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/tests/test.sh b/tests/integrations/harbor/tasks/healthcheck-fails/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/Dockerfile b/tests/integrations/harbor/tasks/hello-mcp/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/Dockerfile rename to tests/integrations/harbor/tasks/hello-mcp/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/hello-mcp/environment/docker-compose.yaml similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/docker-compose.yaml rename to tests/integrations/harbor/tasks/hello-mcp/environment/docker-compose.yaml diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/Dockerfile b/tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/Dockerfile rename to tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/Dockerfile diff --git a/tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/__init__.py b/tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/server.py b/tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/server.py similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/server.py rename to tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/server.py diff --git a/tests/integrations/harbor/tasks/hello-mcp/expected.toml b/tests/integrations/harbor/tasks/hello-mcp/expected.toml new file mode 100644 index 000000000..e382031b7 --- /dev/null +++ b/tests/integrations/harbor/tasks/hello-mcp/expected.toml @@ -0,0 +1,2 @@ +# The host side opens the sidecar's MCP server through the control channel. +capability_probe = { name = "mcp-server", tools = ["get_secret"] } diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/instruction.md b/tests/integrations/harbor/tasks/hello-mcp/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/instruction.md rename to tests/integrations/harbor/tasks/hello-mcp/instruction.md diff --git a/tests/integrations/harbor/tasks/hello-mcp/solution/solve.sh b/tests/integrations/harbor/tasks/hello-mcp/solution/solve.sh new file mode 100644 index 000000000..3e90f2cc8 --- /dev/null +++ b/tests/integrations/harbor/tasks/hello-mcp/solution/solve.sh @@ -0,0 +1,47 @@ +#!/bin/sh +set -eu +# Call the MCP server the way an agent in the workspace would. +python3 - <<'PY' +import json +from pathlib import Path +from urllib.request import Request, urlopen + +url = "http://mcp-server:8000/mcp" +session_id = None + + +def post(payload): + global session_id + headers = { + "Accept": "application/json, text/event-stream", + "Content-Type": "application/json", + } + if session_id is not None: + headers["Mcp-Session-Id"] = session_id + request = Request(url, data=json.dumps(payload).encode(), headers=headers) + with urlopen(request) as response: + body = response.read().decode() + session_id = response.headers.get("Mcp-Session-Id") or session_id + data = [line[6:] for line in body.splitlines() if line.startswith("data: ")] + return json.loads(data[-1] if data else body) if body else None + + +post({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "hud-test", "version": "1"}, + }, +}) +post({"jsonrpc": "2.0", "method": "notifications/initialized"}) +result = post({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "get_secret", "arguments": {}}, +}) +Path("/app/secret.txt").write_text(result["result"]["content"][0]["text"]) +PY diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/task.toml b/tests/integrations/harbor/tasks/hello-mcp/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/task.toml rename to tests/integrations/harbor/tasks/hello-mcp/task.toml diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/tests/test.sh b/tests/integrations/harbor/tasks/hello-mcp/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/tests/test.sh rename to tests/integrations/harbor/tasks/hello-mcp/tests/test.sh diff --git a/tests/integrations/harbor/tasks/main-ports/environment/Dockerfile b/tests/integrations/harbor/tasks/main-ports/environment/Dockerfile new file mode 100644 index 000000000..802c34c04 --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/environment/Dockerfile @@ -0,0 +1,6 @@ +FROM debian:bookworm-slim +WORKDIR /app +COPY entrypoint.sh /usr/local/bin/start-environment +RUN chmod +x /usr/local/bin/start-environment && printf nested > /app/marker +EXPOSE 3128 3129 8765 8080 +ENTRYPOINT ["/usr/local/bin/start-environment"] diff --git a/tests/integrations/harbor/tasks/main-ports/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/main-ports/environment/docker-compose.yaml new file mode 100644 index 000000000..0dc19485f --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/environment/docker-compose.yaml @@ -0,0 +1,6 @@ +services: + main: {} + observer: + image: python:3.11-alpine + command: ["python", "-m", "http.server", "9000"] + expose: [9000] diff --git a/tests/integrations/harbor/tasks/main-ports/environment/entrypoint.sh b/tests/integrations/harbor/tasks/main-ports/environment/entrypoint.sh new file mode 100644 index 000000000..08bce4c9c --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/environment/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +python3 -m http.server 8765 --directory /app >/tmp/inner.log 2>&1 & +python3 -m http.server 8080 --directory /app >/tmp/app.log 2>&1 & +exec "$@" diff --git a/tests/integrations/harbor/tasks/main-ports/instruction.md b/tests/integrations/harbor/tasks/main-ports/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/main-ports/solution/solve.sh b/tests/integrations/harbor/tasks/main-ports/solution/solve.sh new file mode 100644 index 000000000..f388875fa --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +curl -fsS http://127.0.0.1:8765/marker > /app/result diff --git a/tests/integrations/harbor/tasks/main-ports/task.toml b/tests/integrations/harbor/tasks/main-ports/task.toml new file mode 100644 index 000000000..65d636cec --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/task.toml @@ -0,0 +1,18 @@ +[environment.healthcheck] +command = "curl -fsS http://127.0.0.1:8765/marker && curl -fsS http://127.0.0.1:8080/marker" +interval_sec = 0.1 +timeout_sec = 5 +retries = 50 + +[verifier] +timeout_sec = 30 + +[[verifier.collect]] +service = "observer" +command = ''' +python - <<'PY' +from urllib.request import urlopen +assert urlopen('http://main:8080/marker').read() == b'nested' +PY +''' +timeout_sec = 10 diff --git a/tests/integrations/harbor/tasks/main-ports/tests/test.sh b/tests/integrations/harbor/tasks/main-ports/tests/test.sh new file mode 100644 index 000000000..d3b77c54c --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +test "$(cat /app/result)" = nested +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/managed-agents/expected.toml b/tests/integrations/harbor/tasks/managed-agents/expected.toml new file mode 100644 index 000000000..8705ea29b --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/expected.toml @@ -0,0 +1,6 @@ +# A CLI bundle the runtime mounts at /usr/local/lib/agents runs in agent sessions. +base = "agent-answer" + +[[runtime.bind_mounts]] +source = "managed/codex" +target = "/usr/local/lib/agents/codex" diff --git a/tests/integrations/harbor/tasks/managed-agents/managed/codex/bin/codex b/tests/integrations/harbor/tasks/managed-agents/managed/codex/bin/codex new file mode 100755 index 000000000..b793c8c55 --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/managed/codex/bin/codex @@ -0,0 +1,2 @@ +#!/bin/sh +echo managed-codex diff --git a/tests/integrations/harbor/tasks/managed-agents/solution/solve.sh b/tests/integrations/harbor/tasks/managed-agents/solution/solve.sh new file mode 100644 index 000000000..2afd7730b --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +/usr/local/lib/agents/codex/bin/codex > /workspace/result diff --git a/tests/integrations/harbor/tasks/managed-agents/tests/test.sh b/tests/integrations/harbor/tasks/managed-agents/tests/test.sh new file mode 100644 index 000000000..ab9f0503a --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +test "$(cat /workspace/result)" = managed-codex +echo 1 > /logs/verifier/reward.txt diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/environment/Dockerfile b/tests/integrations/harbor/tasks/phase-boundary/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/environment/Dockerfile rename to tests/integrations/harbor/tasks/phase-boundary/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/environment/entrypoint.sh b/tests/integrations/harbor/tasks/phase-boundary/environment/entrypoint.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/environment/entrypoint.sh rename to tests/integrations/harbor/tasks/phase-boundary/environment/entrypoint.sh diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/instruction.md b/tests/integrations/harbor/tasks/phase-boundary/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/instruction.md rename to tests/integrations/harbor/tasks/phase-boundary/instruction.md diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/solution/solve.sh b/tests/integrations/harbor/tasks/phase-boundary/solution/solve.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/solution/solve.sh rename to tests/integrations/harbor/tasks/phase-boundary/solution/solve.sh diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/task.toml b/tests/integrations/harbor/tasks/phase-boundary/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/task.toml rename to tests/integrations/harbor/tasks/phase-boundary/task.toml diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/tests/test.sh b/tests/integrations/harbor/tasks/phase-boundary/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/tests/test.sh rename to tests/integrations/harbor/tasks/phase-boundary/tests/test.sh diff --git a/tests/integrations/harbor/tasks/reward-files/environment/Dockerfile b/tests/integrations/harbor/tasks/reward-files/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/reward-files/expected.toml b/tests/integrations/harbor/tasks/reward-files/expected.toml new file mode 100644 index 000000000..c3c8526d1 --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/expected.toml @@ -0,0 +1,44 @@ +# reward.json wins over reward.txt; only a finite, non-boolean number counts. +[[case]] +name = "reward.json over reward.txt" +reward = 0.75 +runtime.env_vars = { REWARD_SHAPE = "json-preferred" } + +[[case]] +name = "a score key" +reward = 0.5 +runtime.env_vars = { REWARD_SHAPE = "json-score" } + +[[case]] +name = "a bare number" +reward = 0.25 +runtime.env_vars = { REWARD_SHAPE = "json-number" } + +[[case]] +name = "a boolean" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "json-bool" } + +[[case]] +name = "padded text" +reward = 0.5 +runtime.env_vars = { REWARD_SHAPE = "text" } + +[[case]] +name = "nan text" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "text-nan" } + +[[case]] +name = "words" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "text-words" } + +[[case]] +name = "no reward file" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "none" } diff --git a/tests/integrations/harbor/tasks/reward-files/instruction.md b/tests/integrations/harbor/tasks/reward-files/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/reward-files/solution/solve.sh b/tests/integrations/harbor/tasks/reward-files/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/reward-files/task.toml b/tests/integrations/harbor/tasks/reward-files/task.toml new file mode 100644 index 000000000..bcd8bc462 --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/task.toml @@ -0,0 +1,5 @@ +[verifier] +timeout_sec = 30 + +[verifier.env] +SHAPE = "${REWARD_SHAPE}" diff --git a/tests/integrations/harbor/tasks/reward-files/tests/test.sh b/tests/integrations/harbor/tasks/reward-files/tests/test.sh new file mode 100644 index 000000000..122ab7fc3 --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/tests/test.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +mkdir -p /logs/verifier +cd /logs/verifier +case "$SHAPE" in + json-preferred) echo '{"reward": 0.75}' > reward.json; echo 0.1 > reward.txt ;; + json-score) echo '{"score": 0.5}' > reward.json ;; + json-number) echo '0.25' > reward.json ;; + json-bool) echo 'true' > reward.json; echo 1 > reward.txt ;; + text) echo ' 0.5 ' > reward.txt ;; + text-nan) echo nan > reward.txt ;; + text-words) echo 'one point zero' > reward.txt ;; + none) ;; +esac diff --git a/tests/integrations/harbor/tasks/session-reliability/expected.toml b/tests/integrations/harbor/tasks/session-reliability/expected.toml new file mode 100644 index 000000000..83543c9b5 --- /dev/null +++ b/tests/integrations/harbor/tasks/session-reliability/expected.toml @@ -0,0 +1,3 @@ +# The solution runs in 40 separate SSH sessions. +base = "agent-answer" +repeat = 40 diff --git a/tests/integrations/harbor/tasks/session-reliability/solution/solve.sh b/tests/integrations/harbor/tasks/session-reliability/solution/solve.sh new file mode 100644 index 000000000..f34f230c0 --- /dev/null +++ b/tests/integrations/harbor/tasks/session-reliability/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +echo session >> /workspace/sessions diff --git a/tests/integrations/harbor/tasks/session-reliability/tests/test.sh b/tests/integrations/harbor/tasks/session-reliability/tests/test.sh new file mode 100644 index 000000000..01336e8bd --- /dev/null +++ b/tests/integrations/harbor/tasks/session-reliability/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p /logs/verifier +if [ "$(wc -l < /workspace/sessions)" -eq 40 ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/docker-compose.yaml new file mode 100644 index 000000000..e6a19479e --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/docker-compose.yaml @@ -0,0 +1,10 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + build: ./workspace + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/environment/workspace/Dockerfile b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/workspace/Dockerfile new file mode 100644 index 000000000..2861eaa41 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/workspace/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-alpine +EXPOSE 5678 5679 diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/expected.toml b/tests/integrations/harbor/tasks/sidecar-built-ports/expected.toml new file mode 100644 index 000000000..f9f1d4d05 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/expected.toml @@ -0,0 +1 @@ +base = "sidecar-reachability" diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-placed/expected.toml b/tests/integrations/harbor/tasks/sidecar-reachability-placed/expected.toml new file mode 100644 index 000000000..f9f1d4d05 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-placed/expected.toml @@ -0,0 +1 @@ +base = "sidecar-reachability" diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-placed/task.toml b/tests/integrations/harbor/tasks/sidecar-reachability-placed/task.toml new file mode 100644 index 000000000..312671da8 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-placed/task.toml @@ -0,0 +1,33 @@ +artifacts = [ + { source = "/app/sidecar.html", service = "main" }, + { source = "/app/main.html", service = "main" }, + { source = "/app/results", destination = "results", exclude = ["*.tmp"] }, + { source = "/app/binary", service = "main" }, + { source = "/opt/result", service = "main" }, + { source = "/root/agent-output.txt", service = "main" }, + { source = "/tmp/main.txt", service = "main" }, + { source = "/tmp/sidecar.txt", service = "workspace" }, +] + +[task] +name = "sidecar-reachability" + +[environment.env] +VERIFIER_PRECEDENCE = "actor" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +service = "main" +command = "test -s /app/actor.pid && ! kill -0 \"$(cat /app/actor.pid)\" 2>/dev/null && curl -fsS --max-time 10 http://127.0.0.1:8080/ >/dev/null && head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" +timeout_sec = 10 + +[[verifier.collect]] +service = "workspace" +command = "echo collected-from-sidecar > /tmp/sidecar.txt" +timeout_sec = 10 + +[verifier.environment] +cpus = 1 diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/Dockerfile b/tests/integrations/harbor/tasks/sidecar-reachability/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/Dockerfile rename to tests/integrations/harbor/tasks/sidecar-reachability/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/sidecar-reachability/environment/docker-compose.yaml similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/docker-compose.yaml rename to tests/integrations/harbor/tasks/sidecar-reachability/environment/docker-compose.yaml diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/instruction.md b/tests/integrations/harbor/tasks/sidecar-reachability/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/instruction.md rename to tests/integrations/harbor/tasks/sidecar-reachability/instruction.md diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh b/tests/integrations/harbor/tasks/sidecar-reachability/solution/solve.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh rename to tests/integrations/harbor/tasks/sidecar-reachability/solution/solve.sh diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml b/tests/integrations/harbor/tasks/sidecar-reachability/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml rename to tests/integrations/harbor/tasks/sidecar-reachability/task.toml diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/Dockerfile b/tests/integrations/harbor/tasks/sidecar-reachability/tests/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/Dockerfile rename to tests/integrations/harbor/tasks/sidecar-reachability/tests/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/test.sh b/tests/integrations/harbor/tasks/sidecar-reachability/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/test.sh rename to tests/integrations/harbor/tasks/sidecar-reachability/tests/test.sh diff --git a/tests/integrations/harbor/tasks/vendor-uv/environment/Dockerfile b/tests/integrations/harbor/tasks/vendor-uv/environment/Dockerfile new file mode 100644 index 000000000..d68e1c5bf --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.11-slim +WORKDIR /workspace +COPY pyproject.toml . diff --git a/tests/integrations/harbor/tasks/vendor-uv/environment/pyproject.toml b/tests/integrations/harbor/tasks/vendor-uv/environment/pyproject.toml new file mode 100644 index 000000000..cb4e94f57 --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/environment/pyproject.toml @@ -0,0 +1,4 @@ +[tool.uv] +required-version = "==0.0.0" +[tool.uv.pip] +no-index = true diff --git a/tests/integrations/harbor/tasks/vendor-uv/expected.toml b/tests/integrations/harbor/tasks/vendor-uv/expected.toml new file mode 100644 index 000000000..a1ae3d61e --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/expected.toml @@ -0,0 +1 @@ +base = "agent-answer" diff --git a/tests/integrations/harbor/tasks/vendor-uv/solution/solve.sh b/tests/integrations/harbor/tasks/vendor-uv/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/vendor-uv/tests/test.sh b/tests/integrations/harbor/tasks/vendor-uv/tests/test.sh new file mode 100644 index 000000000..be1bb4062 --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p /logs/verifier +if grep -q 'required-version = "==0.0.0"' /workspace/pyproject.toml; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/verifier-child-user/expected.toml b/tests/integrations/harbor/tasks/verifier-child-user/expected.toml new file mode 100644 index 000000000..f9f1d4d05 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/expected.toml @@ -0,0 +1 @@ +base = "sidecar-reachability" diff --git a/tests/integrations/harbor/tasks/verifier-child-user/tests/Dockerfile b/tests/integrations/harbor/tasks/verifier-child-user/tests/Dockerfile new file mode 100644 index 000000000..3635db289 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/tests/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.12-alpine + +COPY . /tests +RUN adduser -D -u 1001 verifier && mkdir -p /logs/verifier + +USER root +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/verifier-child-user/tests/test.sh b/tests/integrations/harbor/tasks/verifier-child-user/tests/test.sh new file mode 100644 index 000000000..dbec9e336 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/tests/test.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +mkdir -p /logs/verifier +if su verifier -s /bin/sh -c 'test "$(cat /root/agent-output.txt)" = private'; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/expected.toml b/tests/integrations/harbor/tasks/verifier-credentials-separate/expected.toml new file mode 100644 index 000000000..b3645eeaa --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/expected.toml @@ -0,0 +1,10 @@ +# A verifier credential is resolved only when grading needs it. +base = "verifier-credentials-shared" +[[case]] +name = "missing" +reward = 0.0 +error = "HARBOR_VERIFIER_KEY" + +[[case]] +name = "provided" +runtime.env_vars = { HARBOR_VERIFIER_KEY = "judge-value" } diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/task.toml b/tests/integrations/harbor/tasks/verifier-credentials-separate/task.toml new file mode 100644 index 000000000..b5d08ed36 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/task.toml @@ -0,0 +1,6 @@ +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[verifier.env] +JUDGE_KEY = "${HARBOR_VERIFIER_KEY}" diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/tests/Dockerfile b/tests/integrations/harbor/tasks/verifier-credentials-separate/tests/Dockerfile new file mode 100644 index 000000000..3d6ba54af --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +COPY . /tests diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/expected.toml b/tests/integrations/harbor/tasks/verifier-credentials-shared/expected.toml new file mode 100644 index 000000000..9f71774a8 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/expected.toml @@ -0,0 +1,10 @@ +# A verifier credential is resolved only when grading needs it. +base = "agent-answer" +[[case]] +name = "missing" +reward = 0.0 +error = "HARBOR_VERIFIER_KEY" + +[[case]] +name = "provided" +runtime.env_vars = { HARBOR_VERIFIER_KEY = "judge-value" } diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-credentials-shared/solution/solve.sh new file mode 100644 index 000000000..8a52a7e29 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +[ "${JUDGE_KEY-unset}" = unset ] diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/task.toml b/tests/integrations/harbor/tasks/verifier-credentials-shared/task.toml new file mode 100644 index 000000000..4ce85c14e --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/task.toml @@ -0,0 +1,6 @@ +[verifier] +environment_mode = "shared" +timeout_sec = 30 + +[verifier.env] +JUDGE_KEY = "${HARBOR_VERIFIER_KEY}" diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/tests/test.sh b/tests/integrations/harbor/tasks/verifier-credentials-shared/tests/test.sh new file mode 100644 index 000000000..706bd41ea --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +[ "$JUDGE_KEY" = "judge-value" ] +echo 1 > /logs/verifier/reward.txt diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-lifecycle/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/environment/Dockerfile rename to tests/integrations/harbor/tasks/verifier-lifecycle/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/environment/entrypoint.sh b/tests/integrations/harbor/tasks/verifier-lifecycle/environment/entrypoint.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/environment/entrypoint.sh rename to tests/integrations/harbor/tasks/verifier-lifecycle/environment/entrypoint.sh diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/instruction.md b/tests/integrations/harbor/tasks/verifier-lifecycle/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/instruction.md rename to tests/integrations/harbor/tasks/verifier-lifecycle/instruction.md diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-lifecycle/solution/solve.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/solution/solve.sh rename to tests/integrations/harbor/tasks/verifier-lifecycle/solution/solve.sh diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/task.toml b/tests/integrations/harbor/tasks/verifier-lifecycle/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/task.toml rename to tests/integrations/harbor/tasks/verifier-lifecycle/task.toml diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/tests/test.sh b/tests/integrations/harbor/tasks/verifier-lifecycle/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/tests/test.sh rename to tests/integrations/harbor/tasks/verifier-lifecycle/tests/test.sh diff --git a/tests/integrations/harbor/tasks/verifier-restore/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-restore/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/verifier-restore/expected.toml b/tests/integrations/harbor/tasks/verifier-restore/expected.toml new file mode 100644 index 000000000..79ff50716 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/expected.toml @@ -0,0 +1,2 @@ +group = 2 +shared_width = 1 diff --git a/tests/integrations/harbor/tasks/verifier-restore/instruction.md b/tests/integrations/harbor/tasks/verifier-restore/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/verifier-restore/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-restore/solution/solve.sh new file mode 100644 index 000000000..75eebb66d --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/solution/solve.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +mkdir -p /opt/result +echo agent > /opt/result/agent.txt diff --git a/tests/integrations/harbor/tasks/verifier-restore/task.toml b/tests/integrations/harbor/tasks/verifier-restore/task.toml new file mode 100644 index 000000000..c5b491e33 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/task.toml @@ -0,0 +1,5 @@ +artifacts = [{ source = "/opt/result", service = "main" }] + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/verifier-restore/tests/Dockerfile b/tests/integrations/harbor/tasks/verifier-restore/tests/Dockerfile new file mode 100644 index 000000000..1a8b84d50 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/tests/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-alpine +RUN mkdir -p /opt/result && echo base > /opt/result/base.txt +COPY . /tests diff --git a/tests/integrations/harbor/tasks/verifier-restore/tests/test.sh b/tests/integrations/harbor/tasks/verifier-restore/tests/test.sh new file mode 100644 index 000000000..d4e1de262 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/tests/test.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -u +mkdir -p /logs/verifier +fail() { echo "$1"; echo 0 > /logs/verifier/reward.txt; exit 0; } +[ ! -e /opt/stray.txt ] || fail "a previous verifier's file survived" +[ ! -e /opt/result.moved ] || fail "a previous verifier's move survived" +[ "$(ls /opt/result)" = "agent.txt" ] || fail "the artifact did not replace the image path" +echo stray > /opt/stray.txt +mv /opt/result /opt/result.moved +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/verifier-shebang/expected.toml b/tests/integrations/harbor/tasks/verifier-shebang/expected.toml new file mode 100644 index 000000000..f9f1d4d05 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/expected.toml @@ -0,0 +1 @@ +base = "sidecar-reachability" diff --git a/tests/integrations/harbor/tasks/verifier-shebang/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-shebang/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/verifier-shebang/task.toml b/tests/integrations/harbor/tasks/verifier-shebang/task.toml new file mode 100644 index 000000000..1574ed7b9 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/task.toml @@ -0,0 +1,6 @@ +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/verifier-shebang/tests/test.sh b/tests/integrations/harbor/tasks/verifier-shebang/tests/test.sh new file mode 100644 index 000000000..8d661956a --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/tests/test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +from pathlib import Path + +Path("/logs/verifier/reward.txt").write_text("1") diff --git a/tests/integrations/harbor/tasks/verifier-timeout/expected.toml b/tests/integrations/harbor/tasks/verifier-timeout/expected.toml new file mode 100644 index 000000000..8603624e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/expected.toml @@ -0,0 +1,3 @@ +base = "agent-answer" +reward = 0.0 +error = "Harbor verifier timed out after 2s" diff --git a/tests/integrations/harbor/tasks/verifier-timeout/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-timeout/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/verifier-timeout/task.toml b/tests/integrations/harbor/tasks/verifier-timeout/task.toml new file mode 100644 index 000000000..72939071e --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/task.toml @@ -0,0 +1,2 @@ +[verifier] +timeout_sec = 2 diff --git a/tests/integrations/harbor/tasks/verifier-timeout/tests/test.sh b/tests/integrations/harbor/tasks/verifier-timeout/tests/test.sh new file mode 100644 index 000000000..d85ae4225 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +sleep 30 +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/test_adapt.py b/tests/integrations/harbor/test_adapt.py new file mode 100644 index 000000000..d10838ee6 --- /dev/null +++ b/tests/integrations/harbor/test_adapt.py @@ -0,0 +1,280 @@ +"""``harbor.adapt`` over case directories of Harbor tasks. + +Each directory under ``cases/`` is a Harbor dataset; adaptation runs against the +local Docker daemon. Every case is held to what holds for any dataset (rows that +match the written ``tasks.json``, environments copied whole, Compose projects +that resolve, no host secret written out) and to its ``expected.toml``, whose +keys are all optional: + +- ``findings``: ``[task, code]`` for each finding adaptation reports; +- ``mentions`` and ``mentions_off_amd64``: per task, text its finding messages + contain (the second exactly when the Docker daemon is not amd64); +- ``projects``: the tasks that share one project, by environment; +- ``task.``: dotted paths into the row's observed fields (``placement``, + ``agent_config``, ``task``, ``service_access``, ``services``, ``compose``, + ``verifier.runtime_config``, ``links``), each equal to its value; +- ``base``: another case this one starts from (``assemble_case``). + +Adapted tasks run end to end in ``test_controller``. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.integrations import harbor +from tests.integrations.harbor.cases import assemble_case, read_expected + +if TYPE_CHECKING: + from collections.abc import Callable + + from hud.eval import RuntimeConfig, Task + from tests.support import HudEnv + +CASES = Path(__file__).parent / "cases" +SECRET = "sk-live-secret" + + +def load_case(case: str, tmp_path: Path, hud_env: HudEnv) -> Path: + """Assemble ``case`` in ``tmp_path``, with a host secret its templates must never leak.""" + dataset = tmp_path / case + assemble_case(CASES / case, dataset) + hud_env.set(HARBOR_JUDGE_KEY=SECRET) + return dataset + + +def placement(config: RuntimeConfig | None) -> dict[str, Any]: + """A runtime config's resources and limits, as the platform reads them.""" + if config is None: + return {} + return config.model_dump(mode="json", exclude_none=True, include={"resources", "limits"}) + + +def file_tree(root: Path) -> dict[str, bytes | str]: + return { + entry.relative_to(root).as_posix(): ( + f"-> {os.readlink(entry)}" if entry.is_symlink() else entry.read_bytes() + ) + for entry in sorted(root.rglob("*")) + if entry.is_symlink() or entry.is_file() + } + + +def observed(row: Task) -> dict[str, Any]: + """The fields of an adapted row that ``expected.toml`` paths point into.""" + config = row.runtime_config + assert config is not None and config.compose is not None + root, document = config.compose.root, config.compose.document + assert isinstance(root, Path) and isinstance(document, Path) + compose = json.loads(document.read_text("utf-8")) + verifier = row.verifier + return { + "placement": placement(config), + "agent_config": row.agent_config, + "task": row.args["task"], + "service_access": config.compose.service_access, + "services": sorted(compose["services"]), + "compose": compose, + "verifier": verifier is not None + and { + "runtime_config": verifier.runtime_config is not None + and placement(verifier.runtime_config) + }, + "links": { + entry.relative_to(root).as_posix(): os.readlink(entry) + for entry in root.rglob("*") + if entry.is_symlink() + }, + } + + +def assert_stock_compose_complete(compose_path: Path) -> None: + """Every service has an image, and every path a build reads resolves inside the project.""" + project_root = compose_path.parent.resolve() + services = json.loads(compose_path.read_text("utf-8"))["services"] + for name, service in services.items(): + assert service.get("image"), name + for volume in service.get("volumes", []): + if isinstance(volume, str) and volume.endswith(":/controller/tests:ro"): + tests = (project_root / volume.partition(":")[0]).resolve() + assert tests.is_relative_to(project_root) and tests.is_dir(), (name, tests) + build = service.get("build") + if build is None: + continue + build = {"context": build} if isinstance(build, str) else build + context = (project_root / build.get("context", ".")).resolve() + assert context.is_relative_to(project_root) and context.is_dir(), (name, context) + assert (context / build.get("dockerfile", "Dockerfile")).is_file(), name + for target in build.get("additional_contexts", {}).values(): + if target.startswith("service:"): + assert target.removeprefix("service:") in services, (name, target) + else: + named = (project_root / target).resolve() + assert named.is_relative_to(project_root) and named.is_dir(), (name, named) + + +@pytest.mark.e2e +@pytest.mark.docker +@pytest.mark.parametrize("case", sorted(path.name for path in CASES.iterdir())) +def test_adapting_a_dataset_writes_the_projects_and_findings_its_case_expects( + case: str, tmp_path: Path, hud_env: HudEnv +) -> None: + spec = read_expected(CASES / case) + dataset = load_case(case, tmp_path, hud_env) + + result = harbor.adapt(dataset, hud_requirement="hud") + + found = [ + (failure.task, finding.code) for failure in result.failures for finding in failure.findings + ] + assert sorted(found) == sorted(map(tuple, spec.get("findings", []))) + messages = { + failure.task: " ".join(finding.message for finding in failure.findings) + for failure in result.failures + } + for task, words in spec.get("mentions", {}).items(): + assert all(word in messages[task] for word in words), (task, messages[task]) + if off_amd64 := spec.get("mentions_off_amd64"): + arch = subprocess.run( + ["docker", "version", "--format", "{{.Server.Arch}}"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + for task, words in off_amd64.items(): + assert all(word in messages[task] for word in words) is (arch != "amd64"), task + + projects = sorted((dataset / ".hud-adapt").iterdir()) + written = sorted( + row["slug"] + for project in projects + for row in json.loads((project / "tasks.json").read_text("utf-8")) + ) + rows = {row.slug or "": row for row in result.taskset} + assert sorted(rows) == written + for project in projects: + assert_stock_compose_complete(project / "compose.yaml") + for slug, row in rows.items(): + assert row.runtime_config is not None and row.runtime_config.compose is not None + root = row.runtime_config.compose.root + assert isinstance(root, Path) + assert file_tree(root / "compose-project" / "environment") == file_tree( + dataset / slug / "environment" + ) + leaked = [ + path + for path in (dataset / ".hud-adapt").rglob("*") + if path.is_file() and not path.is_symlink() and SECRET.encode() in path.read_bytes() + ] + assert leaked == [] + + if "projects" in spec: + groups: dict[str, list[str]] = {} + for slug, row in sorted(rows.items()): + groups.setdefault(row.env, []).append(slug) + assert sorted(groups.values()) == sorted(spec["projects"]) + for slug, fields in spec.get("task", {}).items(): + view = observed(rows[slug]) + for path, value in fields.items(): + actual: Any = view + for key in path.split("."): + actual = actual[key] + assert actual == value, (slug, path) + + +def test_adapt_raises_when_the_source_holds_no_harbor_tasks(tmp_path: Path) -> None: + (tmp_path / "empty").mkdir() + + for source in (tmp_path / "empty", tmp_path / "nowhere"): + with pytest.raises(ValueError, match="no Harbor tasks found"): + harbor.adapt(source) + + +@pytest.mark.parametrize( + ("docker_state", "message"), + [ + ("daemon unreachable", "requires a reachable Docker daemon: docker version"), + ("no docker executable", "requires Docker to resolve authored images"), + ], +) +def test_adapt_aborts_without_a_docker_daemon_instead_of_failing_tasks( + docker_state: str, message: str, tmp_path: Path, hud_env: HudEnv +) -> None: + dataset = load_case("plain-image", tmp_path, hud_env) + if docker_state == "daemon unreachable": + hud_env.set(DOCKER_HOST=f"unix://{tmp_path}/no-daemon.sock") + else: + (tmp_path / "empty-path").mkdir() + hud_env.set(PATH=str(tmp_path / "empty-path")) + + with pytest.raises(RuntimeError, match=message): + harbor.adapt(dataset) + + +def edit(relative: str, content: str) -> Callable[[Path], str]: + def apply(task: Path) -> str: + (task / relative).write_text(content, encoding="utf-8") + return "hud" + + return apply + + +def wheel(task: Path) -> str: + built = task.parent.parent / "hud-0.0.0-py3-none-any.whl" + built.write_bytes(b"wheel") + return str(built) + + +@pytest.mark.e2e +@pytest.mark.docker +@pytest.mark.parametrize( + ("change", "apply", "rebuilds"), + [ + ("an edited instruction", edit("instruction.md", "Second instruction\n"), False), + ("edited inline tests", edit("tests/test.sh", "#!/bin/sh\nexit 1\n"), False), + ("a new verifier timeout", edit("task.toml", "[verifier]\ntimeout_sec = 5\n"), False), + ("new content behind an environment symlink", edit("../outside.txt", "changed\n"), False), + ("an edited environment file", edit("environment/ignored.txt", "changed\n"), True), + ("another hud release", lambda task: "hud==0.0.1", True), + ("a hud wheel", wheel, True), + ], +) +def test_the_environment_image_identity_follows_only_what_the_image_contains( + change: str, + apply: Callable[[Path], str], + rebuilds: bool, + tmp_path: Path, + hud_env: HudEnv, +) -> None: + del change + dataset = load_case("plain-image", tmp_path, hud_env) + task = dataset / "task-a" + (dataset / "outside.txt").write_text("first\n", encoding="utf-8") + (task / "environment" / "outside").symlink_to(dataset / "outside.txt") + + def images(requirement: str) -> tuple[dict[str, Any], Path]: + (row,) = harbor.adapt(dataset, hud_requirement=requirement).taskset + assert row.runtime_config is not None and row.runtime_config.compose is not None + assert isinstance(row.runtime_config.compose.document, Path) + compose = json.loads(row.runtime_config.compose.document.read_text("utf-8")) + return compose["services"]["main"], row.runtime_config.compose.document.parent + + before, _ = images("hud") + after, project = images(apply(task)) + + assert (after["image"] != before["image"]) is rebuilds + requirement = after["build"]["args"]["HUD_REQUIREMENT"] + packages = sorted(path.name for path in (project / "compose-project/main/packages").iterdir()) + if requirement.endswith(".whl"): + assert (requirement, packages) == ( + "/controller/packages/hud-0.0.0-py3-none-any.whl", + ["hud-0.0.0-py3-none-any.whl"], + ) + else: + assert packages == [] diff --git a/tests/integrations/harbor/test_controller.py b/tests/integrations/harbor/test_controller.py new file mode 100644 index 000000000..b68dd6fd7 --- /dev/null +++ b/tests/integrations/harbor/test_controller.py @@ -0,0 +1,216 @@ +"""The in-image Harbor controller, adapted from fixture tasks and run in Docker. + +Each directory under ``tasks/`` is a Harbor task with a reference +``solution/solve.sh`` and an optional ``expected.toml``. The scenario adapts +the task against this checkout's wheel and runs it under ``DockerRuntime`` with +an oracle that only runs the solution, so the verifier's reward proves the +controller kept each phase's contract. ``expected.toml`` keys, all optional, +at the top level or per ``[[case]]`` (a case adds to the top level): + +- ``reward``: the reward every run must get (default ``1.0``); +- ``error``: text the run's error or evaluation must contain; +- ``repeat``: run the solution in this many SSH sessions; +- ``group`` and ``shared_width``: rollouts per task, sharing placements; +- ``capability_probe``: ``{name, tools}`` an MCP capability opened host-side lists; +- ``runtime.env_vars`` and ``runtime.bind_mounts`` (``source`` relative to the + fixture): what ``DockerRuntime`` gives the container; +- ``startup_error``: the controller must refuse to serve, with this message; +- ``base``: another fixture this one starts from (``assemble_case``), so a + variant holds only what it changes. +""" + +from __future__ import annotations + +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.agents.base import Agent +from hud.capabilities import MCPClient, SSHClient +from hud.eval import DockerRuntime, Shared +from hud.eval.runtime import DockerBindMount +from hud.integrations import harbor +from tests.integrations.harbor.cases import assemble_case, read_expected + +if TYPE_CHECKING: + from hud.eval import Provider + from hud.eval.run import Run + +TASKS = Path(__file__).parent / "tasks" + +pytestmark = [ + pytest.mark.e2e, + pytest.mark.docker, + pytest.mark.skipif(sys.platform == "win32", reason="adapted images are Linux containers"), +] + + +@dataclass(frozen=True) +class Case: + fixture: str + name: str + reward: float + error: str | None + repeat: int + group: int | None + shared_width: int | None + probe: dict[str, Any] | None + env_vars: dict[str, str] + bind_mounts: tuple[DockerBindMount, ...] + startup_error: str | None + + @property + def id(self) -> str: + return f"{self.fixture}[{self.name}]" if self.name else self.fixture + + +def load_cases() -> list[Case]: + cases = [] + fixtures = (path for path in TASKS.iterdir() if path.is_dir() and path.name != "__pycache__") + for root in sorted(fixtures): + spec = read_expected(root) + spec.pop("base", None) + for variant in spec.pop("case", None) or [{}]: + merged = {**spec, **variant} + runtime = {**spec.get("runtime", {}), **variant.get("runtime", {})} + cases.append( + Case( + fixture=root.name, + name=merged.get("name", ""), + reward=merged.get("reward", 1.0), + error=merged.get("error"), + repeat=merged.get("repeat", 1), + group=merged.get("group"), + shared_width=merged.get("shared_width"), + probe=merged.get("capability_probe"), + env_vars=runtime.get("env_vars", {}), + bind_mounts=tuple( + DockerBindMount(root / mount["source"], mount["target"]) + for mount in runtime.get("bind_mounts", []) + ), + startup_error=merged.get("startup_error"), + ) + ) + return cases + + +CASES = load_cases() +RUNNABLE = [case for case in CASES if case.startup_error is None] +REFUSED = [case for case in CASES if case.startup_error is not None] + + +class Oracle(Agent): + """Run the fixture's reference solution; its stdout is the answer.""" + + def __init__(self, solution: str, *, repeat: int, probe: str | None) -> None: + super().__init__() + self.solution = solution + self.repeat = repeat + self.probe = probe + self.probed: list[list[str]] = [] + + async def __call__(self, run: Run) -> None: + if self.probe is not None: + mcp = await run.client.open(self.probe) + assert isinstance(mcp, MCPClient) + self.probed.append(sorted(tool.name for tool in await mcp.list_tools())) + ssh = await run.client.open("ssh") + assert isinstance(ssh, SSHClient) + stdout = "" + for session in range(self.repeat): + result = await ssh.run(self.solution, check=False) + if result.exit_status != 0: + raise RuntimeError( + f"solution session {session} exited with {result.exit_status}:\n" + f"{result.stdout}\n{result.stderr}" + ) + stdout = str(result.stdout) + run.trace.content = stdout + + +def adapted(case: Case, sdk_wheel: Path, tmp_path: Path) -> harbor.AdaptResult: + dataset = tmp_path / "harbor-fixtures" + assemble_case(TASKS / case.fixture, dataset / case.fixture, "managed") + result = harbor.adapt(dataset, hud_requirement=str(sdk_wheel)) + assert result.failures == () + return result + + +def report(run: Run) -> str: + """Everything a failed run says about why, for assertions and their messages.""" + evaluation = run.evaluation + info = evaluation.get("info") or {} + return "\n".join( + str(part) + for part in ( + run.trace.content, + run.trace.error, + evaluation.get("content"), + info.get("stdout"), + info.get("stderr"), + ) + if part + ) + + +@pytest.mark.parametrize("case", RUNNABLE, ids=[case.id for case in RUNNABLE]) +async def test_a_harbor_task_grades_as_its_fixture_expects( + case: Case, sdk_wheel: Path, tmp_path: Path +) -> None: + taskset = adapted(case, sdk_wheel, tmp_path).taskset + docker = DockerRuntime(env_vars=case.env_vars, bind_mounts=case.bind_mounts) + runtime: Provider = Shared(docker, width=case.shared_width) if case.shared_width else docker + oracle = Oracle( + (tmp_path / "harbor-fixtures" / case.fixture / "solution" / "solve.sh").read_text("utf-8"), + repeat=case.repeat, + probe=None if case.probe is None else case.probe["name"], + ) + + job = await taskset.run(oracle, runtime=runtime, group=case.group, max_concurrent=1) + + assert len(job.runs) == (case.group or 1) + for run in job.runs: + detail = report(run) + assert run.reward == case.reward, detail + if case.error is not None: + assert case.error in detail + if case.probe is not None: + assert oracle.probed == [sorted(case.probe["tools"])] * len(job.runs) + + +@pytest.mark.parametrize("case", REFUSED, ids=[case.id for case in REFUSED]) +def test_a_controller_with_an_unusable_configuration_refuses_to_serve( + case: Case, sdk_wheel: Path, tmp_path: Path +) -> None: + # Providers yield once the control port is published, before the serve + # process proves itself, so the refusal is observed by running the main + # service of the adapted project in the foreground. + (row,) = adapted(case, sdk_wheel, tmp_path).taskset + assert row.runtime_config is not None and row.runtime_config.compose is not None + compose = row.runtime_config.compose.document + assert isinstance(compose, Path) + + def compose_command(*args: str, timeout: float) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", "compose", "--file", str(compose), *args], + cwd=compose.parent, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + try: + built = compose_command("build", timeout=1200) + assert built.returncode == 0, built.stderr[-4000:] + served = compose_command("run", "--rm", "main", timeout=120) + finally: + compose_command("down", "--volumes", "--remove-orphans", timeout=120) + + assert served.returncode != 0 + assert case.startup_error is not None + assert case.startup_error in served.stdout + served.stderr diff --git a/tests/integrations/harbor/test_export.py b/tests/integrations/harbor/test_export.py new file mode 100644 index 000000000..26bce9334 --- /dev/null +++ b/tests/integrations/harbor/test_export.py @@ -0,0 +1,188 @@ +"""``harbor.export`` writes Harbor task folders whose scripts drive the ``hud`` CLI. + +Each directory under ``sources/`` is a HUD source; ``expected.toml`` says what +exporting it yields. Keys, all optional: + +- ``export``: the file export reads (default ``env.py``); +- ``answer_file``: where the instruction tells the agent to answer; +- ``folders.``: a folder export writes, with the ``task`` and ``args`` its + ``task.toml`` carries and the ``prompt`` its instruction starts with; +- ``folders..grade``: in the docker lane, an ``answer`` written into the + started image and the ``exit`` and ``reward`` (absent: none) of ``tests/test.sh``; +- ``folders..run``: in the docker lane, whether the agent's command + ``succeeds`` in the image under ``env``; +- ``refusal``: the error ``type`` and ``message`` export raises instead, and + whether it ``started`` the source environment first; +- ``base``: another source this one starts from (``assemble_case``). + +Every exported folder is held to what holds for any source: the source image +with HUD's entrypoint as its final startup directives, the source's files in its +build context, and an instruction naming where to answer. Export stops every +environment it starts. +""" + +from __future__ import annotations + +import asyncio +import json +import re +import shutil +import subprocess +import tomllib +import uuid +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.integrations.harbor import export +from tests.integrations.harbor.cases import assemble_case, read_expected + +if TYPE_CHECKING: + from tests.support import HudEnv + +SOURCES = Path(__file__).parent / "sources" +ENTRYPOINT = ['ENTRYPOINT ["/hud_entrypoint.sh"]', 'CMD ["sh", "-c", "sleep infinity"]'] +CASES = sorted(path.parent.name for path in SOURCES.glob("*/expected.toml")) +DOCKER_CASES = [ + case + for case in CASES + if any( + "grade" in folder or "run" in folder + for folder in read_expected(SOURCES / case).get("folders", {}).values() + ) +] + + +def names(directory: Path) -> list[str]: + return sorted(path.name for path in directory.iterdir()) if directory.exists() else [] + + +@pytest.fixture +def markers(tmp_path: Path, hud_env: HudEnv) -> Path: + """Where the source environment's initialize and shutdown hooks leave a file.""" + directory = tmp_path / "markers" + directory.mkdir() + hud_env.set(EXPORT_TEST_MARKERS=str(directory)) + return directory + + +@pytest.mark.parametrize("case", CASES) +async def test_exporting_a_source_writes_the_folders_its_case_expects( + case: str, tmp_path: Path, markers: Path +) -> None: + spec = read_expected(SOURCES / case) + source = tmp_path / "source" + assemble_case(SOURCES / case, source) + exported = source / spec.get("export", "env.py") + answer_file = spec.get("answer_file", "/workspace/answer.txt") + + if refusal := spec.get("refusal"): + with pytest.raises(Exception, match=re.escape(refusal["message"])) as raised: + await export(str(exported), tmp_path / "out", answer_file=answer_file) + assert type(raised.value).__name__ == refusal["type"] + assert names(tmp_path / "out") == [] + assert names(markers) == (["initialized", "stopped"] if refusal["started"] else []) + return + + created = await export(str(exported), tmp_path / "out", answer_file=answer_file) + + folders = spec["folders"] + assert sorted(task_dir.name for task_dir in created) == sorted(folders) + image = next( + source / name for name in ("Dockerfile.hud", "Dockerfile") if (source / name).is_file() + ).read_text("utf-8") + rewritten = {"Dockerfile", "Dockerfile.hud", ".dockerignore"} + if exported.suffix != ".py": + rewritten.add(exported.name) + copied = [path for path in source.iterdir() if path.is_file() and path.name not in rewritten] + for task_dir in created: + expected = folders[task_dir.name] + task = tomllib.loads((task_dir / "task.toml").read_text("utf-8")) + assert task["metadata"]["hud_task"] == expected["task"] + assert json.loads(task["metadata"]["hud_args"]) == expected["args"] + assert (task_dir / "tests" / "test.sh").is_file() + environment = task_dir / "environment" + assert (environment / "hud_entrypoint.sh").is_file() + dockerfile = (environment / "Dockerfile").read_text("utf-8") + assert dockerfile.startswith(image) + assert [line for line in dockerfile.splitlines() if line][-2:] == ENTRYPOINT + if (source / ".dockerignore").is_file(): + ignored = (environment / ".dockerignore").read_text("utf-8") + assert ignored.startswith((source / ".dockerignore").read_text("utf-8")) + assert ignored.splitlines()[-1] == "!hud_entrypoint.sh" + for path in copied: + assert (environment / path.name).read_bytes() == path.read_bytes(), path.name + instruction = (task_dir / "instruction.md").read_text("utf-8") + assert instruction.startswith(expected["prompt"]) + assert f"`{answer_file}`" in instruction + # Export loads the source to read its rows, and stops what it started. + assert names(markers) == ["initialized", "stopped"] + + +def docker(*args: str, timeout: float = 600) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", *args], capture_output=True, text=True, timeout=timeout, check=False + ) + + +async def wait_for_process(container: str, command: str, deadline: float = 240) -> bool: + """Whether ``command`` runs in ``container`` before ``deadline`` seconds pass.""" + loop = asyncio.get_running_loop() + end = loop.time() + deadline + while loop.time() < end: + if command in docker("top", container).stdout: + return True + await asyncio.sleep(1) + return False + + +async def grade(task_dir: Path, image: str, answer: str) -> tuple[int, str | None]: + """Start ``image`` as Harbor does, answer, run ``tests/test.sh``; its exit and reward.""" + started = docker("run", "--detach", "--rm", image) + assert started.returncode == 0, started.stderr + container = started.stdout.strip() + try: + parked = await wait_for_process(container, "sleep infinity") + assert parked, docker("logs", container).stderr[-4000:] + assert docker("cp", str(task_dir / "tests"), f"{container}:/tests").returncode == 0 + written = docker( + "exec", container, "sh", "-c", f"printf %s {answer} > /workspace/answer.txt" + ) + assert written.returncode == 0, written.stderr + graded = docker("exec", container, "sh", "/tests/test.sh", timeout=120) + read = docker("exec", container, "cat", "/logs/verifier/reward.txt") + finally: + docker("rm", "--force", container) + return graded.returncode, read.stdout.strip() if read.returncode == 0 else None + + +@pytest.mark.e2e +@pytest.mark.docker +@pytest.mark.parametrize("case", DOCKER_CASES) +async def test_an_exported_task_runs_and_grades_inside_its_built_image( + case: str, sdk_wheel: Path, tmp_path: Path +) -> None: + spec = read_expected(SOURCES / case) + source = tmp_path / "source" + assemble_case(SOURCES / case, source) + shutil.copy2(sdk_wheel, source / sdk_wheel.name) + created = await export(str(source / spec.get("export", "env.py")), tmp_path / "out") + + for task_dir in created: + expected: dict[str, Any] = spec["folders"][task_dir.name] + if not ("grade" in expected or "run" in expected): + continue + image = f"hud-export-test:{uuid.uuid4().hex[:12]}" + build = docker("build", "--tag", image, str(task_dir / "environment")) + assert build.returncode == 0, build.stderr[-4000:] + try: + for row in expected.get("grade", []): + graded = await grade(task_dir, image, row["answer"]) + assert graded == (row["exit"], row.get("reward")), row + for row in expected.get("run", []): + env = [f"--env={name}={value}" for name, value in row["env"].items()] + ran = docker("run", "--rm", *env, image, "true", timeout=240) + assert (ran.returncode == 0) is row["succeeds"], ran.stderr[-2000:] + finally: + docker("image", "rm", "--force", image) diff --git a/tests/robot/__init__.py b/tests/robot/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/robot/sims.py b/tests/robot/sims.py new file mode 100644 index 000000000..f876c4ea2 --- /dev/null +++ b/tests/robot/sims.py @@ -0,0 +1,424 @@ +"""The sims robot scenarios run, the envs that serve them, and the agents that drive them. + +A spawned sim re-imports its bridge class or env factory from this file by path, +so they live at module level. Each gym env reports, in its observation, the +shape of the last action ``step`` got. + +:class:`Arm` is the direct-control sim: its observed state follows each +commanded action by a ``follow`` law, so one class plays every arm a settle +verdict is about. :class:`ScriptedLLM` stands in for a tool-calling model. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any, cast + +import numpy as np +from gymnasium.spaces import Box + +from hud.agents.base import Agent +from hud.environment import Environment +from hud.environment.robot import DirectControl, RobotBridge, RobotEndpoint +from hud.eval import LocalRuntime + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + + from numpy.typing import NDArray + + from hud.capabilities import MCPClient + from hud.eval import Provider + from hud.eval.run import Run + from hud.types import MCPToolResult + + +@asynccontextmanager +async def sim_env( + sim: RobotBridge, name: str, *, control: DirectControl | None = None +) -> AsyncIterator[tuple[Environment, RobotEndpoint]]: + """The docs' custom-bridge env for *sim*, which this process serves, started. + + ``RobotEndpoint.remote`` attaches to a sim someone else runs; here that is + the test, so the whole control plane is real without a spawned sim program. + *control* adds direct control's motion tools. Each scenario declares the + template it is about. + """ + await sim.start() + server = await sim.serve_control() + env = Environment(name) + endpoint = RobotEndpoint.remote("127.0.0.1", server.sockets[0].getsockname()[1]).attach(env) + if control is not None: + control.attach(endpoint) + + @env.initialize + async def up() -> None: + await endpoint.start() + for capability in await endpoint.capabilities(): + env.add_capability(capability) + + @env.shutdown + async def down() -> None: + await endpoint.stop() + + try: + await env.start() + yield env, endpoint + finally: + await env.stop() + server.close() + await sim.stop() + + +# ─── direct control ──────────────────────────────────────────────────── + + +def contract( + action_type: str, + low: list[float], + high: list[float], + names: list[str] | None = None, + *, + rate: float = 10, +) -> dict[str, Any]: + """One camera, a state, and an action of *action_type*. Default action is ``[x, grip]``.""" + labels = names or ["x", "grip"] + return { + "control_rate": rate, + "features": { + "observation/image": {"role": "observation", "type": "rgb"}, + "observation/state": {"role": "observation", "names": list(labels)}, + "action": { + "role": "action", + "type": action_type, + "names": list(labels), + "stats": {"min": low, "max": high}, + }, + }, + } + + +class Arm(RobotBridge): + """An arm whose state becomes ``follow(arm, action)`` on every tick. + + It succeeds, which ends the episode, once ``done(state)`` holds. ``reads`` + maps the state to what the observation reports; ``side`` is the camera's. + """ + + def __init__( + self, + contract: dict[str, Any], + home: list[float] | None = None, + follow: Callable[[Arm, NDArray[np.float64]], NDArray[np.float64]] | None = None, + *, + done: Callable[[NDArray[np.float64]], bool] | None = None, + reads: Callable[[NDArray[np.float64]], NDArray[np.float64]] | None = None, + side: int = 8, + ) -> None: + super().__init__() + self.contract = contract + width = len(contract["features"]["action"]["names"]) + self.home = np.zeros(width) if home is None else np.asarray(home, dtype=np.float64) + self.follow = follow or snap + self.done = done + self.reads = reads + self.side = side + self.state = self.home.copy() + self.actions: list[NDArray[np.float64]] = [] + + def reset(self, **task_args: Any) -> str: + self.state = self.home.copy() + self.actions.clear() + return "move x to 0.5, then close the gripper" + + def step(self, action: NDArray[Any]) -> None: + target = np.asarray(action[0], dtype=np.float64) + self.actions.append(target.copy()) + self.state = self.follow(self, target) + self.success = self.done is not None and self.done(self.state) + + def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: + state = self.state if self.reads is None else self.reads(self.state) + data = { + "observation/image": np.zeros((1, self.side, self.side, 3), dtype=np.uint8), + "observation/state": state[None].astype(np.float32), + } + return data, np.array([self.success]) + + +def snap(arm: Arm, target: NDArray[np.float64]) -> NDArray[np.float64]: + return target + + +def displace(arm: Arm, target: NDArray[np.float64]) -> NDArray[np.float64]: + return arm.state + target + + +def lag(share: float) -> Callable[[Arm, NDArray[np.float64]], NDArray[np.float64]]: + """Closes *share* of the gap to the command each tick; 0 never moves.""" + return lambda arm, target: arm.state + share * (target - arm.state) + + +def jitter(arm: Arm, target: NDArray[np.float64]) -> NDArray[np.float64]: + """Never moves; its sensor reads a little high then low on alternate ticks.""" + return arm.home + 2e-5 * (-1) ** len(arm.actions) + + +def offset(by: list[float]) -> Callable[[Arm, NDArray[np.float64]], NDArray[np.float64]]: + """Settles a fixed offset, in action columns, away from every command.""" + return lambda arm, target: target + np.asarray(by) + + +def delayed(ticks: int) -> Callable[[Arm, NDArray[np.float64]], NDArray[np.float64]]: + """Follows each command *ticks* ticks late.""" + return lambda arm, target: arm.actions[-1 - ticks] if len(arm.actions) > ticks else arm.home + + +def fingers(arm: Arm, target: NDArray[np.float64]) -> NDArray[np.float64]: + """Two fingers close 5 cm per tick; the left one jams on an object at 0.3.""" + close = np.minimum(arm.state[1:] + 0.05, target[1:]) + close[0] = min(close[0], 0.3) + return np.concatenate([arm.state[:1], close]) + + +def reached(state: NDArray[np.float64]) -> bool: + """The default task: hold ``x >= 0.5`` with the gripper closed.""" + return bool(state[0] >= 0.5 and state[1] >= 0.9) + + +class Sparse(dict[str, Any]): + """The dimensions a scripted call sets, by dotted name; the model nulls the rest.""" + + +def nulled(schema: dict[str, Any], values: Sparse) -> dict[str, Any]: + """``values`` laid out as the schema's ``targets``: every dimension listed, unset ones null.""" + targets: dict[str, Any] = {} + for key, prop in schema["properties"].items(): + if "anyOf" in prop: + leaves = {leaf: values.get(f"{key}.{leaf}") for leaf in prop["anyOf"][0]["properties"]} + targets[key] = leaves if any(v is not None for v in leaves.values()) else None + else: + targets[key] = values.get(key) + return targets + + +class ScriptedLLM(Agent): + """Stands in for a tool-calling LLM: plays fixed MCP calls, keeps their results.""" + + def __init__(self, *calls: tuple[str, dict[str, Any]]) -> None: + super().__init__() + self.calls = calls + self.tools: set[str] = set() + self.schemas: dict[str, dict[str, Any]] = {} + self.descriptions: dict[str, str] = {} + self.results: list[MCPToolResult] = [] + + async def __call__(self, run: Run) -> None: + client = cast("MCPClient", await run.client.open("control")) + listed = await client.list_tools() + self.tools = {tool.name for tool in listed} + self.schemas = {tool.name: tool.inputSchema for tool in listed} + self.descriptions = {tool.name: tool.description or "" for tool in listed} + for name, arguments in self.calls: + properties = self.schemas[name]["properties"] + if isinstance(arguments.get("targets"), Sparse): + arguments = { + **arguments, + "targets": nulled(properties["targets"], arguments["targets"]), + } + if "note" not in properties: + arguments = {k: v for k, v in arguments.items() if k != "note"} + self.results.append(await client.call_tool(name, arguments)) + run.trace.content = "done" + + def replies(self) -> list[str]: + """Each result's text.""" + return [ + "\n".join(block.text for block in result.content if block.type == "text") + for result in self.results + ] + + +def move(tool: str, **values: float) -> tuple[str, dict[str, Any]]: + return tool, {"targets": Sparse(values), "note": "move toward the goal"} + + +def look(duration: float | None = None) -> tuple[str, dict[str, Any]]: + return "wait", {"duration": duration, "note": "look"} + + +@asynccontextmanager +async def arm_env(sim: RobotBridge, control: DirectControl) -> AsyncIterator[Environment]: + """*sim* with direct control attached, as the ``arm`` env whose ``reach`` the sim grades.""" + async with sim_env(sim, "arm", control=control) as (env, endpoint): + + @env.template() + async def reach() -> AsyncGenerator[Any, Any]: + ep = await endpoint.reset() + yield {"prompt": ep["prompt"]} + yield await endpoint.result() + + yield env + + +# ─── policies ────────────────────────────────────────────────────────── + +#: Ticks a stub sim runs before terminating. +EPISODE_TICKS = 3 + +#: A minimal contract, exactly as documented: one camera, a state vector, an action. +STUB_CONTRACT = { + "control_rate": 5, + "features": { + "observation/image": {"role": "observation", "type": "rgb"}, + "observation/state": {"role": "observation", "names": ["slot", "tick"]}, + "action": {"role": "action", "names": ["a0", "a1"]}, + }, +} + + +class StubSim(RobotBridge): + """Cameras and a 2-D state, terminating after ``EPISODE_TICKS`` ticks. + + Slot ``i``'s state carries ``i + 1``, and its grade is the sum of the actions + that slot received, so with an echoing policy the grade names the slot the + connection actually drove. + """ + + def __init__( + self, num_envs: int = 1, *, contract: dict[str, Any] = STUB_CONTRACT, side: int = 16 + ) -> None: + super().__init__() + self.num_envs = num_envs + self.contract = contract + self.side = side + self.tick = 0 + self.executed = np.zeros(num_envs) + + def reset(self, **task_args: Any) -> str: + self.tick = 0 + self.executed[:] = 0.0 + return f"stub task: {task_args.get('goal', 'none')}" + + def step(self, action: NDArray[Any]) -> None: + self.tick += 1 + self.executed += np.asarray(action)[:, 0] # one row per slot + + def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: + slots = np.arange(1, self.num_envs + 1, dtype=np.float32) + cameras = [ + name + for name, feature in self.contract["features"].items() + if feature.get("type") == "rgb" + ] + data: dict[str, NDArray[Any]] = { + camera: np.full((self.num_envs, self.side, self.side, 3), self.tick, dtype=np.uint8) + for camera in cameras + } + data["observation/state"] = np.stack( + [slots, np.full(self.num_envs, self.tick, dtype=np.float32)], axis=1 + ) + return data, np.full(self.num_envs, self.tick >= EPISODE_TICKS) + + def result_slots(self) -> list[dict[str, Any]]: + # Per-slot grades, never one batch-wide scalar: the token has to resolve. + return [ + {"score": float(echoed), "success": bool(echoed), "total_reward": float(echoed)} + for echoed in self.executed + ] + + +@asynccontextmanager +async def stub_episode(sim: StubSim, name: str) -> AsyncIterator[Provider]: + """*sim* as env *name*, whose ``episode`` the sim grades, placed in this process.""" + async with sim_env(sim, name) as (env, endpoint): + + @env.template() + async def episode(goal: str = "lift") -> AsyncGenerator[Any, Any]: + ep = await endpoint.reset(goal=goal) + yield {"prompt": ep["prompt"]} # one slot: the token is optional + yield await endpoint.result() + + yield LocalRuntime(env) + + +# ─── spawned sims and gym envs ───────────────────────────────────────── + + +class Reach(RobotBridge): + """A bridge with a constructor parameter; its contract is set after construction.""" + + def __init__(self, *, use_delta: bool = False) -> None: + super().__init__() + self.use_delta = use_delta + + def reset(self, **task_args: Any) -> str: + return f"reach with use_delta={self.use_delta}" + + def step(self, action: NDArray[Any]) -> None: + del action + + def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: + return {"x": np.zeros((self.num_envs, 1), dtype=np.float32)}, np.zeros(self.num_envs, bool) + + +class PlainEnv: + """A single gym env with a ``Box(2, 2)`` action space.""" + + action_space = Box(-1.0, 1.0, shape=(2, 2), dtype=np.float32) + + def __init__(self) -> None: + self.received: tuple[int, ...] = () + + def reset(self, seed: int | None = None, options: dict[str, Any] | None = None) -> Any: + return self.observation(), {} + + def step(self, action: NDArray[Any]) -> tuple[Any, ...]: + self.received = np.shape(action) + return self.observation(), 0.5, False, False, {} + + def observation(self) -> dict[str, NDArray[Any]]: + return { + "received": np.array(self.received or (0,), dtype=np.float32), + "camera": np.zeros((4, 4, 3), dtype=np.uint8), + } + + def close(self) -> None: + pass + + +class VectorEnv: + """``num_envs`` envs stepped together, exposing the per-env ``single_action_space``.""" + + single_action_space = Box(-1.0, 1.0, shape=(2,), dtype=np.float32) + + def __init__(self, num_envs: int) -> None: + self.num_envs = num_envs + self.action_space = Box(-1.0, 1.0, shape=(num_envs, 2), dtype=np.float32) + self.received: tuple[int, ...] = () + + def reset(self, seed: int | None = None, options: dict[str, Any] | None = None) -> Any: + return self.observation(), {} + + def step(self, action: NDArray[Any]) -> tuple[Any, ...]: + self.received = np.shape(action) + rewards = np.full(self.num_envs, 0.25) + done = np.zeros(self.num_envs, dtype=bool) + return self.observation(), rewards, done, done, {} + + def observation(self) -> dict[str, NDArray[Any]]: + received = np.array(self.received or (0, 0), dtype=np.float32) + return {"received": np.tile(received, (self.num_envs, 1))} + + def close(self) -> None: + pass + + +def plain_env() -> PlainEnv: + return PlainEnv() + + +def vector_env(**build: Any) -> VectorEnv: + """A factory taking its build args as ``**kwargs``, like many registry wrappers.""" + return VectorEnv(int(build["num_envs"])) diff --git a/tests/robot/test_dataset_recording.py b/tests/robot/test_dataset_recording.py new file mode 100644 index 000000000..463404bd6 --- /dev/null +++ b/tests/robot/test_dataset_recording.py @@ -0,0 +1,242 @@ +"""Dataset recording: ``RobotAgent(save=True)`` writes each rollout as a LeRobot episode. + +The ``lerobot`` lane records with the real ``lerobot`` package and reads the +datasets back with it. Without the package, recording warns and the rollout +still runs. +""" + +from __future__ import annotations + +import asyncio +import importlib +import importlib.util +import logging +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest + +from hud.agents.robot import DatasetWriter, Model, RobotAgent +from hud.eval import Task, rollout + +from .sims import EPISODE_TICKS, StubSim, stub_episode + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + from hud.eval import Run + from tests.support import HudEnv + +#: lerobot's video encoder hangs on frames under 32 px. +RECORD_SIDE = 64 + + +def recording_contract( + *, rate: int = 5, robot: str = "arm", cameras: tuple[str, ...] = ("observation/image",) +) -> dict[str, Any]: + features: dict[str, Any] = { + camera: {"role": "observation", "type": "rgb"} for camera in cameras + } + features["observation/state"] = {"role": "observation", "names": ["slot", "tick"]} + features["action"] = {"role": "action", "names": ["a0", "a1"]} + return {"control_rate": rate, "robot_type": robot, "features": features} + + +class TickModel(Model): + """Acts ``[tick, 1]``; raises on the tick named by ``fail_at``.""" + + def __init__(self, fail_at: int | None) -> None: + self.fail_at = fail_at + + def infer(self, batch: Any) -> Any: + state = next(value for value in batch["data"].values() if np.asarray(value).ndim == 1) + tick = float(np.asarray(state).reshape(-1)[1]) + if tick == self.fail_at: + raise RuntimeError("policy crashed") + return np.array([[[tick, 1.0]]], dtype=np.float32) + + +class Recorder(RobotAgent): + save = True + max_steps = EPISODE_TICKS + 2 + log_every = 0 + + def __init__(self, *, fail_at: int | None = None) -> None: + super().__init__() + self.model = TickModel(fail_at) + self.adapter = None + + +async def record(name: str, contract: dict[str, Any], *, fail_at: int | None = None) -> Run: + async with stub_episode(StubSim(contract=contract, side=RECORD_SIDE), name) as runtime: + return await rollout( + Task(env=name, id="episode"), Recorder(fail_at=fail_at), runtime=runtime + ) + + +def datasets(record_dir: Path) -> list[Any]: + """Every dataset under ``record_dir`` as ``lerobot`` reads it back, oldest first.""" + lerobot_dataset: Any = importlib.import_module("lerobot.datasets.lerobot_dataset") + roots = sorted(path for path in record_dir.glob("*") if path.is_dir()) + return [lerobot_dataset.LeRobotDataset(f"hud/{root.name}", root=root) for root in roots] + + +def frames(dataset: Any, episode: int) -> list[dict[str, Any]]: + """One episode's frames as state, action and task.""" + return [ + { + "state": frame["observation.state"].tolist(), + "action": frame["action"].tolist(), + "task": frame["task"], + } + for frame in (dataset[index] for index in range(dataset.num_frames)) + if int(frame["episode_index"]) == episode + ] + + +@pytest.fixture +def recording(hud_env: HudEnv, tmp_path: Path) -> Iterator[Path]: + """``RECORD_DIR`` for this test; datasets still open are finalized afterwards.""" + record_dir = tmp_path / "records" + hud_env.set(RECORD_DIR=str(record_dir)) + yield record_dir + DatasetWriter.finalize() + + +@pytest.mark.parametrize( + ("rollouts", "fail_at", "status", "found", "episode_actions", "features"), + [ + pytest.param( + {"arm": recording_contract()}, + None, + "completed", + [(5, "arm", 1)], + [[float(tick), 1.0] for tick in range(EPISODE_TICKS)], + {"observation.images.observation_image"}, + id="one-rollout-is-one-episode", + ), + pytest.param( + {"left": recording_contract(), "right": recording_contract()}, + None, + "completed", + [(5, "arm", 2)], + [[float(tick), 1.0] for tick in range(EPISODE_TICKS)], + set(), + id="concurrent-rollouts-of-one-contract-share-a-dataset", + ), + pytest.param( + { + "five": recording_contract(rate=5), + "ten": recording_contract(rate=10), + "other": recording_contract(robot="other"), + }, + None, + "completed", + [(5, "arm", 1), (5, "other", 1), (10, "arm", 1)], + None, + set(), + id="a-different-rate-or-robot-gets-its-own-dataset", + ), + pytest.param( + {"arm": recording_contract()}, + 2, + "error", + [(5, "arm", 1)], + [[0.0, 1.0], [1.0, 1.0]], + set(), + id="a-failed-rollout-keeps-the-frames-it-recorded", + ), + pytest.param( + {"stereo": recording_contract(cameras=("left/image", "right/image"))}, + None, + "completed", + [(5, "arm", 1)], + None, + {"observation.images.left_image", "observation.images.right_image"}, + id="nested-camera-names-stay-distinct-features", + ), + pytest.param( + {"clash": recording_contract(cameras=("cam/rgb", "cam_rgb"))}, + None, + "error", + [], + None, + set(), + id="features-that-flatten-to-one-key-fail-the-rollout-before-recording", + ), + ], +) +@pytest.mark.e2e +@pytest.mark.lerobot +async def test_recorded_rollouts_land_in_one_dataset_per_rate_and_robot( + recording: Path, + rollouts: dict[str, dict[str, Any]], + fail_at: int | None, + status: str, + found: list[tuple[int, str, int]], + episode_actions: list[list[float]] | None, + features: set[str], +) -> None: + runs = await asyncio.gather( + *(record(name, contract, fail_at=fail_at) for name, contract in rollouts.items()) + ) + DatasetWriter.finalize() + + recorded = datasets(recording) + assert {run.trace.status for run in runs} == {status} + assert sorted((d.meta.fps, d.meta.robot_type, d.meta.total_episodes) for d in recorded) == found + for dataset in recorded: + assert features <= set(dataset.meta.features) + if episode_actions is not None: + for episode in range(dataset.meta.total_episodes): + assert [frame["action"] for frame in frames(dataset, episode)] == episode_actions + + +@pytest.mark.e2e +@pytest.mark.lerobot +async def test_a_recorded_episode_holds_the_observed_state_executed_action_and_task( + recording: Path, +) -> None: + run = await record("arm", recording_contract()) + DatasetWriter.finalize() + + (dataset,) = datasets(recording) + assert run.reward == sum(range(EPISODE_TICKS)) # the sim saw every action + assert dataset.meta.features["observation.state"]["names"] == ["slot", "tick"] + image = dataset[0]["observation.images.observation_image"] + assert tuple(image.shape) == (3, RECORD_SIDE, RECORD_SIDE) + assert frames(dataset, 0) == [ + {"state": [1.0, float(tick)], "action": [float(tick), 1.0], "task": "stub task: lift"} + for tick in range(EPISODE_TICKS) + ] + + +@pytest.mark.e2e +@pytest.mark.lerobot +@pytest.mark.parametrize("repo", [None, "lab"], ids=["no-repo", "unreachable-hub"]) +async def test_a_dataset_stays_on_disk_whether_or_not_its_push_succeeds( + repo: str | None, recording: Path, hud_env: HudEnv, capsys: pytest.CaptureFixture[str] +) -> None: + hud_env.set(HF_REPO=repo, HF_ENDPOINT="http://127.0.0.1:9") + + await record("arm", recording_contract()) + DatasetWriter.finalize() + + (dataset,) = datasets(recording) + assert dataset.meta.total_episodes == 1 + assert ("HF push failed" in capsys.readouterr().out) is (repo is not None) + + +@pytest.mark.skipif(importlib.util.find_spec("lerobot") is not None, reason="lerobot installed") +async def test_without_lerobot_recording_warns_and_the_rollout_still_runs( + hud_env: HudEnv, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + hud_env.set(RECORD_DIR=str(tmp_path / "records")) + + with caplog.at_level(logging.WARNING, logger="hud.agents.robot.dataset"): + run = await record("arm", recording_contract()) + + assert run.reward == sum(range(EPISODE_TICKS)) + assert any("lerobot" in entry.message for entry in caplog.records) + assert not (tmp_path / "records").exists() diff --git a/tests/robot/test_direct_control.py b/tests/robot/test_direct_control.py new file mode 100644 index 000000000..5268b8d20 --- /dev/null +++ b/tests/robot/test_direct_control.py @@ -0,0 +1,676 @@ +"""Direct control: a tool-calling agent drives a robot sim through MCP motion tools. + +Each case is a whole rollout: an in-process bridge serving the ``openpi/0`` wire, +an env publishing it plus :class:`DirectControl`'s ``mcp`` capability, and a +scripted agent calling the tools through the tunneled manifest binding, graded +by the sim. Every case asserts what the model saw (tools, schemas, replies) and +what the sim was commanded. +""" + +from __future__ import annotations + +import copy +import math +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest + +from hud.agents.openai.tools.strict_schema import ensure_strict_json_schema +from hud.environment.robot import DirectControl +from hud.eval import LocalRuntime, Task, rollout + +from .sims import ( + Arm, + ScriptedLLM, + arm_env, + contract, + delayed, + displace, + fingers, + jitter, + lag, + look, + move, + offset, + reached, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from numpy.typing import NDArray + + from hud.eval.run import Run + +#: The default contract: an absolute ``[x, grip]`` action, x in [0, 1]. +REACH = contract("ee_abs", [0.0, -1.0], [1.0, 1.0]) + + +async def drive( + sim: Arm, control: DirectControl, *calls: tuple[str, dict[str, Any]] +) -> tuple[ScriptedLLM, Run]: + agent = ScriptedLLM(*calls) + async with arm_env(sim, control) as env: + run = await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) + return agent, run + + +async def test_move_to_interpolates_absolute_targets_until_the_sim_succeeds() -> None: + sim = Arm(REACH, done=reached) + + agent, run = await drive( + sim, + DirectControl(max_step={"grip": 2.0}), + move("move_to", x=0.5), + move("move_to", grip=1.0), + ) + + assert agent.tools == {"move_to", "wait"} # the ee_abs contract picked the motion tool + schema = agent.schemas["move_to"] + assert set(schema["required"]) == {"targets", "note"} + # Every dimension is listed and nullable, so strict mode keeps the schema as it is. + assert schema["properties"]["targets"]["required"] == ["x", "grip"] + assert schema["properties"]["targets"]["properties"]["x"]["type"] == ["number", "null"] + assert schema["properties"]["targets"]["properties"]["grip"]["description"] == "[-1, 1]" + assert ensure_strict_json_schema(copy.deepcopy(schema)) == schema + assert run.reward == 1.0 # graded by the sim, not the tools + # 0.1 of x's range per second at 10 Hz: 0.01 per tick, grip held at its reference. + # The sim matches the command; the call holds until two 3-tick windows show it settled. + reach_x, held, close = np.array(sim.actions[:50]), sim.actions[50:56], sim.actions[56:] + np.testing.assert_allclose(reach_x[:, 0], np.linspace(0.01, 0.5, 50), rtol=1e-6) + np.testing.assert_allclose(reach_x[:, 1], 0.0) + np.testing.assert_allclose(held, [[0.5, 0.0]] * 6) + # max_step lets the gripper switch in one tick while x holds its commanded target; + # the sim succeeds on that tick, which ends the episode. + np.testing.assert_allclose(close, [[0.5, 1.0]]) + reached_reply, closed_reply = agent.replies() + assert [block.type for block in agent.results[0].content] == ["text", "text", "image"] + # The trace viewer parses this opening line to map tool calls to ticks. + assert reached_reply.startswith("Played 56 steps (5.6 s).") + assert "observation/state: x=0.5000, grip=0.0000" in reached_reply + assert "pose: x=0.5000, grip=0.0000" in reached_reply + assert "off: x=0.0000, grip=0.0000" in reached_reply + assert "At the target. Remaining offsets: 0 mm on x." in reached_reply + assert "The episode has ended" in closed_reply + # Tolerance only picks the verdict; nothing the model sees mentions it. + seen = " ".join([str(agent.schemas), *agent.descriptions.values(), reached_reply]) + for word in ("toleran", "within", "settle"): + assert word not in seen.lower() + + +@pytest.mark.parametrize(("state_type", "tool"), [(None, "move_by"), ("BASE_DEL_POSE", "walk_by")]) +async def test_a_displacement_kind_plays_in_steps_within_the_per_step_box( + state_type: str | None, tool: str +) -> None: + sim = Arm(contract("ee_del", [-0.1, -1.0], [0.1, 1.0]), follow=displace, done=reached) + sim.contract["features"]["action"]["state_type"] = state_type # wins over the coarse type + + agent, _ = await drive(sim, DirectControl(), move(tool, x=0.35)) + + assert agent.tools == {tool, "wait"} + assert "action's own components" in agent.descriptions[tool] + # Four in-box steps; a displacement returns once played. + np.testing.assert_allclose(sim.actions, [[0.0875, 0.0]] * 4) + (reply,) = agent.replies() + assert "observation/state: x=0.3500" in reply + assert "At the target" not in reply + + +async def test_wait_holds_the_commanded_pose_and_a_null_duration_observes_for_one_tick() -> None: + sim = Arm(REACH, done=reached) + + agent, _ = await drive(sim, DirectControl(), move("move_to", x=0.2), look(), look(1.5)) + + _, looked, waited = agent.replies() + np.testing.assert_allclose(sim.actions[26:], [[0.2, 0.0]] * 16) # held at the setpoint + assert looked.startswith("Played 1 steps (0.1 s).") + assert waited.startswith("Played 15 steps (1.5 s).") + assert "commanded: x=0.2000, grip=0.0000" in looked + assert "At the target" not in looked + + +@pytest.mark.parametrize( + ("call", "error"), + [ + (("move_to", {"targets": {"x": 0.1, "z": 0.1}, "note": "go"}), "'z' was unexpected"), + (move("move_to", x=1.5), "x=1.5 is outside [0, 1]"), + (("move_to", {"targets": {"x": None}, "note": "go"}), "set at least one"), + (("move_to", {"targets": {"x": 0.1}, "note": " "}), "note must say"), + (("wait", {"duration": 101, "note": "sleep"}), "duration must be within [0, 100]"), + ], +) +async def test_an_invalid_move_is_a_correctable_error_that_leaves_the_sim_still( + call: tuple[str, dict[str, Any]], error: str +) -> None: + sim = Arm(REACH, done=reached) + + agent, _ = await drive(sim, DirectControl(dims=["x"], speed=0.05), call) + + (result,) = agent.results + assert result.isError + assert error in agent.replies()[0] + assert sim.actions == [] + + +async def test_a_contract_without_a_motion_type_is_refused_at_start() -> None: + sim = Arm(contract("joint_vel", [0.0, -1.0], [1.0, 1.0])) + + with pytest.raises(ValueError, match="direct control needs an action state_type"): + async with arm_env(sim, DirectControl()) as env: + await env.start() + + +# ─── the tool surface ────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("action_type", "names", "targets"), + [ + pytest.param( + "ee_abs", + [f"target_eef_pos.{a}" for a in "xyz"] + [f"target_eef_axis_angle.r{a}" for a in "xyz"], + ["x", "y", "z", "roll", "pitch", "yaw"], + id="libero", + ), + pytest.param( + "joint_pos", ["panda_joint1", "panda_joint2"], ["joint1", "joint2"], id="panda" + ), + pytest.param( + "ee_abs", + ["left_x", "right_x", "left_gripper"], + ["left_arm", "right_arm"], + id="bimanual", + ), + pytest.param( + "ee_abs", ["left_x", "left_pitch_deg"], ["left_x", "left_pitch_deg"], id="nonstandard" + ), + ], +) +async def test_tools_use_standard_dimension_names_when_every_native_name_has_one( + action_type: str, names: list[str], targets: list[str] +) -> None: + sim = Arm(contract(action_type, [-1.0] * len(names), [1.0] * len(names), names)) + + agent, _ = await drive(sim, DirectControl()) + + (schema,) = (s for tool, s in agent.schemas.items() if tool != "wait") + assert list(schema["properties"]["targets"]["properties"]) == targets + + +async def test_dimensions_with_a_shared_prefix_nest_as_a_group_the_model_can_null_whole() -> None: + names = ["left_arm.x", "left_arm.y", "right_arm.x", "grip"] + sim = Arm(contract("ee_abs", [-1.0] * 4, [1.0] * 4, names)) + + agent, _ = await drive(sim, DirectControl(), move("move_to", **{"left_arm.x": 0.1})) + + schema = agent.schemas["move_to"] + targets = schema["properties"]["targets"] + assert targets["required"] == ["grip", "left_arm", "right_arm"] + group = targets["properties"]["left_arm"]["anyOf"] + assert list(group[0]["properties"]) == ["x", "y"] + assert group[1] == {"type": "null"} + assert ensure_strict_json_schema(copy.deepcopy(schema)) == schema + np.testing.assert_allclose(sim.actions[-1], [0.1, 0.0, 0.0, 0.0], atol=1e-6) + + +@pytest.mark.parametrize( + ("options", "calls", "tools", "annotated", "described", "moved"), + [ + pytest.param( + {"use_annotation": False}, + [("move_to", {"targets": {"x": 0.1, "grip": None}}), ("wait", {"duration": None})], + {"move_to", "wait"}, + False, + "", + True, + id="without-annotation-no-call-takes-a-note", + ), + pytest.param( + {"use_wait": False, "time_limit": 10.0}, + [move("move_to")], + {"move_to"}, + True, + "Time passes only while the robot moves;", + False, + id="without-wait-only-the-motion-tool-is-served", + ), + ], +) +async def test_options_shape_the_served_tools( + options: dict[str, Any], + calls: list[tuple[str, dict[str, Any]]], + tools: set[str], + annotated: bool, + described: str, + moved: bool, +) -> None: + sim = Arm(REACH, done=reached) + + agent, _ = await drive(sim, DirectControl(**options), *calls) + + assert agent.tools == tools + for schema in agent.schemas.values(): + assert ("note" in schema["properties"]) == annotated + assert ("note" in schema["required"]) == annotated + assert described in agent.descriptions["move_to"] + assert bool(sim.actions) == moved + if "wait" not in tools: + assert not any("wait" in reply for reply in agent.replies()) + + +# ─── settle verdicts: how each kind of arm ends an absolute move ─────── + + +def quantity_rows() -> list[Any]: + """Each quantity's default tolerance moves the fingertips 1 mm: directly, over a 0.15 m + gripper, over a 2.5 m lever of joints. An arm settling just inside it is at the target; + just outside, it stopped short.""" + euler = ["ee.x", "ee_euler.x", "ee_euler.y", "ee_euler.z"] + quantities = [ + ("position", "move_to", "ee_abs", ["x", "grip"], {"x": 0.5}, 0, 1e-3, "x"), + ("orientation", "move_to", "ee_abs", euler, {"roll": 0.3}, 1, 1e-3 / 0.15, "orientation"), + ( + "joint", + "move_joints", + "joint_pos", + ["joint1", "grip"], + {"joint1": 0.3}, + 0, + 4e-4, + "joint1", + ), + ] + rows = [] + for quantity, tool, action_type, names, values, column, tolerance, label in quantities: + for share in (0.9, 1.1): + by = [0.0] * len(names) + by[column] = share * tolerance + inside = share < 1 + rows.append( + pytest.param( + lambda action_type=action_type, names=names, by=by: Arm( + contract(action_type, [-1.0] * len(names), [1.0] * len(names), names), + follow=offset(by), + ), + {}, + move(tool, **values), + ( + "At the target. " if inside else "Stopped short of the target; ", + f" on {label}.", + ), + ("Stopped short" if inside else "At the target",), + range(1, 301), + None, + id=f"{quantity}-default-{'inside' if inside else 'outside'}", + ) + ) + return rows + + +STOPPED_SHORT = "Stopped short of the target; the arm is at rest. Remaining offsets: " + + +@pytest.mark.parametrize( + ("arm", "options", "call", "says", "never", "ticks", "pose"), + [ + pytest.param( + lambda: Arm(REACH, done=reached), + {"speed": 0.05, "max_step": {"grip": 2.0}}, + move("move_to", x=0.6), + ("At the target.",), + (), + range(120, 301), # 0.6 / 0.05 s = 12 s at 10 Hz, under the 30 s safety timeout + pytest.approx([0.6, 0.0]), + id="a-move-longer-than-ten-seconds-plays-until-the-arm-arrives", + ), + pytest.param( + lambda: Arm(REACH, done=reached), + {"dims": ["x"], "speed": 0.05, "timeout": 0.3}, + move("move_to", x=1.0), + ( + "Ran out of time before settling; the arm was still moving. " + "Remaining offsets: 985 mm on x.", + "pose:", + "commanded:", + ), + (), + range(3, 4), # 0.3 s at 10 Hz + pytest.approx([0.015, 0.0]), # the prefix of the move, not a jump to the goal + id="the-safety-timeout-returns-the-pose", + ), + pytest.param( + lambda: Arm(REACH, done=reached), + {"max_step": {"grip": 2.0}}, + move("move_to", x=1.0, grip=1.0), + ("The episode ended before the arm settled.", "The episode has ended; stop calling"), + ("Ran out of time",), + range(1, 100), # the sim succeeded before the 100-tick move finished + None, + id="a-sim-that-terminates-mid-move-is-not-out-of-time", + ), + pytest.param( + lambda: Arm(contract("ee_abs", [0.0, -1.0], [1.0, 1.0], rate=1)), + {}, + move("move_to", x=0.05), + ("At the target.",), + ("Ran out of time",), + range(1, 31), + pytest.approx([0.05, 0.0]), + id="an-arm-at-the-target-is-at-rest-at-a-low-control-rate", + ), + pytest.param( + lambda: Arm(REACH, follow=lag(0.0)), + {}, + move("move_to", x=0.5), + ("off: x=-0.5000", STOPPED_SHORT + "500 mm on x."), + (), + range(80, 81), # the 50-tick move, then the whole 3 s settle cap + pytest.approx([0.0, 0.0]), + id="a-still-arm-gives-up-at-the-settle-cap", + ), + pytest.param( + lambda: Arm(REACH, follow=jitter), + {}, + move("move_to", x=0.5), + ("off: x=-0.5000", STOPPED_SHORT + "500 mm on x."), + (), + range(80, 81), + pytest.approx([0.0, 0.0], abs=1e-4), + id="a-jittering-still-arm-gives-up-at-the-settle-cap", + ), + pytest.param( + lambda: Arm(REACH, follow=lag(0.25)), + {"max_step": {"grip": 2.0}}, + move("move_to", x=0.5), + ("pose:", "off:", "At the target. Remaining offsets: "), + (), + range(51, 301), # interpolation, then a hold while the arm catches up + pytest.approx([0.5, 0.0], abs=0.01), + id="a-lagging-arm-keeps-stepping-until-it-reaches-the-target", + ), + pytest.param( + lambda: Arm(REACH, follow=offset([-0.012, 0.0])), + {}, + move("move_to", x=0.5), + (STOPPED_SHORT + "12 mm on x.",), + (), + range(50 + 21, 50 + 22), # held for the grace period before stopping + pytest.approx([0.488, 0.0]), + id="an-arm-at-rest-outside-its-tolerance-is-held-for-the-grace-period", + ), + pytest.param( + lambda: Arm(REACH, follow=offset([-0.012, 0.0])), + {"tolerance": {"position": 0.02}}, + move("move_to", x=0.5), + ("At the target. Remaining offsets: 12 mm on x.",), + (), + range(56, 57), # tolerance only lengthens the wait, never shortens it + pytest.approx([0.488, 0.0]), + id="an-arm-at-rest-inside-a-custom-tolerance-is-at-the-target", + ), + pytest.param( + lambda: Arm(REACH, follow=delayed(10)), + {"tolerance": {"position": 0.05}}, + move("move_to", x=8e-4), + (), + (), + range(17, 18), + pytest.approx([8e-4, 0.0]), + id="a-move-waits-for-a-slow-arm-even-when-already-within-tolerance", + ), + pytest.param( + lambda: Arm( + contract("ee_abs", [0.0] * 3, [1.0] * 3, ["x", "grip_left", "grip_right"]), + follow=fingers, + ), + {"max_step": {"grip_left": 2.0, "grip_right": 2.0}}, + move("move_to", grip_left=1.0, grip_right=1.0), + ("fingers: grip_left=0.3000, grip_right=1.0000",), # the jam reads as no success + ("At the target", "Remaining offsets"), + range(20 + 3, 20 + 30), # the right finger closes for 20 ticks; the call outlasts it + pytest.approx([0.0, 0.3, 1.0]), + id="a-gripper-call-waits-for-every-finger", + ), + pytest.param( + lambda: Arm(REACH, [0.0, -1.0], lag(0.25), reads=lambda s: s * [1.0, 0.0] - [0.0, 1.0]), + {"max_step": {"grip": 2.0}}, + move("move_to", x=0.5, grip=1.0), + ("At the target. Remaining offsets: ", "2 on grip"), # grip never decides the verdict + (), + range(1, 301), + None, + id="gripper-offsets-are-reported-when-the-reference-reads-the-hand", + ), + pytest.param( + lambda: Arm(REACH, [0.0, -1.0], lag(0.25), reads=lambda s: s * [1.0, 0.0] - [0.0, 1.0]), + {"max_step": {"grip": 2.0}, "reference": lambda data: data["observation/state"]}, + move("move_to", x=0.5, grip=1.0), + ("At the target. Remaining offsets: ",), + ("2 on grip",), + range(1, 301), + None, + id="gripper-offsets-are-left-out-under-a-custom-reference", + ), + *quantity_rows(), + ], +) +async def test_an_absolute_move_ends_with_the_verdict_its_arm_earns( + arm: Callable[[], Arm], + options: dict[str, Any], + call: tuple[str, dict[str, Any]], + says: tuple[str, ...], + never: tuple[str, ...], + ticks: range, + pose: Any, +) -> None: + sim = arm() + + agent, _ = await drive(sim, DirectControl(**options), call) + + (result,) = agent.results + (reply,) = agent.replies() + assert not result.isError + for text in says: + assert text in reply + for text in never: + assert text not in reply + assert len(sim.actions) in ticks + if pose is not None: + assert sim.state.tolist() == pose + + +# ─── rotations: roll / pitch / yaw onto whatever the contract stores ─── + + +def quaternion(order: str) -> list[str]: + parts = ("qx", "qy", "qz", "qw") if order == "xyzw" else ("qw", "qx", "qy", "qz") + return [f"tcp_quat.{part}" for part in parts] + + +def unit(actions: NDArray[Any]) -> bool: + """Every commanded quaternion is unit length: slerped, not lerped by component.""" + return len(actions) > 2 and bool(np.allclose(np.linalg.norm(actions, axis=1), 1.0, atol=1e-5)) + + +def evenly_turned(actions: NDArray[Any]) -> bool: + """2pi/3 at a tenth of a turn per second, 10 Hz: equal angular steps until the hold.""" + ticks = math.ceil((2 * np.pi / 3) / (0.1 * 2 * np.pi / 10)) + gaps = np.diff(np.linalg.norm(actions[:ticks], axis=1)) + return bool(gaps.min() > 0 and np.allclose(gaps, gaps[0], rtol=0.05)) + + +WRAPPED = 3.2 - 2 * math.pi +#: Intrinsic XYZ of roll = yaw = pi/2 is the quaternion (0.5, 0.5, 0.5, 0.5), angle 2pi/3. +AXIS_ANGLE = (2.0 * np.pi / 3.0) / np.sqrt(3.0) + + +@pytest.mark.parametrize( + ("names", "low", "high", "home", "calls", "final", "path", "says"), + [ + pytest.param( + quaternion("xyzw"), + [-1.0] * 4, + [1.0] * 4, + [0.0, 0.0, 0.0, 1.0], + [move("move_to", yaw=math.pi / 2)], + [0.0, 0.0, math.sin(math.pi / 4), math.cos(math.pi / 4)], + unit, + (), + id="a-yaw-slerps-an-xyzw-quaternion", + ), + pytest.param( + quaternion("wxyz"), + [-1.0] * 4, + [1.0] * 4, + [1.0, 0.0, 0.0, 0.0], + [move("move_to", roll=math.pi / 2)], + [math.cos(math.pi / 4), math.sin(math.pi / 4), 0.0, 0.0], + unit, + (), + id="a-roll-writes-a-wxyz-quaternion", + ), + pytest.param( + ["x", "tcp_quat.qx", "tcp_quat.qy", "tcp_quat.qz", "tcp_quat.qw"], + [0.0, -1.0, -1.0, -1.0, -1.0], + [1.0] * 5, + [0.0, 0.0, 0.2, 0.0, 0.9], # not a unit quaternion + [move("move_to", x=0.2)], + [0.2, 0.0, 0.2, 0.0, 0.9], + lambda actions: len(actions) > 1 and (actions[:, 1:] == actions[0, 1:]).all(), + (), + id="an-unnamed-quaternion-is-copied-unchanged", + ), + pytest.param( + [f"target_eef_axis_angle.r{a}" for a in "xyz"], + [-np.pi] * 3, + [np.pi] * 3, + [0.0, 0.0, 0.0], + [move("move_to", roll=math.pi / 2, yaw=math.pi / 2)], + [AXIS_ANGLE] * 3, + evenly_turned, + (), + id="euler-commands-slerp-onto-axis-angle", + ), + pytest.param( + [ + "ee_pos.x", + "ee_pos.y", + "ee_pos.z", + "ee_euler.x", + "ee_euler.y", + "ee_euler.z", + "gripper", + ], + [-1.0, -1.0, 0.0, -np.pi, -np.pi, -np.pi, 0.0], + [1.0, 1.0, 2.0, np.pi, np.pi, np.pi, 0.04], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.04], + [move("move_to", z=1.2, yaw=0.4)], + [0.0, 0.0, 1.2, 0.0, 0.0, 0.4, 0.04], + None, + (), + id="an-euler-contract-is-addressed-as-roll-pitch-yaw", + ), + pytest.param( + ["ee.x", "ee_euler.x", "ee_euler.y", "ee_euler.z"], + [-1.0, -4.0, -4.0, -4.0], + [1.0, 4.0, 4.0, 4.0], + [0.0, 3.2, 0.0, 0.0], # a sim that reports roll unwrapped, just past pi + [look(), move("move_to", roll=3.2), move("move_to", roll=-math.pi)], + None, + None, + ( + f"ee_euler.x={WRAPPED:.4f}", + f"pose: x=0.0000, roll={WRAPPED:.4f}", + "At the target. Remaining offsets: ", + "pi and -pi are the same orientation", + ), + id="reported-angles-wrap-into-pi-and-a-command-past-pi-is-accepted", + ), + ], +) +async def test_an_orientation_command_reaches_the_contracts_rotation_encoding( + names: list[str], + low: list[float], + high: list[float], + home: list[float], + calls: list[tuple[str, dict[str, Any]]], + final: list[float] | None, + path: Callable[[NDArray[Any]], bool] | None, + says: tuple[str, ...], +) -> None: + sim = Arm(contract("ee_abs", low, high, names), home) + + agent, _ = await drive(sim, DirectControl(), *calls) + + assert not any(result.isError for result in agent.results) + actions = np.array(sim.actions) + if final is not None: + np.testing.assert_allclose(actions[-1], final, atol=1e-4) + if path is not None: + assert path(actions) + seen = "\n".join([agent.descriptions["move_to"], *agent.replies()]) + for text in says: + assert text in seen + + +# ─── the episode's time budget ───────────────────────────────────────── + + +@pytest.mark.parametrize( + ("limit", "calls", "openings", "says", "ticks", "x"), + [ + pytest.param( + 10.0, + [move("move_to", x=0.2), look(1.0)], + [ + "Played 26 steps (2.6 s). Time remaining: 7.4 s.", + "Played 10 steps (1.0 s). Time remaining: 6.4 s.", + ], + (), + 36, + 0.2, + id="every-reply-reports-the-time-left", + ), + pytest.param( + 2.0, + [move("move_to", x=0.5), move("move_to", x=0.1)], + [ + "Played 20 steps (2.0 s). Time remaining: 0.0 s.", + "Played 0 steps (0.0 s). Time remaining: 0.0 s.", + ], + ( + "The time limit is reached.", + "The episode has ended; stop calling tools.", + "The episode ended before the arm settled. Remaining offsets: 300 mm on x.", + ), + 20, # the move needed 50 ticks; 2 s at 10 Hz were left + 0.2, + id="a-move-crossing-the-limit-is-cut-there-and-ends-the-episode", + ), + ], +) +async def test_the_time_limit_is_spent_by_played_ticks_and_ends_the_episode( + limit: float, + calls: list[tuple[str, dict[str, Any]]], + openings: list[str], + says: tuple[str, ...], + ticks: int, + x: float, +) -> None: + sim = Arm(REACH, done=reached) + + agent, _ = await drive(sim, DirectControl(time_limit=limit), *calls) + + for description in agent.descriptions.values(): + assert f"You have {limit:g} seconds of simulated time." in description + replies = agent.replies() + assert [reply[: len(opening)] for reply, opening in zip(replies, openings, strict=True)] == ( + openings + ) + for text in says: + assert text in replies[0] + assert all("Ran out of time" not in reply for reply in replies) + assert ("The episode has ended" in replies[-1]) == bool(says) + assert len(sim.actions) == ticks + assert sim.state[0] == pytest.approx(x) diff --git a/tests/robot/test_policy_rollout.py b/tests/robot/test_policy_rollout.py new file mode 100644 index 000000000..1840580c3 --- /dev/null +++ b/tests/robot/test_policy_rollout.py @@ -0,0 +1,252 @@ +"""A policy drives a robot rollout: ``RobotAgent`` on the ``robot`` wire, graded by the sim. + +Every case is a real rollout: an env-side bridge serving the ``openpi/0`` +WebSocket and its control channel, an agent-side ``RobotAgent`` dialing it +through ``RobotClient``, graded by the rollout engine. The flows are the ones +``docs/v6/advanced/robots.mdx`` promises: + +- a single-env custom bridge, where the slot token is optional; +- ``env.gym`` behind a pre-written contract, the shape ``hud-evals/robot-template`` + ships (two cameras and an 8-D state, LIBERO-style); +- a vectorized sim, where the per-episode token pins each rollout to its slot. + +The policies are stubs; the plumbing is not. +""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest + +from hud.agents.robot import Adapter, Model, RobotAgent +from hud.eval import LocalRuntime, Shared, Task, Taskset, rollout + +from .sims import EPISODE_TICKS, StubSim, sim_env, stub_episode + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + from pathlib import Path + + from hud.eval import Provider + + +class EchoAdapter(Adapter): + """Wires from the contract alone, and records what it wired.""" + + def __init__(self) -> None: + super().__init__() + self.wired: list[dict[str, Any]] = [] + + def adapt_observation(self, obs: dict[str, Any], prompt: str) -> dict[str, Any]: + data = obs["data"] + batch = {"state": data[self.state_key], "image": data[self.image_keys[0]], "task": prompt} + self.wired.append(batch) + return batch + + +class EchoModel(Model): + """Echoes the state's slot id, in the action width its checkpoint was trained for.""" + + def __init__(self, action_dim: int) -> None: + self.action_dim = action_dim + + def infer(self, batch: Any) -> Any: + slot = float(np.asarray(batch["state"]).reshape(-1)[0]) + return np.full((1, 1, self.action_dim), slot, dtype=np.float32) # [N, T, A] + + +class EchoAgent(RobotAgent): + """The model/adapter pair; the base owns the loop.""" + + max_steps = EPISODE_TICKS + 2 # the env must end the episode, not this cap + + def __init__(self, adapter: EchoAdapter, *, action_dim: int = 2) -> None: + super().__init__() + self.model = EchoModel(action_dim) + self.adapter = adapter + + +class EarlyStopAgent(EchoAgent): + """Stops after one action through the public rollout hook.""" + + def should_stop(self, obs: dict[str, Any], *, step: int, max_steps: int) -> bool: + return step >= 1 or super().should_stop(obs, step=step, max_steps=max_steps) + + +#: ``hud-evals/robot-template``'s ``contract_lerobot.json``: two cameras, an 8-D +#: LIBERO state, a 7-D action. The agent wires the policy from this alone. +TEMPLATE_CONTRACT = { + "control_rate": 80, + "features": { + "observation/image": {"role": "observation", "type": "rgb"}, + "observation/wrist_image": {"role": "observation", "type": "rgb"}, + "observation/state": {"role": "observation", "names": [f"state_{i}" for i in range(8)]}, + "action": {"role": "action", "names": [f"act_{i}" for i in range(7)]}, + }, +} + +#: Stands in for the template's ``lerobot_sim.py``: LIBERO's shape without LIBERO. +TEMPLATE_SIM = '''\ +"""Module-level ``make_env`` for ``env.gym``; suite/id are build args.""" + +from types import SimpleNamespace + +import numpy as np + + +class _Libero: + """Two cameras + an 8-D state, 7-D action, env-reported success on the last tick.""" + + action_space = SimpleNamespace(shape=(7,), dtype=np.float32) + task_description = "pick up the black bowl" # becomes the episode prompt + + def reset(self, seed=None, options=None): + self.tick = 0 + return self._observation(), {} + + def step(self, action): + self.tick += 1 + done = self.tick >= 3 # inside the agent's max_steps + return self._observation(), 1.0, done, False, {"success": done} + + def close(self): + pass + + def _observation(self): + frame = np.zeros((32, 32, 3), dtype=np.uint8) + return { + "observation/image": frame, + "observation/wrist_image": frame, + "observation/state": np.zeros(8, dtype=np.float32), + } + + +def make_env(task_suite="libero_spatial", task_id=0): + return _Libero() +''' + +#: The template's ``environment/env.py``, unchanged but for the sim it imports. +TEMPLATE_ENV = '''\ +"""LIBERO as one declarative HUD env; the task is a template arg.""" + +from pathlib import Path + +from template_sim import make_env + +from hud import Environment + +env = Environment(name="libero") +sim = env.gym(make_env, contract=str(Path(__file__).parent / "contract.json")) + + +@env.template() +async def episode(task_suite: str = "libero_spatial", task_id: int = 0, seed: int = 0): + """One LIBERO episode; the sim provides the instruction as the prompt.""" + ep = await sim.reset(task_suite=task_suite, task_id=task_id, seed=seed) + yield {"prompt": ep["prompt"]} # single env: the slot token is optional + yield await sim.result() +''' + + +@asynccontextmanager +async def custom_bridge(tmp_path: Path) -> AsyncIterator[tuple[Provider, Task]]: + async with stub_episode(StubSim(), "stub-sim") as runtime: + yield runtime, Task(env="stub-sim", id="episode", args={"goal": "lift"}) + + +@asynccontextmanager +async def template_files(tmp_path: Path) -> AsyncIterator[tuple[Provider, Task]]: + (tmp_path / "template_sim.py").write_text(TEMPLATE_SIM, encoding="utf-8") + (tmp_path / "contract.json").write_text(json.dumps(TEMPLATE_CONTRACT), encoding="utf-8") + (tmp_path / "env.py").write_text(TEMPLATE_ENV, encoding="utf-8") + yield LocalRuntime(tmp_path / "env.py"), Task(env="libero", id="episode", args={"task_id": 0}) + + +@pytest.mark.parametrize( + ("placement", "agent", "reward", "prompt", "cameras", "state"), + [ + pytest.param( + custom_bridge, + lambda adapter: EchoAgent(adapter), + EPISODE_TICKS, # every tick's echoed action reached the sim, which ended it + "stub task: lift", # the sim's prompt, built from the task arg + ["observation/image"], + [1.0, 0.0], # slot 1, before any step + id="single-env-custom-bridge", + ), + pytest.param( + custom_bridge, + lambda adapter: EarlyStopAgent(adapter), + 1, # the agent's stop hook ended the rollout after one action + "stub task: lift", + ["observation/image"], + [1.0, 0.0], + id="the-agent-stop-hook-ends-it-before-the-env", + ), + pytest.param( + template_files, + lambda adapter: EchoAgent(adapter, action_dim=7), + 1.0, # the sim's own success outranks accumulated reward + "pick up the black bowl", # the sim's instruction, not the task args + ["observation/image", "observation/wrist_image"], # file order, state after them + [0.0] * 8, + id="gym-behind-a-prewritten-contract", + ), + ], +) +async def test_a_policy_rollout_is_wired_from_the_contract_and_graded_by_the_sim( + tmp_path: Path, + placement: Callable[[Path], Any], + agent: Callable[[EchoAdapter], EchoAgent], + reward: float, + prompt: str, + cameras: list[str], + state: list[float], +) -> None: + adapter = EchoAdapter() + + async with placement(tmp_path) as (runtime, task): + run = await rollout(task, agent(adapter), runtime=runtime) + + assert run.reward == reward + assert run.prompt == prompt + assert adapter.image_keys == cameras + assert [batch["task"] for batch in adapter.wired] == [prompt] * len(adapter.wired) + # A camera arrives HWC uint8, the state as named. + assert adapter.wired[0]["image"].ndim == 3 + assert adapter.wired[0]["image"].dtype == np.uint8 + assert adapter.wired[0]["state"].tolist() == state + + +async def test_the_slot_token_pins_each_rollout_to_its_own_slot() -> None: + sim = StubSim(num_envs=2) + # Both rollouts must hold a slot before either acts: a sequential pair would each + # get their own global reset (and slot 0), which is not the case under test. + claimed = asyncio.Barrier(2) + + async with sim_env(sim, "stub-sim") as (env, endpoint): + + @env.template() + async def episode(goal: str = "lift") -> AsyncGenerator[Any, Any]: + ep = await endpoint.reset(goal=goal) + await claimed.wait() + # Vectorized: the token is this episode's binding, and grades its slot alone. + yield {"prompt": ep["prompt"], "bindings": {"robot": {"token": ep["token"]}}} + yield await endpoint.result(token=ep["token"]) + + job = await Taskset("stub", [Task(env="stub-sim", id="episode")]).run( + EchoAgent(EchoAdapter()), + runtime=Shared(LocalRuntime(env), width=2), + group=2, + max_concurrent=2, + ) + + # Slot i echoes i + 1 per tick, so distinct grades mean neither rollout read or + # graded the other's slot: a dropped or crossed token grades them the same. + assert sorted(run.reward for run in job.runs) == [EPISODE_TICKS, 2 * EPISODE_TICKS] diff --git a/tests/robot/test_sim_serving.py b/tests/robot/test_sim_serving.py new file mode 100644 index 000000000..3b2b4e13b --- /dev/null +++ b/tests/robot/test_sim_serving.py @@ -0,0 +1,197 @@ +"""Serving a sim: what a sim process reports, and the shapes a gym env receives and returns. + +``RobotEndpoint(bridge)`` and ``env.gym(factory)`` both run the sim in a child +process; a ``GymBridge`` can also be served in this one. These scenarios drive +the sims in ``sims`` through the public surface: the endpoint's control calls, +the served env's ``robot`` binding, and ``RobotClient`` on the ``openpi/0`` +wire. A gym env reports in its observation the shape of the action it was +stepped with. +""" + +from __future__ import annotations + +import sys +from types import SimpleNamespace +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest +import websockets +from openpi_client import msgpack_numpy + +from hud.capabilities import Capability +from hud.capabilities.robot import RobotClient +from hud.environment import Environment +from hud.environment.robot import GymBridge, RobotEndpoint +from tests.fixtures.environments import served + +from .sims import Reach, plain_env, vector_env + +if TYPE_CHECKING: + from collections.abc import Callable + + from numpy.typing import NDArray + +CONTRACT = {"control_rate": 10, "features": {"action": {"role": "action", "names": ["a"]}}} + + +async def test_a_spawned_bridge_carries_its_declaration_into_the_child() -> None: + declared = Reach(use_delta=True) + declared.contract = CONTRACT + declared.num_envs = 2 + declared.metadata = {"backend": "test"} + endpoint = RobotEndpoint(declared) + + await endpoint.start() + try: + contract = await endpoint.contract() + (robot,) = await endpoint.capabilities() + first = await endpoint.reset() + second = await endpoint.reset() + async with websockets.connect(robot.url) as socket: + greeting = msgpack_numpy.unpackb(await socket.recv()) + finally: + await endpoint.stop() + + assert contract == CONTRACT + assert robot.params["contract"] == CONTRACT + assert first["prompt"] == "reach with use_delta=True" + assert [first["token"][:7], second["token"][:7]] == ["slot-0-", "slot-1-"] + assert greeting == {"backend": "test", "claim_required": True} + with pytest.raises(OSError): + await websockets.connect(robot.url, open_timeout=5) + + +async def test_a_sim_that_exits_before_announcing_its_port_fails_start() -> None: + endpoint = RobotEndpoint.spawn([sys.executable, "-c", "import sys; sys.exit(3)"]) + + with pytest.raises(RuntimeError, match="exited with code 3 before announcing its port"): + await endpoint.start() + + +def gym_env(factory: Callable[..., Any], **defaults: Any) -> Environment: + env = Environment("gym") + sim = env.gym(factory, contract=None, **defaults) + + @env.template() + async def episode(): + claimed = await sim.reset() + yield {"prompt": claimed["prompt"], "bindings": {"robot": {"token": claimed["token"]}}} + yield await sim.result(token=claimed["token"]) + + return env + + +@pytest.mark.parametrize( + ("factory", "defaults", "action", "received", "reward"), + [ + pytest.param(plain_env, {}, [0.1, 0.2, 0.3, 0.4], [2.0, 2.0], 0.5, id="box-2x2-action"), + pytest.param( + vector_env, {"num_envs": 2}, [0.1, 0.2], [2.0, 2.0], 0.25, id="kwargs-vector-env" + ), + ], +) +async def test_a_flat_action_reaches_the_gym_env_in_its_own_shape( + factory: Callable[..., Any], + defaults: dict[str, Any], + action: list[float], + received: list[float], + reward: float, +) -> None: + async with served(gym_env(factory, **defaults)) as client: + started = await client.start_task("episode") + agent = await RobotClient.connect( + client.binding("robot"), token=started["bindings"]["robot"]["token"] + ) + try: + await agent.get_observation() + await agent.send_action(action) + stepped = await agent.get_observation() + finally: + await agent.close() + graded = await client.grade({"answer": "done"}) + + assert stepped["data"]["received"].tolist() == received + assert float(stepped["reward"]) == reward + assert stepped["terminated"] is False + assert graded == {"score": reward, "success": False, "total_reward": reward} + + +async def test_a_plain_env_observation_is_sliced_to_one_slot_with_its_frames() -> None: + async with served(gym_env(plain_env)) as client: + started = await client.start_task("episode") + agent = await RobotClient.connect( + client.binding("robot"), token=started["bindings"]["robot"]["token"] + ) + try: + observation = await agent.get_observation() + finally: + await agent.close() + await client.cancel() + + shapes = {name: np.shape(value) for name, value in observation["data"].items()} + assert shapes == {"received": (1,), "camera": (4, 4, 3)} + assert float(observation["reward"]) == 0.0 + + +class VectorAutoReset: + """Two lockstep envs that reset inside ``step``; env 1 finishes on the first step. + + ``per_env`` reports the final observation as gymnasium vector envs do (one obs per env), + otherwise as Isaac Lab does (an obs-shaped batch). + """ + + num_envs = 2 + action_space = SimpleNamespace(shape=(2, 1), dtype=np.float32) + + def __init__(self, per_env: bool) -> None: + self.per_env = per_env + self.ticks = np.zeros(2) + + def observe(self) -> dict[str, NDArray[Any]]: + return {"state": self.ticks[:, None].astype(np.float32)} + + def reset(self, seed: int | None = None, options: Any = None) -> tuple[Any, dict[str, Any]]: + self.ticks[:] = 0 + return self.observe(), {} + + def step(self, action: NDArray[Any]) -> tuple[Any, ...]: + self.ticks += 1 + final = self.observe() + done = np.array([False, True]) + self.ticks[done] = 0 + if self.per_env: + final = np.array([None, {"state": final["state"][1]}], dtype=object) + return self.observe(), np.zeros(2), np.zeros(2, dtype=bool), done, {"final_obs": final} + + def close(self) -> None: + pass + + +@pytest.mark.parametrize("per_env", [False, True], ids=["batched-final-obs", "per-env-final-obs"]) +async def test_a_finished_env_is_observed_on_the_frame_it_ended_on(per_env: bool) -> None: + bridge = GymBridge(lambda: VectorAutoReset(per_env), contract=None) + await bridge.start() + control = await bridge.serve_control() + endpoint = RobotEndpoint.remote("127.0.0.1", control.sockets[0].getsockname()[1]) + await endpoint.start() + try: + robot = Capability.robot(url=bridge.url, contract=bridge.contract) + tokens = [(await endpoint.reset())["token"] for _ in range(2)] + agents = [await RobotClient.connect(robot, token=token) for token in tokens] + try: + for agent in agents: + await agent.get_observation() + await agent.send_action([0.0]) + observed = [await agent.get_observation() for agent in agents] + finally: + for agent in agents: + await agent.close() + finally: + await endpoint.stop() + control.close() + await bridge.stop() + + assert [observation["terminated"] for observation in observed] == [False, True] + # Env 1 was reset after its first tick; its observation is still that tick. + assert [observation["data"]["state"].tolist() for observation in observed] == [[1.0], [1.0]] diff --git a/tests/robot/test_slots.py b/tests/robot/test_slots.py new file mode 100644 index 000000000..068959907 --- /dev/null +++ b/tests/robot/test_slots.py @@ -0,0 +1,333 @@ +"""Sim slots: how sessions claim, step and free the lockstep slots of a robot sim. + +A bridge in this process serves its control channel and its ``openpi/0`` +WebSocket. Sessions claim and free slots the way a served env does: a client on +the env's control channel starts, grades, cancels or drops a task whose template +drives a ``RobotEndpoint``. Agents step slots the way a policy does +(``RobotClient``). Cases assert what the sim saw (its steps and ``result`` +calls) and what each peer got. +""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest + +from hud.capabilities import Capability +from hud.capabilities.robot import RobotClient +from hud.clients import HudProtocolError, connect +from hud.environment import Environment +from hud.environment.robot import RobotBridge, RobotEndpoint +from hud.eval import LocalRuntime, Task + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + from contextlib import AbstractAsyncContextManager + + from numpy.typing import NDArray + + from hud.eval import Runtime + + +class Sim(RobotBridge): + """``num_envs`` lockstep slots of a one-dimensional action. + + Records every batched step and every ``result`` call; the first ``failures`` + calls to ``result`` raise, and every slot terminates after ``terminal_at`` steps. + """ + + def __init__( + self, + num_envs: int = 1, + *, + failures: int = 0, + terminal_at: int | None = None, + step_timeout: float = 30.0, + ) -> None: + super().__init__() + self.num_envs = num_envs + self.failures = failures + self.terminal_at = terminal_at + self.step_timeout = step_timeout + self.contract = { + "control_rate": 10, + "features": {"action": {"role": "action", "names": ["a"]}}, + } + self.steps: list[list[float]] = [] + self.results = 0 + + def reset(self, **task_args: Any) -> str: + return f"prompt {sorted(task_args.items())}" + + def step(self, action: NDArray[Any]) -> None: + self.steps.append(np.asarray(action).reshape(-1).tolist()) + + def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: + done = self.terminal_at is not None and len(self.steps) >= self.terminal_at + tick = np.full((self.num_envs, 1), len(self.steps), dtype=np.float32) + return {"tick": tick}, np.full(self.num_envs, done) + + def result(self) -> dict[str, Any]: + self.results += 1 + if self.results <= self.failures: + raise RuntimeError("the sim is busy") + return {"score": 0.42, "success": True, "total_reward": 3.0, "detail": "custom"} + + +class AsyncResetSim(Sim): + """A bridge whose reset is a coroutine.""" + + def __init__(self) -> None: + super().__init__() + + async def reset(**task_args: Any) -> str: + await asyncio.sleep(0) + return f"async prompt {sorted(task_args.items())}" + + setattr(self, "reset", reset) + + +@asynccontextmanager +async def serving(sim: Sim) -> AsyncIterator[tuple[int, RobotEndpoint, Capability]]: + """Serve ``sim``: its control port, a started endpoint on it, and the ``robot`` capability.""" + await sim.start() + control = await sim.serve_control() + port = control.sockets[0].getsockname()[1] + endpoint = RobotEndpoint.remote("127.0.0.1", port) + await endpoint.start() + try: + yield port, endpoint, Capability.robot(url=sim.url, contract=sim.contract) + finally: + await endpoint.stop() + control.close() + await sim.stop() + + +def slot_env(port: int) -> Environment: + """An env whose ``episode`` claims a slot of the sim on *port* and grades it.""" + env = Environment("slots") + endpoint = RobotEndpoint.remote("127.0.0.1", port).attach(env) + + @env.initialize + async def connect_sim() -> None: + await endpoint.start() + + @env.shutdown + async def disconnect_sim() -> None: + await endpoint.stop() + + @env.template() + async def episode(task: str = "pick"): + claimed = await endpoint.reset(task=task) + yield claimed["prompt"] + yield await endpoint.result() + + return env + + +def episodes(port: int) -> AbstractAsyncContextManager[Runtime]: + """The ``slots`` env served on loopback for the block.""" + return LocalRuntime(slot_env(port))(Task(env="slots", id="episode")) + + +PROMPT = "prompt [('task', 'pick')]" +GRADE = {"score": 0.42, "success": True, "total_reward": 3.0, "detail": "custom"} + + +@pytest.mark.parametrize( + ("sim", "ending", "results_while_served", "results_after_stop", "restarts"), + [ + pytest.param(Sim, "cancel", 2, 2, True, id="cancel-frees-the-slot"), + pytest.param(Sim, "grade", 2, 2, True, id="grade-frees-the-slot-with-the-sims-grade"), + pytest.param(Sim, "drop", 2, 2, True, id="a-dropped-session-frees-it-when-graded"), + pytest.param( + lambda: Sim(failures=1), "cancel", 3, 3, True, id="cancel-retries-a-failed-release" + ), + pytest.param( + lambda: Sim(failures=3), + "cancel", + 3, + 4, + False, + id="stopping-frees-what-cancel-could-not", + ), + ], +) +async def test_ending_a_session_frees_its_slot_exactly_once( + sim: Callable[[], Sim], + ending: str, + results_while_served: int, + results_after_stop: int, + restarts: bool, +) -> None: + bridge = sim() + + async with serving(bridge) as (port, peer, _): + async with episodes(port) as runtime: + async with connect(runtime) as first: + assert await first.start_task("episode") == {"prompt": PROMPT} + if ending == "cancel": + await first.cancel() + elif ending == "grade": + assert await first.grade({"answer": "x"}) == GRADE + async with connect(runtime) as second: + if ending == "drop": + assert await second.grade({"answer": "x"}) == GRADE + if restarts: + started = await asyncio.wait_for(second.start_task("episode"), timeout=10) + assert started == {"prompt": PROMPT} + assert await second.grade({"answer": "x"}) == GRADE + while_served = bridge.results + reclaimed = await asyncio.wait_for(peer.reset(task="pick"), timeout=10) + + assert (while_served, bridge.results) == (results_while_served, results_after_stop) + assert reclaimed["prompt"] == PROMPT + + +async def test_a_session_waits_for_a_held_slot_until_its_holder_ends() -> None: + sim = Sim() + + async with ( + serving(sim) as (port, _, _), + episodes(port) as runtime, + connect(runtime) as holder, + connect(runtime) as waiter, + ): + await holder.start_task("episode") + waiting = asyncio.create_task(waiter.start_task("episode")) + with pytest.raises(TimeoutError): + await asyncio.wait_for(asyncio.shield(waiting), timeout=0.2) + await holder.grade({"answer": "x"}) + started = await asyncio.wait_for(waiting, timeout=10) + await waiter.cancel() + + assert started == {"prompt": PROMPT} + assert sim.results == 2 + + +async def test_an_async_reset_is_awaited_for_the_sessions_prompt() -> None: + async with ( + serving(AsyncResetSim()) as (port, _, _), + episodes(port) as runtime, + connect(runtime) as session, + ): + started = await session.start_task("episode") + await session.cancel() + + assert started == {"prompt": "async " + PROMPT} + + +async def test_concurrent_sessions_with_different_args_cannot_share_a_batch() -> None: + async with ( + serving(Sim(num_envs=2)) as (port, _, _), + episodes(port) as runtime, + connect(runtime) as first, + connect(runtime) as second, + ): + started = await first.start_task("episode", {"task": "A"}) + with pytest.raises(HudProtocolError): + await second.start_task("episode", {"task": "B"}) + + assert started == {"prompt": "prompt [('task', 'A')]"} + + +@pytest.mark.parametrize( + ("num_envs", "token", "error"), + [ + pytest.param(1, "slot-9-nope", "unknown episode token", id="an-unknown-token-is-refused"), + pytest.param(2, None, "tokenless claim", id="a-vectorized-sim-refuses-a-tokenless-grade"), + ], +) +async def test_a_grade_must_name_exactly_one_claimed_slot( + num_envs: int, token: str | None, error: str +) -> None: + sim = Sim(num_envs=num_envs) + + async with serving(sim) as (_, endpoint, _): + claims = [await endpoint.reset(goal="lift") for _ in range(num_envs)] + with pytest.raises(RuntimeError, match=error): + await endpoint.result(token=token) + + assert len({claim["token"] for claim in claims}) == num_envs # one token per slot + + +# ─── stepping a batch ────────────────────────────────────────────────── + + +async def test_a_silent_slot_times_out_while_its_peer_keeps_stepping() -> None: + sim = Sim(num_envs=2, step_timeout=0.05) + + async with serving(sim) as (_, endpoint, robot): + tokens = [(await endpoint.reset())["token"] for _ in range(2)] + active, silent = [await RobotClient.connect(robot, token=token) for token in tokens] + try: + await active.get_observation() + await silent.get_observation() + await active.send_action([1.0]) + stepped = await active.get_observation() + with pytest.raises(RuntimeError, match="slot timed out waiting for an action"): + await silent.get_observation() + await active.send_action([2.0]) + stepped_again = await active.get_observation() + finally: + await active.close() + await silent.close() + + assert (stepped["data"]["tick"].tolist(), stepped_again["data"]["tick"].tolist()) == ( + [1.0], + [2.0], + ) + assert sim.steps == [[1.0, 0.0], [2.0, 0.0]] + + +async def test_a_claim_that_never_dials_holds_the_batch_until_it_joins() -> None: + sim = Sim(num_envs=2, step_timeout=0.5) + + async with serving(sim) as (_, endpoint, robot): + tokens = [(await endpoint.reset())["token"] for _ in range(2)] + early = await RobotClient.connect(robot, token=tokens[0]) + try: + await early.get_observation() + await early.send_action([1.0]) + # Longer than the step timeout: a claim still dialing is never timed out. + with pytest.raises(TimeoutError): + await asyncio.wait_for(early.get_observation(), timeout=1.0) + steps_before_join = len(sim.steps) + late = await RobotClient.connect(robot, token=tokens[1]) + try: + await late.get_observation() + await late.send_action([2.0]) + joined = await asyncio.wait_for(late.get_observation(), timeout=10) + finally: + await late.close() + finally: + await early.close() + + assert steps_before_join == 0 + assert joined["data"]["tick"].tolist() == [1.0] + assert sim.steps == [[1.0, 2.0]] + + +async def test_a_terminated_slot_is_not_stepped_without_new_actions() -> None: + sim = Sim(terminal_at=3) + + async with serving(sim) as (_, endpoint, robot): + await endpoint.reset() + agent = await RobotClient.connect(robot) + try: + await agent.get_observation() + terminated = [] + for value in (1.0, 2.0, 3.0): + await agent.send_action([value]) + terminated.append((await agent.get_observation())["terminated"]) + with pytest.raises(TimeoutError): + await asyncio.wait_for(agent.get_observation(), timeout=0.3) + finally: + await agent.close() + + assert terminated == [False, False, True] + assert sim.steps == [[1.0], [2.0], [3.0]] diff --git a/tests/robot/test_trace_reporting.py b/tests/robot/test_trace_reporting.py new file mode 100644 index 000000000..60e39f712 --- /dev/null +++ b/tests/robot/test_trace_reporting.py @@ -0,0 +1,257 @@ +"""What a robot run reports: its traces, their outcomes, and every tick as state and video. + +Three producers report robot telemetry: ``wrap`` around a vectorized gym env, +``JobRecorder`` for a shared job, and direct control streaming each played tick +of a rollout. Each is run as a user runs it; the cases assert what the platform +recorded and the trace spans it can replay. +""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import itertools +import logging +import threading +from datetime import datetime +from typing import TYPE_CHECKING, Any, ClassVar, cast + +import av +import numpy as np +import pytest + +from hud.agents.base import Agent +from hud.environment.robot import DirectControl, wrap +from hud.eval import LocalRuntime, Task, rollout +from hud.telemetry.robot import JobRecorder +from tests.support import ROBOT_STEP_SCHEMA, steps + +from .sims import Arm, ScriptedLLM, arm_env, contract, move, reached + +if TYPE_CHECKING: + from numpy.typing import NDArray + + from hud.capabilities import MCPClient + from hud.eval.run import Run + from tests.support import FakeServices + + +class TwoArmSim: + """Two envs: env 0 finishes on the second step (succeeding with a shaped reward of + 0.2), env 1 on the third (failing with a shaped reward of 5.0).""" + + num_envs = 2 + metadata: ClassVar[dict[str, int]] = {"render_fps": 10} + DONES = ([False, False], [True, False], [False, True]) + + def __init__(self) -> None: + self.t = 0 + + def observe(self) -> dict[str, Any]: + return { + "state": np.full((2, 3), self.t, dtype=np.float32), + "cam": np.full((2, 16, 16, 3), self.t, dtype=np.uint8), + } + + def reset(self, **kwargs: Any) -> tuple[dict[str, Any], dict[str, Any]]: + del kwargs + return self.observe(), {} + + def step(self, action: Any) -> tuple[Any, ...]: + del action + done = np.array(self.DONES[self.t]) + self.t += 1 + info = {"success": np.array([True, False])} + return self.observe(), np.array([0.2, 5.0]), done, np.array([False, False]), info + + def close(self) -> None: + pass + + +def test_a_wrapped_vectorized_sim_reports_each_episode_as_its_own_trace( + reporting: FakeServices, +) -> None: + env = wrap(TwoArmSim(), contract=None) + env.reset() + for _ in TwoArmSim.DONES: + env.step(np.zeros((2, 2))) + env.close() + + entered = [ + r.path.split("/")[3] for r in reporting.requests("api", "POST", "/v2/trace/{id}/enter") + ] + labels = {trace_id: f"trace-{n}" for n, trace_id in enumerate(entered)} + exits = { + labels[r.path.split("/")[3]]: r.json + for r in reporting.requests("api", "POST", "/v2/trace/{id}/exit") + } + spans: dict[str, list[dict[str, Any]]] = {label: [] for label in labels.values()} + for upload in reporting.requests("telemetry", "POST", "/trace/{id}/telemetry-upload"): + spans[labels[upload.path.split("/")[2].replace("-", "")]].extend(upload.json["telemetry"]) + ticks = { + trace: [ + (span["name"], span["attributes"]["hud.payload"]["tick"]) + for span in sorted(recorded, key=lambda span: span["start_time"]) + if span["name"] != "step.video_segment" + ] + for trace, recorded in spans.items() + } + assert exits == { + "trace-0": { + "status": "completed", + "reward": 1.0, + "metadata": {"env_index": 0, "episode_index": 0, "seed": None, "success": True}, + }, + "trace-1": { + "status": "completed", + "reward": 0.0, + "metadata": {"env_index": 1, "episode_index": 0, "seed": None, "success": False}, + }, + "trace-2": { + "status": "completed", + "reward": 0.2, + "metadata": {"env_index": 0, "episode_index": 1, "seed": None}, + }, + } + assert dict(sorted(ticks.items())) == { + "trace-0": [("step.observation", 0), ("step.inference", 0), ("step.inference", 1)], + "trace-1": [ + ("step.observation", 0), + ("step.inference", 0), + ("step.observation", 1), + ("step.inference", 1), + ("step.inference", 2), + ], + "trace-2": [("step.observation", 0), ("step.inference", 0)], + } + + +CANONICAL_JOB = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" + + +@pytest.mark.parametrize( + ("job_id", "trace_id"), + [ + pytest.param( + "03dd2a73d3df4d10a54ae3d87c2d530d", "e9266339c6fa58b08102ce7f41c4f372", id="compact" + ), + pytest.param(CANONICAL_JOB, "7b3269f689b65c5887542a9bc1ffb5c8", id="canonical"), + ], +) +def test_a_shared_job_is_reported_canonically_and_seeds_traces_by_its_spelling( + reporting: FakeServices, job_id: str, trace_id: str +) -> None: + recorder = JobRecorder("suite", 1, record_indices=[0], seed=7, job_id=job_id) + recorder.record(done=np.array([True])) + + assert recorder.job_url == f"{reporting.url('web')}/jobs/{CANONICAL_JOB}" + assert f"/v2/trace/job/{CANONICAL_JOB}/enter" in [ + request.path for request in reporting.requests("api") + ] + (entered,) = reporting.requests("api", "POST", "/v2/trace/{id}/enter") + assert (entered.path, entered.json) == ( + f"/v2/trace/{trace_id}/enter", + {"job_id": CANONICAL_JOB}, + ) + + +# ─── direct control streams every played tick ────────────────────────── + + +def frames_in(segments: list[dict[str, Any]]) -> int: + """How many frames the recorded MP4 segments decode to.""" + mp4 = b"".join(base64.b64decode(segment["segment"]["data"]) for segment in segments) + with av.open(io.BytesIO(mp4), mode="r") as container: + return sum(1 for _ in container.decode(video=0)) + + +def recorded(run: Run) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """The trace's observation steps and video segments, in order.""" + payloads = steps(run.trace_id, schema=ROBOT_STEP_SCHEMA) + return ( + [p for p in payloads if p.get("source") == "observation"], + [p for p in payloads if p.get("source") == "video_segment"], + ) + + +@pytest.mark.parametrize(("control_rate", "tick_seconds"), [(10, 0.1), (0.4, 1.0)]) +async def test_every_played_tick_streams_to_the_trace_as_state_and_one_video_per_camera( + control_rate: float, tick_seconds: float +) -> None: + sim = Arm(contract("ee_abs", [0.0, -1.0], [1.0, 1.0], rate=control_rate), done=reached) + agent = ScriptedLLM(move("move_to", x=0.3), move("move_to", x=0.5)) + + async with arm_env(sim, DirectControl()) as env: + run = await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) + + observations, segments = recorded(run) + starts = [datetime.fromisoformat(obs["started_at"]) for obs in observations] + assert {segment["camera"] for segment in segments} == {"observation/image"} + # The opening scene, then one observation and one frame per tick of both moves: + # the recording keeps every frame however far the encoder falls behind. + assert frames_in(segments) == 1 + len(sim.actions) + assert [obs["tick"] for obs in observations] == list(range(1 + len(sim.actions))) + assert observations[-1]["state"] + # The viewer's clock is the stamps' span: the video's length, not the calls' latency. + assert [(b - a).total_seconds() for a, b in itertools.pairwise(starts)] == pytest.approx( + [tick_seconds] * len(sim.actions) + ) + + +class AbandonsMidMove(Agent): + """Starts a long move, then fails once the sim has stepped ``after`` ticks.""" + + def __init__(self, moving: threading.Event) -> None: + super().__init__() + self.moving = moving + + async def __call__(self, run: Run) -> None: + client = cast("MCPClient", await run.client.open("control")) + targets = {"targets": {"x": 0.5, "grip": None}, "note": "move toward the goal"} + playing = asyncio.create_task(client.call_tool("move_to", targets)) + assert await asyncio.to_thread(self.moving.wait, 30) + playing.cancel() + raise RuntimeError("the agent gave up") + + +async def test_ending_the_episode_mid_move_ends_the_recording_with_the_ticks_played() -> None: + moving = threading.Event() + + def announcing(arm: Arm, target: NDArray[np.float64]) -> NDArray[np.float64]: + if len(arm.actions) >= 3: + moving.set() + return target + + sim = Arm(contract("ee_abs", [0.0, -1.0], [1.0, 1.0]), follow=announcing) + + async with arm_env(sim, DirectControl(speed=0.001)) as env: + run = await rollout( + Task(env="arm", id="reach"), AbandonsMidMove(moving), runtime=LocalRuntime(env) + ) + + observations, segments = recorded(run) + ticks = [obs["tick"] for obs in observations] + assert run.trace.status == "error" + assert ticks == list(range(len(ticks))) + assert 3 <= len(ticks) < 1 + len(sim.actions) + assert frames_in(segments) == len(ticks) + + +async def test_a_camera_the_encoder_cannot_encode_costs_the_video_not_the_rollout( + caplog: pytest.LogCaptureFixture, +) -> None: + # A 1x1 frame is a valid image for the model but crops to nothing for H.264, + # so the encoder dies on the first frame while the move keeps streaming ticks. + sim = Arm(contract("ee_abs", [0.0, -1.0], [1.0, 1.0]), done=reached, side=1) + agent = ScriptedLLM(move("move_to", x=0.5), move("move_to", grip=1.0)) + + with caplog.at_level(logging.WARNING, logger="hud.telemetry.robot.video"): + async with arm_env(sim, DirectControl(max_step={"grip": 2.0})) as env: + run = await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) + + observations, segments = recorded(run) + assert run.reward == 1.0 + assert [obs["tick"] for obs in observations] == list(range(1 + len(sim.actions))) + assert segments == [] + assert "video encode failed (camera observation/image)" in caplog.text diff --git a/tests/support/__init__.py b/tests/support/__init__.py new file mode 100644 index 000000000..594aa9d8f --- /dev/null +++ b/tests/support/__init__.py @@ -0,0 +1,104 @@ +"""Stand-ins for what lies outside the SDK: the user's machine, the HUD services, model +providers, a browser, a VNC screen, MCP servers, the network, and a raw peer on either +side of the control channel. The SDK code tests run is in ``tests/fixtures``.""" + +from __future__ import annotations + +from .asgi import mcp_server, serve_asgi +from .browser import Command, FakeBrowser, fake_browser +from .internet import internet +from .machine import ( + ROBOT_STEP_SCHEMA, + Hud, + HudEnv, + Result, + Uv, + scrub, + spans, + steps, + words, + write_files, +) +from .models import ( + ModelRequest, + Models, + ToolCall, + Turn, + call, + calls, + click, + fail, + say, + shell, + shell_call, + stream_error, +) +from .peers import ( + FRAME_LIMIT, + SESSION_ID, + ControlPeer, + Wire, + answer, + control_peer, + encode, + hang_up, + hold, + wire, +) +from .platform import Platform +from .relay import FlakyRelay, relay +from .screen import FakeScreen, KeyEvent, PointerEvent, fake_screen +from .services import FakeServices, Reply, Request +from .waiting import eventually + +__all__ = [ + "FRAME_LIMIT", + "ROBOT_STEP_SCHEMA", + "SESSION_ID", + "Command", + "ControlPeer", + "FakeBrowser", + "FakeScreen", + "FakeServices", + "FlakyRelay", + "Hud", + "HudEnv", + "KeyEvent", + "ModelRequest", + "Models", + "Platform", + "PointerEvent", + "Reply", + "Request", + "Result", + "ToolCall", + "Turn", + "Uv", + "Wire", + "answer", + "call", + "calls", + "click", + "control_peer", + "encode", + "eventually", + "fail", + "fake_browser", + "fake_screen", + "hang_up", + "hold", + "internet", + "mcp_server", + "relay", + "say", + "scrub", + "serve_asgi", + "shell", + "shell_call", + "spans", + "steps", + "stream_error", + "wire", + "words", + "write_files", +] diff --git a/tests/support/asgi.py b/tests/support/asgi.py new file mode 100644 index 000000000..c32f312a1 --- /dev/null +++ b/tests/support/asgi.py @@ -0,0 +1,51 @@ +"""Serve an ASGI app, or an MCP server, on a loopback port inside the test's event loop.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Literal + +import uvicorn + +from hud.capabilities import Capability + +from .waiting import eventually + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from fastmcp import FastMCP + from starlette.types import ASGIApp + + +@asynccontextmanager +async def serve_asgi(app: ASGIApp) -> AsyncIterator[int]: + """Serve ``app`` on ``127.0.0.1`` and yield its port; stop it when the block ends.""" + server = uvicorn.Server( + uvicorn.Config( + app, host="127.0.0.1", port=0, log_level="error", lifespan="on", ws="websockets-sansio" + ) + ) + serving = asyncio.create_task(server.serve()) + await eventually(lambda: server.started or serving.done()) + if serving.done(): + serving.result() + try: + yield server.servers[0].sockets[0].getsockname()[1] + finally: + server.should_exit = True + await serving + + +@asynccontextmanager +async def mcp_server( + server: FastMCP, + *, + name: str = "tools", + transport: Literal["streamable-http", "sse"] = "streamable-http", +) -> AsyncIterator[Capability]: + """Serve ``server`` over HTTP on loopback; yield the capability that reaches it.""" + path = "/mcp" if transport == "streamable-http" else "/sse" + async with serve_asgi(server.http_app(path=path, transport=transport)) as port: + yield Capability.mcp(name=name, url=f"http://127.0.0.1:{port}{path}", transport=transport) diff --git a/tests/support/browser.py b/tests/support/browser.py new file mode 100644 index 000000000..82a350183 --- /dev/null +++ b/tests/support/browser.py @@ -0,0 +1,115 @@ +"""A fake Chromium DevTools endpoint: target discovery over HTTP and scripted CDP replies. + +:func:`fake_browser` serves, on one loopback port like Chrome's +``--remote-debugging-port``, ``GET /json`` (the page targets plus a service +worker), ``PUT /json/new`` (a new page), and a WebSocket per page at +``/devtools/page/``. Each command frame is recorded; its reply comes from +``replies[method]``: ``{"result": ...}`` or ``{"error": ...}`` is sent back, +``{"close": True}`` closes the socket instead, and ``{"hold": True}`` never +answers. Unscripted methods answer ``{"result": {}}``. Before every reply the +browser sends an unsolicited ``Page.frameNavigated`` event. +""" + +from __future__ import annotations + +import json +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +from starlette.applications import Starlette +from starlette.responses import JSONResponse +from starlette.routing import Route, WebSocketRoute + +from .asgi import serve_asgi + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from starlette.requests import Request + from starlette.websockets import WebSocket + + +@dataclass(frozen=True) +class Command: + target: str + method: str + params: dict[str, Any] + + +@dataclass +class FakeBrowser: + """The browser side. ``http`` and ``commands`` hold what the client sent.""" + + pages: list[str] = field(default_factory=lambda: ["page-1"]) + replies: dict[str, dict[str, Any]] = field(default_factory=dict) + #: Whether ``PUT /json/new`` creates a page; otherwise it answers ``{}``. + can_create: bool = True + http: list[str] = field(default_factory=list) + commands: list[Command] = field(default_factory=list) + port: int = 0 + + @property + def url(self) -> str: + return f"ws://127.0.0.1:{self.port}" + + def target(self, page: str) -> dict[str, str]: + return { + "id": page, + "type": "page", + "webSocketDebuggerUrl": f"{self.url}/devtools/page/{page}", + } + + async def _list(self, request: Request) -> JSONResponse: + self.http.append(f"{request.method} {request.url.path}") + worker = {"id": "worker", "type": "service_worker", "webSocketDebuggerUrl": "ws://x"} + return JSONResponse([worker, *(self.target(page) for page in self.pages)]) + + async def _new(self, request: Request) -> JSONResponse: + self.http.append(f"{request.method} {request.url.path}?{request.url.query}") + if not self.can_create: + return JSONResponse({}) + page = f"new-{len(self.pages) + 1}" + self.pages.append(page) + return JSONResponse(self.target(page)) + + async def _session(self, websocket: WebSocket) -> None: + page = websocket.path_params["page"] + await websocket.accept() + while True: + frame = json.loads(await websocket.receive_text()) + self.commands.append(Command(page, frame["method"], frame.get("params", {}))) + reply = self.replies.get(frame["method"], {"result": {}}) + if reply.get("close"): + await websocket.close() + return + if reply.get("hold"): + continue + event = {"method": "Page.frameNavigated", "params": {"frame": {"id": page}}} + await websocket.send_text(json.dumps(event)) + await websocket.send_text(json.dumps({"id": frame["id"], **reply})) + + +@asynccontextmanager +async def fake_browser( + *, + pages: list[str] | None = None, + replies: dict[str, dict[str, Any]] | None = None, + can_create: bool = True, +) -> AsyncIterator[FakeBrowser]: + """Serve a :class:`FakeBrowser` on a loopback port for the duration of the block.""" + browser = FakeBrowser( + pages=["page-1"] if pages is None else list(pages), + replies=replies or {}, + can_create=can_create, + ) + app = Starlette( + routes=[ + Route("/json", browser._list, methods=["GET"]), + Route("/json/new", browser._new, methods=["PUT"]), + WebSocketRoute("/devtools/page/{page}", browser._session), + ] + ) + async with serve_asgi(app) as port: + browser.port = port + yield browser diff --git a/tests/support/clis.py b/tests/support/clis.py new file mode 100644 index 000000000..d25e0f035 --- /dev/null +++ b/tests/support/clis.py @@ -0,0 +1,313 @@ +"""Stand-ins for the ``claude`` and ``codex`` executables a CLI agent launches. + +Each records what the agent handed it (argv, the variables it set, stdin, ``.hud_*`` +files), runs the commands it is told to, prints the JSONL stream the real CLI would, and +exits as told. :func:`answer`, :func:`shell`, :func:`fault` and :func:`prompt` speak +either CLI's stream by its executable name. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from pathlib import Path + +#: Variables the CLI agents set for the process they launch. +AGENT_VARIABLES = ("ANTHROPIC_", "CLAUDE_", "CODEX_", "DISABLE_", "HUD_", "IS_SANDBOX") + +STUB = """#!{python} +import json, os, subprocess, sys, time + +behavior = json.load(open({behavior!r})) +record = {{ + "argv": sys.argv[1:], + "env": {{ + key: value + for key, value in os.environ.items() + if key.startswith({prefixes!r}) and value + }}, + "stdin": sys.stdin.read(), + "cwd": os.getcwd(), + "files": {{ + name: open(name).read() for name in sorted(os.listdir(".")) if name.startswith(".hud_") + }}, + "pid": os.getpid(), +}} + + +def drive_computer(server, actions): + relay = subprocess.Popen( + [server["command"], *server["args"]], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + + def ask(message): + relay.stdin.write(json.dumps(message) + "\\n") + relay.stdin.flush() + while True: + reply = json.loads(relay.stdout.readline()) + if reply.get("id") == message["id"]: + return reply + + ask({{"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {{ + "protocolVersion": "2025-06-18", "capabilities": {{}}, + "clientInfo": {{"name": "stub", "version": "0"}}}}}}) + initialized = {{"jsonrpc": "2.0", "method": "notifications/initialized"}} + relay.stdin.write(json.dumps(initialized) + "\\n") + results = [] + for index, arguments in enumerate(actions, start=1): + reply = ask({{"jsonrpc": "2.0", "id": index, "method": "tools/call", + "params": {{"name": "computer", "arguments": arguments}}}}) + result = reply["result"] + results.append({{ + "isError": result.get("isError", False), + "content": [block.get("text") or block["type"] for block in result["content"]], + }}) + # Like the real CLI, end the stdio server rather than wait for it to drain. + relay.terminate() + relay.wait() + return results + + +for command in behavior["commands"]: + subprocess.run(command, shell=True, check=True) + +if behavior.get("computer"): + servers = json.load(open(".hud_mcp_config.json"))["mcpServers"] + record["computer"] = drive_computer(servers["computer-use"], behavior["computer"]) + +with open(os.path.join({captures!r}, str(os.getpid()) + ".json"), "w") as capture: + json.dump(record, capture) + +sys.stdout.write(behavior["stdout"]) +sys.stdout.flush() +sys.stderr.write(behavior.get("stderr", "")) +sys.stderr.flush() +if behavior.get("hang"): + time.sleep(60) +sys.exit(behavior.get("exit", 0)) +""" + + +@dataclass +class Stub: + """A stub CLI installed in ``bin``; ``captures()`` reads what each invocation recorded.""" + + root: Path + name: str + stdout: str = "" + stderr: str = "" + exit: int = 0 + hang: bool = False + commands: list[str] = field(default_factory=list[str]) + computer: list[dict[str, Any]] = field(default_factory=list[dict[str, Any]]) + + @property + def bin(self) -> Path: + return self.root / "bin" + + @property + def captures_dir(self) -> Path: + return self.root / "captures" + + def install(self) -> None: + """Write the stub and its behavior, discarding what earlier invocations recorded.""" + self.bin.mkdir(parents=True, exist_ok=True) + self.captures_dir.mkdir(parents=True, exist_ok=True) + for capture in self.captures_dir.glob("*.json"): + capture.unlink() + behavior = self.root / "behavior.json" + behavior.write_text( + json.dumps( + { + "stdout": self.stdout, + "stderr": self.stderr, + "exit": self.exit, + "hang": self.hang, + "commands": self.commands, + "computer": self.computer, + } + ) + ) + script = self.bin / self.name + script.write_text( + STUB.format( + python=sys.executable, + behavior=str(behavior), + prefixes=AGENT_VARIABLES, + captures=str(self.captures_dir), + ) + ) + script.chmod(0o755) + + def captures(self) -> list[dict[str, Any]]: + return [json.loads(path.read_text()) for path in sorted(self.captures_dir.glob("*.json"))] + + def shell_env(self, *, path: str | None = None) -> dict[str, str]: + """The workspace shell environment, with every agent variable this process holds + blanked so a capture shows only what the agent set. + + Workspace sessions are login shells, and system profiles prepend their own + directories (where a real ``claude`` may live), so the session's home gets a + profile that puts ``path`` (default: the stub, then the system) on ``PATH`` last. + """ + home = self.root / "home" + home.mkdir(parents=True, exist_ok=True) + search = path or f"{self.bin}:/usr/local/bin:/usr/bin:/bin" + (home / ".bash_profile").write_text(f"export PATH={search}\n") + blanked = {key: "" for key in os.environ if key.startswith(AGENT_VARIABLES)} + return {**blanked, "HOME": str(home)} + + +def process_exited(pid: int, *, within: float = 10.0) -> bool: + """Whether ``pid`` has exited (gone or a zombie), polling for up to ``within`` seconds.""" + deadline = time.monotonic() + within + while time.monotonic() < deadline: + state = subprocess.run( + ["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True, check=False + ).stdout.strip() + if not state or state.startswith("Z"): + return True + time.sleep(0.05) + return False + + +def jsonl(*events: dict[str, Any]) -> str: + return "".join(json.dumps(event) + "\n" for event in events) + + +# ─── what the real CLIs print ────────────────────────────────────────── + + +def claude_assistant(*content: dict[str, Any], stop: str = "end_turn") -> dict[str, Any]: + return { + "type": "assistant", + "message": { + "id": "msg", + "type": "message", + "role": "assistant", + "model": "claude-test", + "content": list(content), + "stop_reason": stop, + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 2}, + }, + } + + +def claude_result(text: str = "done", *, is_error: bool = False) -> dict[str, Any]: + return { + "type": "result", + "subtype": "success", + "is_error": is_error, + "result": text, + "session_id": "session-1", + "duration_ms": 5, + "num_turns": 2, + "total_cost_usd": 0.01, + } + + +def claude_answer(text: str) -> str: + return jsonl(claude_assistant({"type": "text", "text": text}), claude_result(text)) + + +def claude_shell(command: str, then: str) -> str: + return jsonl( + claude_assistant( + {"type": "tool_use", "id": "tool-1", "name": "Bash", "input": {"command": command}}, + stop="tool_use", + ), + { + "type": "user", + "message": { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "tool-1", "content": "ran"}], + }, + }, + claude_assistant({"type": "text", "text": then}), + claude_result(then), + ) + + +def claude_fault(message: str) -> str: + return jsonl(claude_result(message, is_error=True)) + + +def codex_item(kind: str, item_id: str, **fields: Any) -> dict[str, Any]: + return {"type": "item.completed", "item": {"id": item_id, "type": kind, **fields}} + + +CODEX_STARTED = {"type": "thread.started", "thread_id": "thread-1"} +CODEX_COMPLETED = { + "type": "turn.completed", + "usage": {"input_tokens": 20, "cached_input_tokens": 5, "output_tokens": 8}, +} + + +def codex_answer(text: str) -> str: + return jsonl( + CODEX_STARTED, codex_item("agent_message", "message-1", text=text), CODEX_COMPLETED + ) + + +def codex_shell(command: str, then: str) -> str: + return jsonl( + CODEX_STARTED, + codex_item( + "command_execution", + "command-1", + command=command, + aggregated_output="ran", + exit_code=0, + status="completed", + ), + codex_item("agent_message", "message-1", text=then), + CODEX_COMPLETED, + ) + + +def codex_fault(message: str) -> str: + return jsonl({"type": "turn.started"}, {"type": "turn.failed", "error": {"message": message}}) + + +# ─── the command line a CLI was launched with ────────────────────────── + + +def overrides(argv: list[str]) -> dict[str, str]: + """Codex's ``-c key=value`` overrides, values as written (TOML).""" + return dict(argv[index + 1].split("=", 1) for index, arg in enumerate(argv) if arg == "-c") + + +# ─── either CLI, by executable name ──────────────────────────────────── + + +def answer(cli: str, text: str) -> str: + """The stream ``cli`` prints when it answers ``text``.""" + return {"claude": claude_answer, "codex": codex_answer}[cli](text) + + +def shell(cli: str, command: str, then: str) -> str: + """The stream ``cli`` prints when it runs ``command`` and then answers ``then``.""" + return {"claude": claude_shell, "codex": codex_shell}[cli](command, then) + + +def fault(cli: str, message: str) -> str: + """The stream ``cli`` prints when its turn fails with ``message``.""" + return {"claude": claude_fault, "codex": codex_fault}[cli](message) + + +def prompt(cli: str, stdin: str) -> str: + """The prompt ``cli`` read from its stdin: Claude's stream-json message, or codex's text.""" + if cli == "claude": + return json.loads(stdin)["message"]["content"][0]["text"] + return stdin diff --git a/tests/support/internet.py b/tests/support/internet.py new file mode 100644 index 000000000..7fa34800e --- /dev/null +++ b/tests/support/internet.py @@ -0,0 +1,57 @@ +"""The internet, as a process in the test reaches it: by name, through DNS. + +:func:`internet` stands in for it for the rest of a test. Each name in +``servers`` resolves to a public address of its own, and a connection to that +address reaches the local port the test serves the host on, so code that +refuses anything but the public internet still reaches a test server. A name in +``resolve`` resolves to the address given (``"127.0.0.1"`` for a name that +points back into the substrate). Every other name fails to resolve, as it would +offline. IP literals and ``localhost`` resolve as usual, so loopback traffic in +the test process is unaffected. +""" + +from __future__ import annotations + +import ipaddress +import socket +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Mapping + + import pytest + + +def internet( + monkeypatch: pytest.MonkeyPatch, + servers: Mapping[str, int] | None = None, + resolve: Mapping[str, str] | None = None, +) -> None: + """Serve each of ``servers`` (name to local port) as a host on the public internet. + + In force until the test ends. + """ + public = {name: f"6.0.0.{index}" for index, name in enumerate(servers or {}, start=1)} + ports = {public[name]: port for name, port in (servers or {}).items()} + addresses = {**public, **(resolve or {})} + getaddrinfo = socket.getaddrinfo + connect = socket.socket.connect + + def lookup(host: Any, port: Any, *args: Any, **kwargs: Any) -> Any: + name = host.decode() if isinstance(host, bytes) else host + if name in addresses: + return getaddrinfo(addresses[name], port, *args, **kwargs) + try: + ipaddress.ip_address(name) + except ValueError: + if name not in (None, "localhost"): + raise socket.gaierror(socket.EAI_NONAME, f"{name} does not resolve") from None + return getaddrinfo(host, port, *args, **kwargs) + + def dial(self: socket.socket, address: Any) -> None: + if isinstance(address, tuple) and address[0] in ports: + address = ("127.0.0.1", ports[address[0]]) + connect(self, address) + + monkeypatch.setattr(socket, "getaddrinfo", lookup) + monkeypatch.setattr(socket.socket, "connect", dial) diff --git a/tests/support/machine.py b/tests/support/machine.py new file mode 100644 index 000000000..663ccb7ed --- /dev/null +++ b/tests/support/machine.py @@ -0,0 +1,280 @@ +"""The user's machine, as the SDK under test meets it: its environment variables, the +``hud`` command and ``uv`` it runs, the files it writes, and the spans the SDK leaves +on its disk. + +:class:`HudEnv` is the only place tests touch ``hud.settings``: the SDK reads its +configuration from environment variables once, when ``hud.settings`` is imported, so +:class:`HudEnv` sets them for this process and every subprocess it starts and rebuilds +the settings object from them. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from hud.settings import Settings, settings +from hud.utils.gateway import list_gateway_models + +from .services import SERVICE_ENV + +if TYPE_CHECKING: + from collections.abc import Mapping + + import pytest + + +# Every service the SDK calls points at a closed local port until a test opts into +# the fake services, so nothing a test does can reach a real HUD deployment. The +# web app is only ever printed as a link, so it keeps its default. +CLOSED_PORT = "http://127.0.0.1:9" + +ISOLATED = { + "HUD_TELEMETRY_ENABLED": "0", + "HUD_CLI_ANALYTICS_ENABLED": "0", + "HUD_SKIP_VERSION_CHECK": "1", + **{var: f"{CLOSED_PORT}/{service}" for service, var in SERVICE_ENV.items() if service != "web"}, +} + +# Provider SDKs read these directly, bypassing hud.settings. +PROVIDER_ENV = ( + "OPENAI_BASE_URL", + "ANTHROPIC_BASE_URL", + "GOOGLE_API_KEY", + "GOOGLE_GENAI_USE_VERTEXAI", +) + + +def setting_variables() -> list[str]: + """Environment variable names the SDK settings read.""" + return [ + field.validation_alias + for field in Settings.model_fields.values() + if isinstance(field.validation_alias, str) + ] + + +class HudEnv: + """Isolated SDK configuration for one test: a fresh home and no developer credentials.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch, home: Path) -> None: + self._monkeypatch = monkeypatch + self.home = home + home.mkdir(parents=True, exist_ok=True) + # The Docker CLI finds its plugins (compose) and current context under its + # config directory; keep the real one so a real daemon stays reachable. + monkeypatch.setenv( + "DOCKER_CONFIG", os.environ.get("DOCKER_CONFIG") or str(Path.home() / ".docker") + ) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + for name in (*setting_variables(), *PROVIDER_ENV): + monkeypatch.delenv(name, raising=False) + for name, value in ISOLATED.items(): + monkeypatch.setenv(name, value) + self._apply() + + def set(self, **values: str | None) -> None: + """Set variables (``None`` unsets one) and re-read the SDK settings from them.""" + for name, value in values.items(): + if value is None: + self._monkeypatch.delenv(name, raising=False) + else: + self._monkeypatch.setenv(name, value) + self._apply() + + def get(self, name: str) -> str | None: + return os.environ.get(name) + + def _apply(self) -> None: + fresh = Settings(_env_file=None) + for name in Settings.model_fields: + self._monkeypatch.setattr(settings, name, getattr(fresh, name)) + # The gateway model catalog is cached per process; a new configuration + # may point at a different catalog. + list_gateway_models.cache_clear() + + +UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + + +BOX_DRAWING = re.compile(r"[\u2500-\u259f]+") + + +def words(text: str) -> list[str]: + """``text`` as a reader sees it: box drawing and padding gone, blank lines dropped. + + Rich lays panels out for the terminal's width, so output is compared by its words. + """ + stripped = (" ".join(BOX_DRAWING.sub(" ", line).split()) for line in text.splitlines()) + return [line for line in stripped if line] + + +@dataclass(frozen=True) +class Result: + exit_code: int + stdout: str + stderr: str + + @property + def json(self) -> Any: + """The single JSON document a ``--json`` command prints on stdout.""" + return json.loads(self.stdout) + + @property + def lines(self) -> list[str]: + """Stdout as a reader sees its words; see :func:`words`.""" + return words(self.stdout) + + def __str__(self) -> str: + return f"exit {self.exit_code}\n--- stdout\n{self.stdout}\n--- stderr\n{self.stderr}" + + +class Hud: + """``hud`` bound to a working directory and a terminal width. Use the ``hud`` fixture. + + The subprocess inherits this process's environment, which the ``hud_env`` + fixture has already isolated: a temporary home, no developer credentials, + and every service URL pointed at the fake services or a closed port. + ``python`` runs the CLI from another interpreter, such as a project's own + virtual environment; it defaults to this one. + """ + + def __init__(self, cwd: Path, *, python: Path | None = None, columns: int = 120) -> None: + self.cwd = cwd + self.python = str(python) if python is not None else sys.executable + self.columns = columns + cwd.mkdir(parents=True, exist_ok=True) + + def __call__( + self, + *args: str, + cwd: Path | None = None, + input: str | None = None, + env: dict[str, str] | None = None, + timeout: float = 120, + ) -> Result: + completed = subprocess.run( + [self.python, "-m", "hud.cli", *args], + cwd=cwd or self.cwd, + env=self._env(env), + input=input, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + return Result(completed.returncode, completed.stdout, completed.stderr) + + def start( + self, *args: str, cwd: Path | None = None, env: dict[str, str] | None = None + ) -> subprocess.Popen[str]: + """Start ``hud`` without waiting, for commands that run until signalled. + + Stdout and stderr are pipes; the caller stops the process and reads them. + """ + return subprocess.Popen( + [self.python, "-m", "hud.cli", *args], + cwd=cwd or self.cwd, + env=self._env(env), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def _env(self, extra: dict[str, str] | None) -> dict[str, str]: + terminal = {"COLUMNS": str(self.columns), "NO_COLOR": "1", "TERM": "dumb"} + return {**os.environ, **terminal, **(extra or {})} + + +def scrub(text: str, *paths: Path) -> str: + """Replace temporary paths and UUIDs so output compares across runs.""" + for index, path in enumerate(paths): + text = text.replace(str(path), f"") + return UUID.sub("", text) + + +class Uv: + """``uv`` with the developer's cache and Python installs, whatever ``HOME`` a test set.""" + + def __init__(self, real_environment: dict[str, str]) -> None: + executable = shutil.which("uv", path=real_environment.get("PATH")) + if executable is None: + raise RuntimeError("the template scenarios need uv on PATH") + self.executable = executable + self.env = { + "UV_CACHE_DIR": self.ask(real_environment, "cache", "dir"), + "UV_PYTHON_INSTALL_DIR": self.ask(real_environment, "python", "dir"), + } + + def ask(self, environment: dict[str, str], *args: str) -> str: + return subprocess.run( + [self.executable, *args], env=environment, capture_output=True, text=True, check=True + ).stdout.strip() + + def __call__(self, *args: str, cwd: Path, env: dict[str, str] | None = None) -> None: + completed = subprocess.run( + [self.executable, *args], + cwd=cwd, + env={**os.environ, **self.env, **(env or {})}, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, f"uv {' '.join(args)}\n{completed.stderr}" + + +def write_files(root: Path, files: Mapping[str, bytes | str]) -> None: + """Write ``files`` under ``root``; a name ending in ``/`` is an empty directory.""" + root.mkdir(parents=True, exist_ok=True) + for relative, content in files.items(): + path = root / relative + if relative.endswith("/"): + path.mkdir(parents=True, exist_ok=True) + continue + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, str): + path.write_text(content) + else: + path.write_bytes(content) + + +STEP_SCHEMA = "hud.step.v1" +ROBOT_STEP_SCHEMA = "hud.robot.step.v1" + + +def spans(trace_id: str | None, span_dir: Path | None = None) -> list[dict[str, Any]]: + """Every span exported for the run ``trace_id``, in start order. + + Spans land in ``/.jsonl``; the span directory + is the SDK's own (``HUD_TELEMETRY_LOCAL_DIR``, else ``~/.hud/spans`` while + uploads are off). + """ + if trace_id is None: + raise ValueError("the run has no trace id, so it exported no spans") + directory = span_dir or Path(settings.span_dir or "") + path = directory / f"{uuid.UUID(trace_id).hex}.jsonl" + if not path.exists(): + return [] + records = [json.loads(line) for line in path.read_text("utf-8").splitlines() if line.strip()] + return sorted(records, key=lambda span: span.get("start_time", "")) + + +def steps( + trace_id: str | None, span_dir: Path | None = None, *, schema: str = STEP_SCHEMA +) -> list[dict[str, Any]]: + """The step payloads of a run, in order; ``schema=ROBOT_STEP_SCHEMA`` for robot telemetry.""" + return [ + span["attributes"]["hud.payload"] + for span in spans(trace_id, span_dir) + if span.get("attributes", {}).get("hud.schema") == schema + ] diff --git a/tests/support/models.py b/tests/support/models.py new file mode 100644 index 000000000..286a0dcb1 --- /dev/null +++ b/tests/support/models.py @@ -0,0 +1,617 @@ +"""Scripted model providers behind the fake inference gateway. + +One provider-neutral script answers the four wire protocols hud's agents speak: + +- OpenAI Chat Completions: ``POST /chat/completions`` +- OpenAI Responses: ``POST /responses`` +- Anthropic Messages, streamed as server-sent events: ``POST /v1/messages`` +- Gemini: ``POST /v1beta/models/{model}:generateContent`` + +A script is a list of turns, or a function from :class:`ModelRequest` to a turn. +The turn a request gets is read from the conversation the request carries (how +many assistant turns it already holds), so one script serves many concurrent +rollouts. Agents reach the script through ``HUD_GATEWAY_URL`` or a ``base_url``. +""" + +from __future__ import annotations + +import itertools +import json +import threading +from dataclasses import dataclass, field, replace +from typing import TYPE_CHECKING, Any + +from .services import Reply + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Sequence + + from .services import FakeServices, Request + + +@dataclass(frozen=True) +class ToolCall: + name: str + arguments: dict[str, Any] = field(default_factory=dict) + id: str | None = None + + +@dataclass(frozen=True) +class Turn: + """One assistant turn. ``fail`` answers with an HTTP error instead, such as a 529 or 400. + + ``native`` holds provider-native entries appended after the tool calls as they + are: Responses output items, Anthropic content blocks, Gemini parts or Chat + Completions tool calls. ``finish`` replaces the protocol's finish reason, and + ``extra`` is merged into the turn's envelope: the Chat Completions choice, the + Responses response, the Anthropic message or the Gemini candidate. + """ + + text: str | None = None + tool_calls: tuple[ToolCall, ...] = () + reasoning: str | None = None + truncated: bool = False + #: A command to run with whichever tool the request offers for running commands. + shell: str | None = None + #: A left click at pixel (x, y) of a screen (width, height), in the form the offered + #: computer tool takes. + click: tuple[int, int, int, int] | None = None + fail: Reply | None = None + native: tuple[dict[str, Any], ...] = () + finish: str | None = None + extra: dict[str, Any] = field(default_factory=dict[str, Any]) + + +def say(text: str, *, reasoning: str | None = None) -> Turn: + return Turn(text=text, reasoning=reasoning) + + +def call(name: str, /, **arguments: Any) -> Turn: + return Turn(tool_calls=(ToolCall(name, arguments),)) + + +def shell(command: str) -> Turn: + """Run ``command`` with whichever shell tool the request offers, as a model would.""" + return Turn(shell=command) + + +def click(x: int, y: int, *, screen: tuple[int, int]) -> Turn: + """Left-click pixel (``x``, ``y``) of a ``screen`` the model sees, with its computer tool.""" + return Turn(click=(x, y, *screen)) + + +def calls(*tool_calls: ToolCall, text: str | None = None) -> Turn: + return Turn(text=text, tool_calls=tool_calls) + + +def fail(status: int, message: str = "scripted provider error") -> Turn: + return Turn(fail=Reply(status=status, json={"error": {"type": "error", "message": message}})) + + +def shell_call(*commands: str, id: str = "call_shell", **action: Any) -> Turn: + """A Responses ``shell_call``; ``action`` adds ``timeout_ms`` or ``max_output_length``.""" + return Turn( + native=( + { + "type": "shell_call", + "id": f"sh_{id}", + "call_id": id, + "action": {"commands": list(commands), **action}, + "status": "completed", + }, + ) + ) + + +def stream_error(kind: str, *, headers: dict[str, str] | None = None) -> Turn: + """An Anthropic stream that opens and then reports an ``error`` event of type ``kind``.""" + events = [ + _sse("message_start", {"message": _message_start("m")}), + _sse("error", {"error": {"type": kind, "message": f"scripted {kind}"}}), + ] + return Turn(fail=Reply(stream=events, headers=headers or {}, content_type="text/event-stream")) + + +@dataclass(frozen=True) +class ModelRequest: + """A model request as the provider received it, with the turn it asks for.""" + + protocol: str + model: str + body: dict[str, Any] + turn: int + headers: dict[str, str] + + @property + def tools(self) -> list[str]: + """Names of the tools the request offers.""" + names: list[str] = [] + for tool in self.body.get("tools") or []: + if self.protocol == "gemini": + names.extend(decl["name"] for decl in tool.get("functionDeclarations", [])) + elif self.protocol == "chat": + names.append(tool["function"]["name"]) + elif "name" in tool: + names.append(tool["name"]) + return names + + @property + def shell_tool(self) -> str | None: + """The offered tool that runs a shell command: a provider-defined shell tool, or a + function taking a ``command``.""" + for tool in self.body.get("tools") or []: + if self.protocol == "gemini": + for decl in tool.get("functionDeclarations", []): + schema = ( + decl.get("parameters_json_schema") + or decl.get("parametersJsonSchema") + or decl.get("parameters") + or {} + ) + if "command" in schema.get("properties", {}): + return decl["name"] + elif self.protocol == "chat": + schema = tool["function"].get("parameters") or {} + if "command" in schema.get("properties", {}): + return tool["function"]["name"] + elif self.protocol == "responses" and tool.get("type") == "shell": + return "shell" + else: + schema = tool.get("input_schema") or tool.get("parameters") or {} + native = str(tool.get("type", "")).startswith("bash_") + if native or "command" in schema.get("properties", {}): + return tool["name"] + return None + + @property + def credential(self) -> str: + """The credential the request carries, in its protocol's header.""" + header = {"anthropic": "x-api-key", "gemini": "x-goog-api-key"}.get( + self.protocol, "authorization" + ) + return self.headers.get(header, "").removeprefix("Bearer ") + + @property + def tool_results(self) -> list[str]: + """Texts of the tool results this request returns to the model, oldest first.""" + if self.protocol == "chat": + return [ + _text(message.get("content")) + for message in self.body.get("messages", []) + if message.get("role") == "tool" + ] + if self.protocol == "anthropic": + return [ + _text(block.get("content")) + for message in self.body.get("messages", []) + for block in message.get("content") or [] + if isinstance(block, dict) and block.get("type") == "tool_result" + ] + if self.protocol == "responses": + return [ + _text(item.get("output")) + for item in _input_items(self.body) + if item.get("type", "").endswith("_call_output") + ] + return [ + json.dumps(part["functionResponse"].get("response")) + for content in self.body.get("contents", []) + for part in content.get("parts", []) + if "functionResponse" in part + ] + + @property + def prompt(self) -> str: + """Text of the first user turn.""" + if self.protocol in {"chat", "anthropic"}: + for message in self.body.get("messages", []): + if message.get("role") == "user": + return _text(message.get("content")) + if self.protocol == "responses": + for item in _input_items(self.body): + if item.get("role") == "user": + return _text(item.get("content")) + if self.protocol == "gemini": + for content in self.body.get("contents", []): + if content.get("role") == "user": + return "".join(part.get("text", "") for part in content.get("parts", [])) + return "" + + +class Models: + """The scripted providers on a :class:`FakeServices` gateway. Use the ``models`` fixture.""" + + def __init__(self, services: FakeServices) -> None: + self._services = services + self._turns: list[Turn] = [] + self._policy: Callable[[ModelRequest], Turn] | None = None + self._responses: dict[str, int] = {} + self._lock = threading.Lock() + self._ids = itertools.count(1) + services.route("gateway", "POST", "/chat/completions", handler=self._chat) + services.route("gateway", "POST", "/responses", handler=self._responses_api) + services.route("gateway", "POST", "/v1/messages", handler=self._anthropic) + services.route( + "gateway", "POST", "/v1beta/models/{model}:generateContent", handler=self._gemini + ) + + @property + def url(self) -> str: + return self._services.url("gateway") + + def script(self, turns: Sequence[Turn]) -> None: + """Answer the n-th assistant turn of every conversation with ``turns[n]``.""" + self._turns, self._policy = list(turns), None + + def respond(self, policy: Callable[[ModelRequest], Turn]) -> None: + """Answer every request with ``policy(request)``.""" + self._turns, self._policy = [], policy + + def queue(self, turns: Sequence[Turn]) -> None: + """Answer requests with ``turns`` in arrival order, so a retried request gets the next.""" + remaining = iter(turns) + self.respond(lambda _request: next(remaining)) + + def requests(self) -> list[ModelRequest]: + """Every model request so far, decoded, in arrival order.""" + return [self._decode(request) for request in self._services.requests("gateway", "POST")] + + # ─── protocol endpoints ─────────────────────────────────────────── + + def _next(self, request: ModelRequest) -> Turn: + if self._policy is not None: + turn = self._policy(request) + elif request.turn >= len(self._turns): + raise AssertionError( + f"model script has {len(self._turns)} turns; request asked for turn " + f"{request.turn + 1} ({request.protocol}, tool results {request.tool_results})" + ) + else: + turn = self._turns[request.turn] + if turn.click is not None: + return self._click(request, *turn.click) + if turn.shell is None: + return turn + tool = request.shell_tool + if tool is None: + raise AssertionError(f"the request offers no shell tool: {request.tools}") + if request.protocol == "responses" and tool == "shell": + return replace(shell_call(turn.shell), text=turn.text) + return replace(turn, tool_calls=(ToolCall(tool, {"command": turn.shell}),)) + + def _click(self, request: ModelRequest, x: int, y: int, width: int, height: int) -> Turn: + """A left click in the protocol's computer-use form: Anthropic's ``computer`` tool, + a Responses ``computer_call``, or Gemini's ``click_at`` on its 0-999 grid.""" + tools = request.body.get("tools") or [] + if request.protocol == "anthropic": + (name,) = [t["name"] for t in tools if str(t.get("type", "")).startswith("computer_")] + return call(name, action="left_click", coordinate=[x, y]) + if request.protocol == "responses": + call_id = self._id("call") + return Turn( + native=( + { + "type": "computer_call", + "id": f"cu_{call_id}", + "call_id": call_id, + "action": {"type": "click", "button": "left", "x": x, "y": y}, + "pending_safety_checks": [], + "status": "completed", + }, + ) + ) + if request.protocol == "gemini": + return call("click_at", x=x * 1000 // width, y=y * 1000 // height) + raise AssertionError(f"{request.protocol} has no computer-use tool") + + def _id(self, prefix: str) -> str: + return f"{prefix}_{next(self._ids)}" + + def _decode(self, raw: Request) -> ModelRequest: + body = raw.json or {} + if raw.path == "/chat/completions": + protocol, model = "chat", body.get("model", "") + turn = sum(1 for m in body.get("messages", []) if m.get("role") == "assistant") + elif raw.path == "/v1/messages": + protocol, model = "anthropic", body.get("model", "") + turn = sum(1 for m in body.get("messages", []) if m.get("role") == "assistant") + elif raw.path == "/responses": + protocol, model = "responses", body.get("model", "") + previous = body.get("previous_response_id") + with self._lock: + turn = ( + self._responses[previous] + 1 + if previous is not None + else sum( + 1 + for item in _input_items(body) + if item.get("role") == "assistant" or item.get("type") == "function_call" + ) + ) + else: + protocol, model = "gemini", raw.params["model"] + turn = sum(1 for c in body.get("contents", []) if c.get("role") == "model") + return ModelRequest(protocol, model, body, turn, raw.headers) + + def _chat(self, raw: Request) -> Reply: + request = self._decode(raw) + body = request.body + turn = self._next(request) + if turn.fail is not None: + return turn.fail + tool_calls = [ + { + "id": tool.id or self._id("call"), + "type": "function", + "function": {"name": tool.name, "arguments": json.dumps(tool.arguments)}, + } + for tool in turn.tool_calls + ] + tool_calls.extend(turn.native) + message: dict[str, Any] = {"role": "assistant", "content": turn.text} + if tool_calls: + message["tool_calls"] = tool_calls + if turn.reasoning is not None: + message["reasoning_content"] = turn.reasoning + finish = turn.finish or ( + "length" if turn.truncated else ("tool_calls" if tool_calls else "stop") + ) + choice = {"index": 0, "message": message, "finish_reason": finish, "logprobs": None} + return Reply( + json={ + "id": self._id("chatcmpl"), + "object": "chat.completion", + "created": 0, + "model": body.get("model", ""), + "choices": [{**choice, **turn.extra}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + + def _responses_api(self, raw: Request) -> Reply: + request = self._decode(raw) + body = request.body + turn = self._next(request) + if turn.fail is not None: + return turn.fail + response_id = self._id("resp") + with self._lock: + self._responses[response_id] = request.turn + output: list[dict[str, Any]] = [] + if turn.reasoning is not None: + output.append( + { + "type": "reasoning", + "id": self._id("rs"), + "summary": [{"type": "summary_text", "text": turn.reasoning}], + } + ) + if turn.text is not None: + output.append( + { + "type": "message", + "id": self._id("msg"), + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": turn.text, "annotations": []}], + } + ) + output.extend( + { + "type": "function_call", + "id": self._id("fc"), + "call_id": tool.id or self._id("call"), + "name": tool.name, + "arguments": json.dumps(tool.arguments), + "status": "completed", + } + for tool in turn.tool_calls + ) + output.extend(turn.native) + incomplete = turn.finish or ("max_output_tokens" if turn.truncated else None) + return Reply( + json={ + "id": response_id, + "object": "response", + "created_at": 0, + "status": "incomplete" if incomplete else "completed", + "incomplete_details": {"reason": incomplete} if incomplete else None, + "model": body.get("model", ""), + "output": output, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + **turn.extra, + } + ) + + def _anthropic(self, raw: Request) -> Reply: + request = self._decode(raw) + body = request.body + turn = self._next(request) + if turn.fail is not None: + return turn.fail + blocks: list[dict[str, Any]] = [] + if turn.reasoning is not None: + blocks.append({"type": "thinking", "thinking": turn.reasoning, "signature": "sig"}) + if turn.text is not None: + blocks.append({"type": "text", "text": turn.text}) + blocks.extend( + { + "type": "tool_use", + "id": tool.id or self._id("toolu"), + "name": tool.name, + "input": tool.arguments, + } + for tool in turn.tool_calls + ) + blocks.extend(turn.native) + stop = turn.finish or ( + "max_tokens" if turn.truncated else ("tool_use" if turn.tool_calls else "end_turn") + ) + message = { + "id": self._id("msg"), + "type": "message", + "role": "assistant", + "model": body.get("model", ""), + "content": blocks, + "stop_reason": stop, + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + **turn.extra, + } + if not body.get("stream"): + return Reply(json=message) + return Reply(stream=_anthropic_events(message), content_type="text/event-stream") + + def _gemini(self, raw: Request) -> Reply: + turn = self._next(self._decode(raw)) + if turn.fail is not None: + return turn.fail + parts: list[dict[str, Any]] = [] + if turn.reasoning is not None: + parts.append({"text": turn.reasoning, "thought": True}) + if turn.text is not None: + parts.append({"text": turn.text}) + parts.extend( + {"functionCall": {"name": tool.name, "args": tool.arguments}} + for tool in turn.tool_calls + ) + parts.extend(turn.native) + candidate = { + "content": {"role": "model", "parts": parts}, + "finishReason": turn.finish or ("MAX_TOKENS" if turn.truncated else "STOP"), + "index": 0, + } + return Reply( + json={ + "candidates": [{**candidate, **turn.extra}], + "usageMetadata": { + "promptTokenCount": 1, + "candidatesTokenCount": 1, + "totalTokenCount": 2, + }, + "modelVersion": raw.params["model"], + } + ) + + +def _sse(kind: str, data: dict[str, Any]) -> bytes: + return f"event: {kind}\ndata: {json.dumps({'type': kind, **data})}\n\n".encode() + + +def _message_start(model: str) -> dict[str, Any]: + return { + "id": "msg_stream", + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + +def _anthropic_events(message: dict[str, Any]) -> Iterator[bytes]: + event = _sse + start = {**message, "content": [], "stop_reason": None} + start.pop("stop_details", None) + yield event("message_start", {"message": start}) + for index, block in enumerate(message["content"]): + if block["type"] == "text": + yield event( + "content_block_start", + {"index": index, "content_block": {"type": "text", "text": ""}}, + ) + yield event( + "content_block_delta", + {"index": index, "delta": {"type": "text_delta", "text": block["text"]}}, + ) + for citation in block.get("citations") or []: + yield event( + "content_block_delta", + {"index": index, "delta": {"type": "citations_delta", "citation": citation}}, + ) + elif block["type"] == "thinking": + yield event( + "content_block_start", + { + "index": index, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + }, + ) + yield event( + "content_block_delta", + { + "index": index, + "delta": {"type": "thinking_delta", "thinking": block["thinking"]}, + }, + ) + yield event( + "content_block_delta", + { + "index": index, + "delta": {"type": "signature_delta", "signature": block["signature"]}, + }, + ) + elif "input" in block: + # A string input streams verbatim, so a script can send malformed tool JSON. + partial = block["input"] + yield event( + "content_block_start", + {"index": index, "content_block": {**block, "input": {}}}, + ) + yield event( + "content_block_delta", + { + "index": index, + "delta": { + "type": "input_json_delta", + "partial_json": partial + if isinstance(partial, str) + else json.dumps(partial), + }, + }, + ) + else: + yield event("content_block_start", {"index": index, "content_block": block}) + yield event("content_block_stop", {"index": index}) + yield event( + "message_delta", + { + "delta": { + "stop_reason": message["stop_reason"], + "stop_sequence": None, + "stop_details": message.get("stop_details"), + }, + "usage": {"output_tokens": 1}, + }, + ) + yield event("message_stop", {}) + + +def _input_items(body: dict[str, Any]) -> list[dict[str, Any]]: + items = body.get("input", []) + if isinstance(items, str): + return [{"role": "user", "content": items}] + return [item for item in items if isinstance(item, dict)] + + +def _text(content: Any) -> str: + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join( + part.get("text", "") if isinstance(part, dict) else str(part) for part in content + ) + return json.dumps(content) diff --git a/tests/support/peers.py b/tests/support/peers.py new file mode 100644 index 000000000..1989de0a7 --- /dev/null +++ b/tests/support/peers.py @@ -0,0 +1,257 @@ +"""Raw control-channel peers, for the side of the wire a test does not run through the SDK. + +The control channel, like ``filetracking/1``, is newline-delimited JSON-RPC over TCP. +:func:`wire` speaks it to a served environment byte for byte, as a client would; +:func:`control_peer` is the environment side, answering an SDK client from a script. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +from dirty_equals import IsStr + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Sequence + + +#: A control session id as the server mints it. +SESSION_ID = IsStr(regex=r"sess-[0-9a-f]{8}") + +#: The control channel's frame limit, in bytes, excluding the newline. +FRAME_LIMIT = 16 * 1024 * 1024 + + +def encode(frame: dict[str, Any]) -> bytes: + """One frame as the control channel carries it.""" + return json.dumps(frame, separators=(",", ":")).encode() + b"\n" + + +class Wire: + """One TCP connection to a served control channel.""" + + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self.reader = reader + self.writer = writer + self._ids = 0 + + async def send(self, method: str, params: dict[str, Any] | None = None) -> None: + self._ids += 1 + frame: dict[str, Any] = {"jsonrpc": "2.0", "id": self._ids, "method": method} + if params is not None: + frame["params"] = params + await self.write(encode(frame)) + + async def write(self, data: bytes) -> None: + self.writer.write(data) + await self.writer.drain() + + async def read(self) -> dict[str, Any] | None: + """The next reply frame, or ``None`` when the server hung up.""" + line = await asyncio.wait_for(self.reader.readline(), timeout=30) + return json.loads(line) if line else None + + async def call(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + await self.send(method, params) + reply = await self.read() + assert reply is not None, f"the server hung up on {method}" + return reply + + async def hang_up(self) -> None: + """Close the sending side and wait until the server has ended the connection.""" + self.writer.write_eof() + while await asyncio.wait_for(self.reader.read(65536), timeout=30): + pass + await self.close() + + async def close(self) -> None: + self.writer.close() + with contextlib.suppress(ConnectionError): + await self.writer.wait_closed() + + +@asynccontextmanager +async def wire(url: str) -> AsyncIterator[Wire]: + """Open a raw connection to the JSON-RPC port at ``tcp://host:port``.""" + address = urlsplit(url) + reader, writer = await asyncio.open_connection( + address.hostname, address.port, limit=2 * FRAME_LIMIT + ) + connection = Wire(reader, writer) + try: + yield connection + finally: + await connection.close() + + +_DEFAULT_ID = object() + + +@dataclass(frozen=True) +class Action: + kind: str + result: Any = None + error: dict[str, Any] | None = None + reply_id: Any = _DEFAULT_ID + immediately: bool = False + + +def answer( + result: Any = None, *, error: dict[str, Any] | None = None, id: Any = _DEFAULT_ID +) -> Action: + return Action("answer", result=result, error=error, reply_id=id) + + +def hang_up(*, immediately: bool = False) -> Action: + return Action("hang_up", immediately=immediately) + + +def hold() -> Action: + return Action("hold") + + +@dataclass(frozen=True) +class Frame: + connection: int + message: dict[str, Any] + + +@dataclass +class ControlPeer: + """The peer's record. Use :func:`control_peer`.""" + + scripts: Sequence[Sequence[Action]] + upstreams: dict[str, tuple[str, int]] + port: int = 0 + #: Control connections accepted, tunnel streams excluded. + accepted: int = 0 + tunnels: int = 0 + frames: list[Frame] = field(default_factory=list) + _server: asyncio.Server | None = None + _handlers: set[asyncio.Task[None]] = field(default_factory=set) + + @property + def url(self) -> str: + return f"tcp://127.0.0.1:{self.port}" + + def requests(self, method: str | None = None) -> list[dict[str, Any]]: + """The request frames received, those calling ``method`` when given.""" + return [ + frame.message + for frame in self.frames + if method is None or frame.message.get("method") == method + ] + + def stop_accepting(self) -> None: + """Close the listening socket; open connections stay up.""" + assert self._server is not None + self._server.close() + + async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + task = asyncio.current_task() + assert task is not None + self._handlers.add(task) + try: + await self._converse(reader, writer) + except (OSError, asyncio.IncompleteReadError): + pass + finally: + self._handlers.discard(task) + writer.close() + + async def _converse(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + script = list(self.scripts[min(self.accepted, len(self.scripts) - 1)]) + if script and script[0].kind == "hang_up" and script[0].immediately: + self.accepted += 1 + return + first = await _read(reader) + if first is None: + self.accepted += 1 + return + if first.get("method") == "tunnel.open": + await self._tunnel(first, reader, writer) + return + connection = self.accepted + self.accepted += 1 + message: dict[str, Any] | None = first + while message is not None: + self.frames.append(Frame(connection, message)) + action = script.pop(0) if script else hold() + if action.kind == "hang_up": + return + if action.kind == "answer": + reply_id = message.get("id") if action.reply_id is _DEFAULT_ID else action.reply_id + body = ( + {"error": action.error} + if action.error is not None + else {"result": action.result} + ) + await _write(writer, {"jsonrpc": "2.0", "id": reply_id, **body}) + message = await _read(reader) + + async def _tunnel( + self, opened: dict[str, Any], reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + self.tunnels += 1 + self.frames.append(Frame(-1, opened)) + name = opened.get("params", {}).get("capability") + upstream = self.upstreams.get(name) + if upstream is None: + error = {"code": -32602, "message": f"unknown capability: {name!r}"} + await _write(writer, {"jsonrpc": "2.0", "id": opened.get("id"), "error": error}) + return + up_reader, up_writer = await asyncio.open_connection(*upstream) + await _write( + writer, {"jsonrpc": "2.0", "id": opened.get("id"), "result": {"capability": name}} + ) + try: + await asyncio.gather(_pipe(reader, up_writer), _pipe(up_reader, writer)) + finally: + up_writer.close() + + +async def _read(reader: asyncio.StreamReader) -> dict[str, Any] | None: + line = await reader.readline() + return json.loads(line) if line else None + + +async def _write(writer: asyncio.StreamWriter, message: dict[str, Any]) -> None: + writer.write(json.dumps(message).encode() + b"\n") + await writer.drain() + + +async def _pipe(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + except OSError: + pass + finally: + writer.close() + + +@asynccontextmanager +async def control_peer( + *scripts: Sequence[Action], upstreams: dict[str, tuple[str, int]] | None = None +) -> AsyncIterator[ControlPeer]: + """Serve a :class:`ControlPeer` answering connections from ``scripts`` for the block.""" + peer = ControlPeer(scripts=scripts or ([],), upstreams=upstreams or {}) + server = await asyncio.start_server(peer._handle, "127.0.0.1", 0) + peer._server = server + peer.port = server.sockets[0].getsockname()[1] + try: + yield peer + finally: + server.close() + for handler in list(peer._handlers): + handler.cancel() + with contextlib.suppress(asyncio.CancelledError): + await asyncio.gather(*peer._handlers, return_exceptions=True) + await server.wait_closed() diff --git a/tests/support/platform.py b/tests/support/platform.py new file mode 100644 index 000000000..ab200cc63 --- /dev/null +++ b/tests/support/platform.py @@ -0,0 +1,668 @@ +"""The HUD platform as tests see it: records a test seeds, and requests it records. + +One ``Platform`` answers the platform API through ``FakeServices``: projects, +the environment registry, tasksets, builds, Harbor imports, the model catalog +and its training service, jobs and traces, QA and CLI analytics. It answers from +what a test seeded and records what the code sent; it implements none of the +platform's own rules (what a cancel stops, which checkpoint is served, how QA +results settle), which the platform tests itself. It keeps state only where a +flow reads back what it wrote: a build triggered with the id its upload got, a +taskset found again by the name a sync created. A fault is a route override on +``platform.services``, which wins over the platform's own routes. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import uuid +from dataclasses import dataclass, field, replace +from itertools import count +from typing import TYPE_CHECKING, Any + +from .services import Reply + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + from starlette.websockets import WebSocket + + from .services import FakeServices, Request + + Handler = Callable[[Request], Reply | Awaitable[Reply]] + +API_KEY = "sk-hud-test" +USER_ID = "11111111-1111-4111-8111-111111111111" +TEAM_ID = "22222222-2222-4222-8222-222222222222" +WEB_URL = "https://hud.example" +DEFAULT_PROJECT_ID = "aaaaaaaa-0000-4000-8000-000000000001" +BROWSER_PROJECT_ID = "aaaaaaaa-0000-4000-8000-000000000002" +LOCKED_PROJECT_ID = "aaaaaaaa-0000-4000-8000-000000000003" +LIVE_IMPORT = {"queued", "starting", "running", "cancelling"} + + +def key(identifier: str) -> str: + """One spelling for an id the platform receives dashed or compact.""" + try: + return uuid.UUID(identifier).hex + except ValueError: + return identifier + + +def not_found(what: str) -> Reply: + return Reply(status=404, json={"detail": f"{what} not found"}) + + +@dataclass +class Build: + """A build: the context uploaded for it, the trigger that started it, how it ended.""" + + id: str + context: bytes | None = None + trigger: dict[str, Any] | None = None + status: dict[str, Any] = field( + default_factory=lambda: { + "status": "SUCCEEDED", + "version": "1", + "uri": "registry.hud.test/env:1", + "manifest": {"tasks": [], "capabilities": []}, + } + ) + #: The frames the build-log socket sends; ``None`` refuses the socket. + log: list[dict[str, Any]] | None = field( + default_factory=lambda: [ + {"type": "log", "message": "Step 1/2 : FROM python:3.12\n", "timestamp": 0}, + {"type": "status_update", "status": "SUCCEEDED"}, + {"type": "complete", "message": "Build SUCCEEDED"}, + ] + ) + + +@dataclass +class Trace: + """What the platform knows of one trace. + + A rollout reported from this process has an ``enter`` and an ``exit``; a hosted + one has the ``submitted`` request and the ``states`` its polls read, each read + moving to the next and the last one staying. + """ + + enter: dict[str, Any] | None = None + exit: dict[str, Any] | None = None + spans: list[dict[str, Any]] = field(default_factory=list) + events: list[dict[str, Any]] = field(default_factory=list) + submitted: dict[str, Any] | None = None + states: list[dict[str, Any]] = field(default_factory=list) + cancelled: bool = False + + +@dataclass +class Import: + """A Harbor import: the bundle uploaded for it and the statuses it moves through.""" + + id: str + taskset_id: str + bundle: bytes | None = None + statuses: list[str] = field(default_factory=lambda: ["completed"]) + format: str = "harbor" + error: str | None = None + report: dict[str, Any] | None = None + cancelled: bool = False + + def record(self, status: str) -> dict[str, Any]: + reported = status in {"completed", "stopped"} + return { + "id": self.id, + "format": self.format, + "taskset_id": self.taskset_id, + "file": { + "id": self.id, + "filename": "bundle.zip", + "size_bytes": len(self.bundle or b""), + }, + "status": status, + "summary": "Imported 1 task(s) and 1 run(s)" if reported else None, + "metrics": {"tasks_imported": 1.0} if reported else None, + "error": self.error, + "created_at": "2026-10-07T12:00:00Z", + "completed_at": None if status in LIVE_IMPORT else "2026-10-07T12:01:00Z", + } + + def advance(self) -> dict[str, Any]: + """The record at its current status; each read moves to the next, the last one stays.""" + status = self.statuses.pop(0) if len(self.statuses) > 1 else self.statuses[0] + return self.record(status) + + +@dataclass +class Training: + """One model's training: the gradient batches sent to it and the optimizer steps taken.""" + + batches: list[dict[str, Any]] = field(default_factory=list) + steps: list[dict[str, Any]] = field(default_factory=list) + + +class Platform: + """The fake HUD platform; see the module docstring.""" + + def __init__(self, services: FakeServices) -> None: + self.services = services + self._ids = count(1) + #: The keys the platform accepts; any other bearer is answered 401. + self.keys = {API_KEY} + #: The most items a list endpoint returns at once, whatever the caller asks for. + self.page_size = 500 + #: How the next build ends: what its status reports and what its log sends. + self.next_build = Build("") + self.projects: list[dict[str, Any]] = [ + { + "id": DEFAULT_PROJECT_ID, + "name": "default", + "is_default": True, + "capabilities": {"view": True, "create": True}, + }, + {"id": BROWSER_PROJECT_ID, "name": "browser-evals", "capabilities": {"create": True}}, + {"id": LOCKED_PROJECT_ID, "name": "locked-down", "capabilities": {"view": True}}, + ] + self.environments: dict[str, dict[str, Any]] = {} + self.tasksets: dict[str, dict[str, Any]] = {} + self.uploads: list[dict[str, Any]] = [] + self.builds: dict[str, Build] = {} + self.models: list[dict[str, Any]] = [] + self.checkpoints: dict[str, list[dict[str, Any]]] = {} + self.heads: dict[str, str] = {} + self.training: dict[str, Training] = {} + #: What every optimizer step answers. + self.optim_result: dict[str, Any] = { + "step": 1, + "checkpoint_id": "ckpt-trained", + "sampler_path": "sampler/ckpt-trained", + "state_path": "state/ckpt-trained", + "model": "owner/model", + } + self.jobs: dict[str, dict[str, Any]] = {} + self.job_traces: dict[str, list[dict[str, Any]]] = {} + #: Rollout cancellations as (scope, request body): ``job``, ``trace`` or ``all``. + self.cancellations: list[tuple[str, dict[str, Any]]] = [] + self.traces: dict[str, Trace] = {} + #: The states the next hosted rollout's trace moves through as it is polled. + self.next_rollout: list[dict[str, Any]] = [{"status": "completed", "reward": 1.0}] + self.imports: dict[str, Import] = {} + #: How the next import runs: the statuses it moves through and the report it leaves. + self.next_import = Import("", taskset_id="") + self.import_formats: list[dict[str, Any]] = [ + {"format": "harbor", "title": "Harbor", "max_input_bytes": 1024**3} + ] + self.qa_checks: list[dict[str, Any]] = [] + self.qa_results: list[dict[str, Any]] = [] + #: What successive reads of the results return, the last one repeating; empty + #: returns ``qa_results``, which starting a run adds to. + self.qa_polls: list[list[dict[str, Any]]] = [] + self.cli_events: list[dict[str, Any]] = [] + self._serve() + + # ─── seeding ────────────────────────────────────────────────────── + + def _id(self, prefix: str) -> str: + return f"{prefix}-0000-4000-8000-{next(self._ids):012d}" + + def add_environment(self, name: str, **record: Any) -> str: + """A deployed environment in the registry; returns its id.""" + environment_id = record.pop("id", None) or self._id("bbbbbbbb") + self.environments[environment_id] = { + "id": environment_id, + "name": name, + "latest_build": {"version": 1}, + **record, + } + return environment_id + + def add_taskset( + self, name: str, tasks: list[dict[str, Any]] | None = None, **record: Any + ) -> str: + """A taskset holding ``tasks`` as the platform exports them; returns its id.""" + taskset_id = record.pop("id", None) or self._id("eeeeeeee") + self.tasksets[taskset_id] = { + "id": taskset_id, + "name": name, + "can_edit": True, + "tasks": {task["name"]: task for task in tasks or []}, + **record, + } + return taskset_id + + def taskset(self, name: str) -> dict[str, Any]: + (found,) = [taskset for taskset in self.tasksets.values() if taskset["name"] == name] + return found + + def add_model(self, **record: Any) -> str: + """A model in the gateway catalog; returns its id.""" + model_id = record.setdefault("id", self._id("cccccccc")) + self.models.append(record) + return model_id + + def add_job(self, traces: list[dict[str, Any]] | None = None, **record: Any) -> str: + """A finished job the platform lists, with the traces it lists under it.""" + job_id = record.setdefault("id", self._id("dddddddd")) + self.jobs[key(job_id)] = record + self.job_traces[key(job_id)] = traces or [] + return job_id + + def trace(self, trace_id: str) -> Trace: + return self.traces.setdefault(key(trace_id), Trace()) + + # ─── serving ────────────────────────────────────────────────────── + + def _serve(self) -> None: + def route(service: str, method: str, path: str, handler: Handler) -> None: + self.services.route(service, method, path, handler=self._signed_in(handler)) + + route( + "api", + "GET", + "/v2/auth/me", + lambda _: Reply(json={"user_id": USER_ID, "team_id": TEAM_ID}), + ) + route("api", "GET", "/v2/projects", self._list_projects) + route("api", "GET", "/v2/projects/{id}", self._get_project) + route("api", "POST", "/v2/projects", self._create_project) + route("api", "GET", "/v2/registry", self._list_environments) + route("api", "GET", "/v2/registry/{id}", self._get_environment) + route("api", "GET", "/v2/tasksets/by-name/{name}", self._taskset_by_name) + route("api", "GET", "/v2/tasksets/{id}", self._get_taskset) + route("api", "GET", "/v2/tasksets/{id}/export", self._export_taskset) + route("api", "POST", "/v2/tasksets", self._create_taskset) + route("api", "POST", "/v2/tasks/upload", self._upload_tasks) + route("api", "POST", "/v2/builds/upload-url", self._build_upload_url) + self.services.route("api", "PUT", "/uploads/{id}", handler=self._receive_upload) + route("api", "POST", "/v2/builds/trigger", self._trigger_build) + route("api", "GET", "/v2/builds/{id}/status", self._build_status) + self.services.websocket("api", "/v2/builds/{id}/logs", self._build_log) + route("api", "GET", "/v2/models", self._list_models) + route("api", "GET", "/v2/models/{id}/checkpoints", self._checkpoints) + route("api", "PUT", "/v2/models/{id}/head", self._set_head) + route("api", "POST", "/v2/models/fork", self._fork_model) + route("rl", "POST", "/v1/models/{id}/train/forward-backward", self._accumulate) + route("rl", "POST", "/v1/models/{id}/train/backward", self._accumulate) + route("rl", "POST", "/v1/models/{id}/train/optim-step", self._optim_step) + route("api", "POST", "/v2/trace/job/{id}/enter", self._enter_job) + route("api", "POST", "/v2/trace/{id}/enter", self._enter_trace) + route("api", "POST", "/v2/trace/{id}/exit", self._exit_trace) + route("api", "GET", "/v2/trace/{id}/events", self._trace_events) + route("api", "POST", "/v2/rollouts/submit", self._submit_rollout) + route("api", "GET", "/v2/trace/{id}", self._poll_trace) + route("telemetry", "POST", "/trace/{id}/telemetry-upload", self._upload_spans) + route("api", "GET", "/v2/jobs", self._list_jobs) + route("api", "GET", "/v2/jobs/{id}/traces", self._job_traces) + route("api", "POST", "/v2/rollouts/cancel_job", self._cancel_job) + route("api", "POST", "/v2/rollouts/cancel", self._cancel_trace) + route("api", "POST", "/v2/rollouts/cancel_user_jobs", self._cancel_every_job) + route("api", "POST", "/v2/imports/uploads", self._import_upload_url) + route("api", "POST", "/v2/data/{id}/complete", self._complete_upload) + route("api", "DELETE", "/v2/data/{id}", self._delete_upload) + route("api", "POST", "/v2/imports", self._start_import) + route("api", "GET", "/v2/imports", self._list_imports) + route("api", "GET", "/v2/imports/{id}", self._get_import) + route("api", "GET", "/v2/imports/{id}/report", self._import_report) + route("api", "POST", "/v2/imports/{id}/cancel", self._cancel_import) + route("api", "GET", "/v2/imports/formats", self._import_formats) + route("api", "GET", "/v2/qa/checks", lambda _: Reply(json={"checks": self.qa_checks})) + route("api", "POST", "/v2/qa/runs", self._start_qa) + route("api", "GET", "/v2/qa/results", self._qa_results) + self.services.route("telemetry", "POST", "/sdk-events/cli", handler=self._cli_events) + + def _signed_in(self, handler: Handler) -> Handler: + """``handler``, answering 401 to a request without an accepted key.""" + + def answer(request: Request) -> Reply | Awaitable[Reply]: + if request.bearer not in self.keys: + return Reply(status=401, json={"detail": "Invalid API key"}) + return handler(request) + + return answer + + # projects and the registry + + def _list_projects(self, request: Request) -> Reply: + offset = int(request.query.get("offset", ["0"])[0]) + limit = min(int(request.query.get("limit", ["500"])[0]), self.page_size) + page = self.projects[offset : offset + limit] + return Reply(json={"items": page, "total": len(self.projects)}) + + def _get_project(self, request: Request) -> Reply: + found = [project for project in self.projects if project["id"] == request.params["id"]] + return Reply(json=found[0]) if found else not_found("Project") + + def _create_project(self, request: Request) -> Reply: + body = request.json + project = { + "id": self._id("aaaaaaaa"), + "name": body["name"], + "capabilities": {"view": True, "create": True}, + **({"description": body["description"]} if "description" in body else {}), + } + self.projects.append(project) + return Reply(json=project) + + def _list_environments(self, request: Request) -> Reply: + del request + listed = list(self.environments.values()) + return Reply(json={"items": listed, "total": len(listed)}) + + def _get_environment(self, request: Request) -> Reply: + found = self.environments.get(request.params["id"]) + return Reply(json=found) if found else not_found("Environment") + + # tasksets + + def _taskset_by_name(self, request: Request) -> Reply: + for taskset in self.tasksets.values(): + if taskset["name"] == request.params["name"]: + return Reply(json={"taskset_id": taskset["id"], "name": taskset["name"]}) + return not_found("Taskset") + + def _get_taskset(self, request: Request) -> Reply: + taskset = self.tasksets.get(request.params["id"]) + if taskset is None: + return not_found("Taskset") + return Reply(json={key: value for key, value in taskset.items() if key != "tasks"}) + + def _export_taskset(self, request: Request) -> Reply: + taskset = self.tasksets.get(request.params["id"]) + if taskset is None: + return not_found("Taskset") + return Reply(json={"name": taskset["name"], "tasks": list(taskset["tasks"].values())}) + + def _create_taskset(self, request: Request) -> Reply: + body = request.json + taskset_id = self.add_taskset(body["name"], project_id=body.get("project_id")) + taskset = self.tasksets[taskset_id] + return Reply(json={key: value for key, value in taskset.items() if key != "tasks"}) + + def _upload_tasks(self, request: Request) -> Reply: + body = request.json + self.uploads.append(body) + taskset_id = body.get("taskset_id") or self.add_taskset( + body["taskset_name"], project_id=body.get("project_id") + ) + tasks = self.tasksets[taskset_id]["tasks"] + created = sum(task["name"] not in tasks for task in body["tasks"]) + for task in body["tasks"]: + tasks[task["name"]] = { + "name": task["name"], + "env": task["env"]["name"], + "scenario": task["task_id"], + "args": task.get("args") or {}, + } + return Reply( + json={ + "taskset_id": taskset_id, + "tasks_created": created, + "tasks_updated": len(body["tasks"]) - created, + } + ) + + # builds + + def _build_upload_url(self, request: Request) -> Reply: + del request + build = replace(self.next_build, id=f"build-{next(self._ids)}") + self.builds[build.id] = build + url = f"{self.services.url('api')}/uploads/{build.id}" + return Reply(json={"upload_url": url, "build_id": build.id}) + + def _receive_upload(self, request: Request) -> Reply: + upload_id = request.params["id"] + if upload_id in self.builds: + self.builds[upload_id].context = request.body + else: + self.imports[upload_id].bundle = request.body + return Reply(json={}) + + def _trigger_build(self, request: Request) -> Reply: + body = request.json + build = self.builds[body["build_id"]] + build.trigger = body + environment_id = body.get("registry_id") or next( + (env["id"] for env in self.environments.values() if env["name"] == body["name"]), + None, + ) + if environment_id is None: + environment_id = self.add_environment(body["name"], project_id=body.get("project_id")) + return Reply(json={"id": build.id, "registry_id": environment_id}) + + def _build_status(self, request: Request) -> Reply: + build = self.builds.get(request.params["id"]) + return Reply(json=build.status) if build else not_found("Build") + + async def _build_log(self, websocket: WebSocket, params: dict[str, str]) -> None: + log = self.builds[params["id"]].log + if log is None: + await websocket.close(code=4404) + return + await websocket.accept() + for frame in log: + await websocket.send_text(json.dumps(frame)) + await websocket.close() + + # models + + def _list_models(self, request: Request) -> Reply: + offset = int(request.query.get("offset", ["0"])[0]) + limit = min(int(request.query.get("limit", ["100"])[0]), self.page_size) + page = self.models[offset : offset + limit] + return Reply(json={"items": page, "total": len(self.models)}) + + def _checkpoints(self, request: Request) -> Reply: + return Reply(json=self.checkpoints.get(request.params["id"], [])) + + def _set_head(self, request: Request) -> Reply: + self.heads[request.params["id"]] = request.json["checkpoint_id"] + return Reply(json={}) + + def _fork_model(self, request: Request) -> Reply: + body = request.json + self.add_model( + name=body["name"], + model_name=f"team/{body['name']}", + forked_from=body["source_model_id"], + ) + return Reply(json=self.models[-1]) + + # training + + def _accumulate(self, request: Request) -> Reply: + batch = request.json + self.training.setdefault(request.params["id"], Training()).batches.append(batch) + datums = batch.get("inputs") or batch.get("weights") or [] + return Reply(json={"metrics": {}, "num_datums": len(datums)}) + + def _optim_step(self, request: Request) -> Reply: + training = self.training.setdefault(request.params["id"], Training()) + training.steps.append(request.json) + return Reply(json=self.optim_result) + + # jobs and traces + + def _enter_job(self, request: Request) -> Reply: + self.jobs[key(request.params["id"])] = request.json + return Reply(json={}) + + def _enter_trace(self, request: Request) -> Reply: + self.trace(request.params["id"]).enter = request.json + return Reply(json={}) + + def _exit_trace(self, request: Request) -> Reply: + self.trace(request.params["id"]).exit = request.json + return Reply(json={}) + + def _trace_events(self, request: Request) -> Reply: + trace = self.traces.get(key(request.params["id"])) + return Reply(json={"events": trace.events}) if trace else not_found("Trace") + + def _upload_spans(self, request: Request) -> Reply: + self.trace(request.params["id"]).spans.extend(request.json["telemetry"]) + return Reply(json={}) + + def _list_jobs(self, request: Request) -> Reply: + del request + return Reply(json={"items": list(self.jobs.values())}) + + def _job_traces(self, request: Request) -> Reply: + traces = self.job_traces.get(key(request.params["id"])) + return Reply(json={"items": traces}) if traces is not None else not_found("Job") + + def _cancel_job(self, request: Request) -> Reply: + self.cancellations.append(("job", request.json)) + return Reply(json={"cancelled": 2}) + + def _cancel_trace(self, request: Request) -> Reply: + self.cancellations.append(("trace", request.json)) + self.trace(request.json["trace_id"]).cancelled = True + return Reply(json={"status": "accepted"}) + + def _submit_rollout(self, request: Request) -> Reply: + trace = self.trace(request.json["trace_id"]) + trace.submitted = request.json + trace.states = [dict(state) for state in self.next_rollout] + return Reply(json={"status": "queued"}) + + def _poll_trace(self, request: Request) -> Reply: + trace = self.traces.get(key(request.params["id"])) + if trace is None or not trace.states: + return not_found("Trace") + return Reply(json=trace.states.pop(0) if len(trace.states) > 1 else trace.states[0]) + + def _cancel_every_job(self, request: Request) -> Reply: + self.cancellations.append(("all", request.json)) + return Reply(json={"jobs_cancelled": 1, "total_tasks_cancelled": 2, "job_details": []}) + + # Harbor imports + + def _import_formats(self, request: Request) -> Reply: + del request + return Reply(json={"formats": self.import_formats}) + + def _import_upload_url(self, request: Request) -> Reply: + del request + upload_id = self._id("ffffffff") + self.imports[upload_id] = replace( + self.next_import, id=upload_id, statuses=list(self.next_import.statuses) + ) + url = f"{self.services.url('api')}/uploads/{upload_id}?signature=secret" + return Reply(json={"file": {"id": upload_id, "filename": "bundle.zip"}, "upload_url": url}) + + def _complete_upload(self, request: Request) -> Reply: + return Reply(json={"id": request.params["id"]}) + + def _delete_upload(self, request: Request) -> Reply: + self.imports.pop(request.params["id"], None) + return Reply(status=204) + + def _start_import(self, request: Request) -> Reply: + body = request.json + started = self.imports[body["file_id"]] + started.taskset_id = body["taskset_id"] + return Reply(json=started.advance()) + + def _list_imports(self, request: Request) -> Reply: + del request + listed = [ + started.record(started.statuses[0]) + for started in self.imports.values() + if started.taskset_id + ] + return Reply(json={"imports": listed, "total": len(listed)}) + + def _get_import(self, request: Request) -> Reply: + started = self.imports.get(request.params["id"]) + return Reply(json=started.advance()) if started else not_found("Import") + + def _import_report(self, request: Request) -> Reply: + started = self.imports.get(request.params["id"]) + if started is None or started.report is None: + return not_found("Import report") + return Reply(json={"taskset_id": started.taskset_id, **started.report}) + + def _cancel_import(self, request: Request) -> Reply: + started = self.imports[request.params["id"]] + started.cancelled = True + return Reply(json=started.record("cancelling")) + + # QA + + def _start_qa(self, request: Request) -> Reply: + body = request.json + level = "trace" if "trace_ids" in body else "task" + started = [ + { + "id": self._id("99999999"), + "check_key": check, + "subject_type": level, + f"subject_{level}_id": subject_id, + "status": "queued", + "verdict": None, + "result": None, + } + for check in body["check_keys"] + for subject_id in body[f"{level}_ids"] + ] + self.qa_results.extend(started) + return Reply(json={"results": started}) + + def _qa_results(self, request: Request) -> Reply: + del request + polled = self.qa_polls.pop(0) if len(self.qa_polls) > 1 else (self.qa_polls or [None])[0] + return Reply(json={"results": self.qa_results if polled is None else polled}) + + # CLI analytics + + def _cli_events(self, request: Request) -> Reply: + self.cli_events.extend(request.json["events"]) + return Reply(json={}) + + +# ─── the runtime gateway ─────────────────────────────────────────────── +# +# The HUD runtime service's tunnel: each WebSocket is spliced to a local control channel. + +SESSIONS = "/runtime/sessions" +SESSION = "/runtime/sessions/{id}" +TUNNEL = "/runtime/tunnels/{id}" + + +def tunnel(port: int) -> Callable[[WebSocket, dict[str, str]], Awaitable[None]]: + """A tunnel endpoint splicing each WebSocket to the control channel on ``port``.""" + + async def handler(websocket: WebSocket, params: dict[str, str]) -> None: + del params + await websocket.accept() + reader, writer = await asyncio.open_connection("127.0.0.1", port) + + async def upstream() -> None: + while (message := await websocket.receive())["type"] != "websocket.disconnect": + writer.write(message.get("bytes") or message.get("text", "").encode()) + await writer.drain() + + async def downstream() -> None: + while data := await reader.read(65536): + await websocket.send_bytes(data) + + tasks = [asyncio.create_task(upstream()), asyncio.create_task(downstream())] + await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + writer.close() + with contextlib.suppress(Exception): + await websocket.close() + + return handler + + +def host_on_runtime(services: FakeServices, port: int, session: str = "sess-1") -> None: + """Let the fake runtime service lease ``session`` and tunnel it to the env on ``port``.""" + services.route("runtime", "POST", SESSIONS, json={"id": session}) + services.route("runtime", "DELETE", SESSION, json={}) + services.websocket("runtime", TUNNEL, tunnel(port)) diff --git a/tests/support/relay.py b/tests/support/relay.py new file mode 100644 index 000000000..413b9e03d --- /dev/null +++ b/tests/support/relay.py @@ -0,0 +1,142 @@ +"""A TCP relay in front of one upstream that can sever, refuse and stall connections. + +Clients that reconnect (``SSHClient``, the control-channel tunnels) are driven +through :func:`relay` to fake an unreliable network between them and a real +server: :meth:`FlakyRelay.sever` drops every relayed connection, +:meth:`FlakyRelay.refuse` closes the next connections as soon as they are +accepted, and :meth:`FlakyRelay.stall` holds new connections without +forwarding them until :meth:`FlakyRelay.release`. +""" + +from __future__ import annotations + +import asyncio +import contextlib +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +class FlakyRelay: + """The relay's controls. Use :func:`relay`.""" + + def __init__(self, upstream: tuple[str, int]) -> None: + self.upstream = upstream + self.port = 0 + #: Connections accepted so far, refused and stalled ones included. + self.accepted = 0 + self._refusals = 0 + self._released = asyncio.Event() + self._released.set() + self._changed = asyncio.Condition() + self._writers: set[asyncio.StreamWriter] = set() + self._handlers: set[asyncio.Task[None]] = set() + self._stalled = 0 + + @property + def open(self) -> int: + """Client connections the relay currently holds open.""" + return len(self._writers) + + @property + def stalled(self) -> int: + """Connections waiting for :meth:`release`.""" + return self._stalled + + def sever(self) -> None: + """Drop every connection the relay holds, as a network failure would.""" + for writer in list(self._writers): + writer.transport.abort() + + def refuse(self, count: int) -> None: + """Close each of the next ``count`` connections as soon as it is accepted.""" + self._refusals = count + + def stall(self) -> None: + """Hold new connections open without forwarding them until :meth:`release`.""" + self._released.clear() + + def release(self) -> None: + self._released.set() + + async def wait_until(self, *, accepted: int | None = None, stalled: int | None = None) -> None: + """Wait until at least ``accepted`` connections were accepted or ``stalled`` wait.""" + async with self._changed: + await asyncio.wait_for( + self._changed.wait_for( + lambda: ( + (accepted is None or self.accepted >= accepted) + and (stalled is None or self._stalled >= stalled) + ) + ), + timeout=10, + ) + + async def _notify(self) -> None: + async with self._changed: + self._changed.notify_all() + + async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + task = asyncio.current_task() + assert task is not None + self._handlers.add(task) + self._writers.add(writer) + self.accepted += 1 + try: + if self._refusals: + self._refusals -= 1 + await self._notify() + return + if not self._released.is_set(): + self._stalled += 1 + await self._notify() + try: + await self._released.wait() + finally: + self._stalled -= 1 + else: + await self._notify() + up_reader, up_writer = await asyncio.open_connection(*self.upstream) + try: + await asyncio.gather(_pipe(reader, up_writer), _pipe(up_reader, writer)) + finally: + up_writer.transport.abort() + except OSError: + pass + finally: + self._writers.discard(writer) + self._handlers.discard(task) + writer.transport.abort() + await self._notify() + + +async def _pipe(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + except OSError: + pass + finally: + writer.transport.abort() + + +@asynccontextmanager +async def relay(host: str, port: int) -> AsyncIterator[FlakyRelay]: + """Serve a :class:`FlakyRelay` to ``host:port`` on a loopback port for the block.""" + controls = FlakyRelay((host, port)) + server = await asyncio.start_server(controls._handle, "127.0.0.1", 0) + controls.port = server.sockets[0].getsockname()[1] + try: + yield controls + finally: + server.close() + controls.release() + controls.sever() + for handler in list(controls._handlers): + handler.cancel() + with contextlib.suppress(asyncio.CancelledError): + await asyncio.gather(*controls._handlers, return_exceptions=True) + await server.wait_closed() diff --git a/tests/support/screen.py b/tests/support/screen.py new file mode 100644 index 000000000..89fab5a7e --- /dev/null +++ b/tests/support/screen.py @@ -0,0 +1,126 @@ +"""A minimal RFB (VNC) server: a fixed framebuffer that records input events. + +It speaks RFB 3.8 with no authentication and Raw encoding, enough for +``asyncvnc`` and therefore ``hud.capabilities.RFBClient`` and the computer-use +tools built on it. +""" + +from __future__ import annotations + +import asyncio +import struct +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +@dataclass(frozen=True) +class KeyEvent: + key: int + down: bool + + +@dataclass(frozen=True) +class PointerEvent: + x: int + y: int + buttons: int + + +@dataclass +class FakeScreen: + """The server side of a VNC session. ``events`` holds every key and pointer event received.""" + + width: int = 64 + height: int = 48 + color: tuple[int, int, int] = (40, 120, 200) + events: list[KeyEvent | PointerEvent] = field(default_factory=list) + connections: int = 0 + port: int = 0 + #: Close the connection instead of answering this framebuffer update request + #: (counted across connections, from 1); 0 never drops. + drop_on_update: int = 0 + updates: int = 0 + sessions: set[asyncio.Task[None]] = field(default_factory=set) + + @property + def url(self) -> str: + return f"rfb://127.0.0.1:{self.port}" + + def key_events(self) -> list[KeyEvent]: + return [event for event in self.events if isinstance(event, KeyEvent)] + + def pointer_events(self) -> list[PointerEvent]: + return [event for event in self.events if isinstance(event, PointerEvent)] + + async def _session(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self.connections += 1 + session = asyncio.current_task() + assert session is not None + self.sessions.add(session) + try: + writer.write(b"RFB 003.008\n") + await reader.readexactly(12) + writer.write(b"\x01\x01") # one security type: None + await reader.readexactly(1) + writer.write(struct.pack(">I", 0)) + await reader.readexactly(1) # ClientInit + name = b"hud-fake-screen" + pixel_format = struct.pack(">BBBBHHHBBB3x", 32, 24, 0, 1, 255, 255, 255, 0, 8, 16) + writer.write(struct.pack(">HH", self.width, self.height) + pixel_format) + writer.write(struct.pack(">I", len(name)) + name) + await writer.drain() + while True: + (kind,) = await reader.readexactly(1) + if kind == 0: # SetPixelFormat + await reader.readexactly(19) + elif kind == 2: # SetEncodings + _, count = struct.unpack(">BH", await reader.readexactly(3)) + await reader.readexactly(4 * count) + elif kind == 3: # FramebufferUpdateRequest + await reader.readexactly(9) + self.updates += 1 + if self.updates == self.drop_on_update: + return + pixel = bytes([*self.color, 255]) + writer.write(struct.pack(">BxH", 0, 1)) + writer.write(struct.pack(">HHHHi", 0, 0, self.width, self.height, 0)) + writer.write(pixel * (self.width * self.height)) + await writer.drain() + elif kind == 4: # KeyEvent + down, _, key = struct.unpack(">B2sI", await reader.readexactly(7)) + self.events.append(KeyEvent(key, bool(down))) + elif kind == 5: # PointerEvent + buttons, x, y = struct.unpack(">BHH", await reader.readexactly(5)) + self.events.append(PointerEvent(x, y, buttons)) + elif kind == 6: # ClientCutText + _, length = struct.unpack(">3sI", await reader.readexactly(7)) + await reader.readexactly(length) + else: + raise ValueError(f"unexpected RFB client message {kind}") + except (asyncio.IncompleteReadError, ConnectionError): + pass + finally: + writer.close() + self.sessions.discard(session) + + +@asynccontextmanager +async def fake_screen( + width: int = 64, + height: int = 48, + color: tuple[int, int, int] = (40, 120, 200), + *, + drop_on_update: int = 0, +) -> AsyncIterator[FakeScreen]: + """Serve a :class:`FakeScreen` on loopback for the duration of the block.""" + screen = FakeScreen(width, height, color, drop_on_update=drop_on_update) + server = await asyncio.start_server(screen._session, "127.0.0.1", 0) + screen.port = server.sockets[0].getsockname()[1] + async with server: + yield screen + # Read what a client sent before it hung up; 3.11's server exit does not wait. + await asyncio.gather(*screen.sessions) diff --git a/tests/support/services.py b/tests/support/services.py new file mode 100644 index 000000000..60b2ef171 --- /dev/null +++ b/tests/support/services.py @@ -0,0 +1,300 @@ +"""A local stand-in for every HUD service the SDK talks to. + +One HTTP and WebSocket server answers for the API, telemetry, inference gateway, +runtime gateway, training service and web app, each under its own path prefix. +Every request is recorded. Tests declare the replies they need with +:meth:`FakeServices.route`; anything undeclared answers 404, so a test notices +traffic it did not expect. + +The server runs on its own thread and event loop, so the SDK reaches it the +same way from an in-process rollout and from a ``hud`` subprocess. +""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import re +import socket +import threading +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any +from urllib.parse import parse_qs + +import uvicorn +from starlette.applications import Starlette +from starlette.responses import Response, StreamingResponse +from starlette.routing import Route, WebSocketRoute + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable, Iterable + + from starlette.requests import Request as StarletteRequest + from starlette.websockets import WebSocket + +SERVICE_ENV = { + "api": "HUD_API_URL", + "telemetry": "HUD_TELEMETRY_URL", + "gateway": "HUD_GATEWAY_URL", + "runtime": "HUD_RUNTIME_URL", + "rl": "HUD_RL_URL", + "web": "HUD_WEB_URL", +} + +METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"] + + +@dataclass(frozen=True) +class Request: + """One recorded request, addressed relative to its service's prefix.""" + + service: str + method: str + path: str + query: dict[str, list[str]] + headers: dict[str, str] + body: bytes + params: dict[str, str] = field(default_factory=dict) + + @property + def json(self) -> Any: + return json.loads(self.body) if self.body else None + + @property + def bearer(self) -> str | None: + value = self.headers.get("authorization", "") + return value.removeprefix("Bearer ") if value.startswith("Bearer ") else None + + +@dataclass +class Reply: + """What a route answers. ``stream`` sends server-sent events or other chunks as they come.""" + + status: int = 200 + json: Any = None + body: bytes | str | None = None + headers: dict[str, str] = field(default_factory=dict) + content_type: str | None = None + stream: Iterable[bytes] | None = None + delay: float = 0.0 + + +@dataclass +class _Route: + service: str + method: str + pattern: re.Pattern[str] + answer: Any + + +def _compile(path: str) -> re.Pattern[str]: + """``/trace/{id}`` matches one segment per name; ``{rest:path}`` matches the remainder.""" + regex = re.sub(r"\{(\w+):path\}", r"(?P<\1>.*)", path) + regex = re.sub(r"\{(\w+)\}", r"(?P<\1>[^/]+)", regex) + return re.compile(f"^{regex}$") + + +class FakeServices: + """The fake HUD backend. Use the ``services`` fixture rather than constructing one.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._routes: list[_Route] = [] + self._sockets: list[_Route] = [] + self._requests: list[Request] = [] + self._socket = socket.socket() + self._socket.bind(("127.0.0.1", 0)) + self.port: int = self._socket.getsockname()[1] + app = Starlette( + routes=[ + Route("/{service}/{rest:path}", self._http, methods=METHODS), + WebSocketRoute("/{service}/{rest:path}", self._websocket), + ] + ) + self._server = uvicorn.Server( + uvicorn.Config(app, log_level="error", lifespan="off", ws="websockets-sansio") + ) + self._thread = threading.Thread(target=self._serve, name="fake-hud-services", daemon=True) + + def _serve(self) -> None: + asyncio.run(self._server.serve(sockets=[self._socket])) + + def start(self) -> None: + self._thread.start() + while not self._server.started: + if not self._thread.is_alive(): + raise RuntimeError("fake HUD services failed to start") + threading.Event().wait(0.01) + + def stop(self) -> None: + self._server.should_exit = True + # A delayed reply a test walked away from must not hold up the end of the session. + self._server.force_exit = True + self._thread.join(timeout=10) + + def reset(self) -> None: + with self._lock: + self._routes.clear() + self._sockets.clear() + self._requests.clear() + + # ─── configuration ──────────────────────────────────────────────── + + def url(self, service: str) -> str: + if service not in SERVICE_ENV: + raise KeyError(f"unknown service {service!r}; one of {sorted(SERVICE_ENV)}") + return f"http://127.0.0.1:{self.port}/{service}" + + def env(self) -> dict[str, str]: + """``HUD_*_URL`` variables pointing every service here.""" + return {var: self.url(service) for service, var in SERVICE_ENV.items()} + + def route( + self, + service: str, + method: str, + path: str, + *replies: Reply, + handler: Callable[[Request], Reply | Awaitable[Reply]] | None = None, + json: Any = None, + status: int = 200, + ) -> None: + """Answer ``method path`` on ``service``. + + ``replies`` are served in order and the last repeats; ``handler`` computes a + reply from each :class:`Request`; with neither, the route answers ``json`` + with ``status``. Later routes win over earlier ones, so a test can override + a fixture's default. + """ + if handler is not None and replies: + raise ValueError("a route takes replies or a handler, not both") + answer = handler or _Sequence(list(replies) or [Reply(status=status, json=json)]) + with self._lock: + self._routes.insert(0, _Route(service, method.upper(), _compile(path), answer)) + + def websocket(self, service: str, path: str, handler: Callable[..., Awaitable[None]]) -> None: + """Serve a WebSocket at ``path``; ``handler(websocket, params)`` owns the session.""" + with self._lock: + self._sockets.insert(0, _Route(service, "WS", _compile(path), handler)) + + # ─── observation ────────────────────────────────────────────────── + + def requests( + self, service: str | None = None, method: str | None = None, path: str | None = None + ) -> list[Request]: + """Recorded requests, filtered; ``path`` accepts the same ``{name}`` patterns as routes.""" + pattern = _compile(path) if path is not None else None + with self._lock: + recorded = list(self._requests) + return [ + request + for request in recorded + if (service is None or request.service == service) + and (method is None or request.method == method.upper()) + and (pattern is None or pattern.match(request.path)) + ] + + def bodies(self, service: str, method: str, path: str) -> list[Any]: + return [request.json for request in self.requests(service, method, path)] + + # ─── serving ────────────────────────────────────────────────────── + + def _match( + self, routes: list[_Route], service: str, method: str, path: str + ) -> tuple[Any, dict[str, str]]: + with self._lock: + candidates = list(routes) + for route in candidates: + if ( + route.service == service + and route.method == method + and (found := route.pattern.match(path)) + ): + return route.answer, found.groupdict() + return None, {} + + async def _http(self, incoming: StarletteRequest) -> Response: + service = incoming.path_params["service"] + path = "/" + incoming.path_params["rest"] + answer, params = self._match(self._routes, service, incoming.method, path) + request = Request( + service=service, + method=incoming.method, + path=path, + query=parse_qs(incoming.url.query), + headers={key.lower(): value for key, value in incoming.headers.items()}, + body=await incoming.body(), + params=params, + ) + with self._lock: + self._requests.append(request) + if answer is None: + return Response( + json.dumps({"detail": f"no fake route for {incoming.method} {service}{path}"}), + status_code=404, + media_type="application/json", + ) + reply = answer(request) + if inspect.isawaitable(reply): + reply = await reply + if reply.delay: + await asyncio.sleep(reply.delay) + if reply.stream is not None: + return StreamingResponse( + _chunks(reply.stream), + status_code=reply.status, + headers=reply.headers, + media_type=reply.content_type or "text/event-stream", + ) + if reply.body is not None: + return Response( + reply.body, + status_code=reply.status, + headers=reply.headers, + media_type=reply.content_type or "application/octet-stream", + ) + return Response( + json.dumps(reply.json), + status_code=reply.status, + headers=reply.headers, + media_type="application/json", + ) + + async def _websocket(self, websocket: WebSocket) -> None: + service = websocket.path_params["service"] + path = "/" + websocket.path_params["rest"] + handler, params = self._match(self._sockets, service, "WS", path) + request = Request( + service=service, + method="WS", + path=path, + query=parse_qs(websocket.url.query), + headers={key.lower(): value for key, value in websocket.headers.items()}, + body=b"", + params=params, + ) + with self._lock: + self._requests.append(request) + if handler is None: + await websocket.close(code=4404) + return + await handler(websocket, params) + + +class _Sequence: + def __init__(self, replies: list[Reply]) -> None: + if not replies: + raise ValueError("a reply sequence needs at least one reply") + self._replies = replies + self._lock = threading.Lock() + + def __call__(self, request: Request) -> Reply: + del request + with self._lock: + return self._replies.pop(0) if len(self._replies) > 1 else self._replies[0] + + +async def _chunks(stream: Iterable[bytes]) -> AsyncIterator[bytes]: + for chunk in stream: + yield chunk diff --git a/tests/support/waiting.py b/tests/support/waiting.py new file mode 100644 index 000000000..add2a5fec --- /dev/null +++ b/tests/support/waiting.py @@ -0,0 +1,25 @@ +"""Waiting for what the SDK does in the background.""" + +from __future__ import annotations + +import asyncio +import inspect +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + +async def eventually( + condition: Callable[[], bool | Awaitable[bool]], *, within: float = 10.0 +) -> None: + """Poll until ``condition()`` holds; raise ``TimeoutError`` after ``within`` seconds. + + ``condition`` may be a coroutine function, for a check that has to await. + """ + async with asyncio.timeout(within): + while True: + held = condition() + if await held if inspect.isawaitable(held) else held: + return + await asyncio.sleep(0.01) diff --git a/tests/telemetry/__init__.py b/tests/telemetry/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/telemetry/test_file_tracking.py b/tests/telemetry/test_file_tracking.py new file mode 100644 index 000000000..7984e9553 --- /dev/null +++ b/tests/telemetry/test_file_tracking.py @@ -0,0 +1,190 @@ +"""File tracking in a trace: a rollout on a tracked workspace exports what the environment +set up and what the agent changed as ``filetracking`` spans.""" + +from __future__ import annotations + +import asyncio +import itertools +from typing import TYPE_CHECKING, Any + +import pytest + +from hud import Environment +from hud.agents.base import Agent +from hud.capabilities import Capability +from hud.environment.file_tracker import FileTracker, serve_file_tracking +from hud.eval import LocalRuntime, Task, rollout +from tests.support import spans + +if TYPE_CHECKING: + from pathlib import Path + + from hud.eval import Run + from tests.support import HudEnv + +SCHEMA = "hud.filetracking.v1" + + +@pytest.fixture(autouse=True) +def tracking_often(hud_env: HudEnv, tmp_path: Path) -> None: + hud_env.set( + HUD_TELEMETRY_ENABLED="1", + HUD_TELEMETRY_LOCAL_DIR=str(tmp_path / "spans"), + HUD_FILE_TRACKING_INTERVAL="0.05", + ) + + +class Writer(Agent): + """Writes ``files`` into the workspace, then waits long enough to be sampled.""" + + def __init__(self, root: Path, files: dict[str, bytes]) -> None: + super().__init__() + self.root = root + self.files = files + + async def __call__(self, run: Run) -> None: + for name, data in self.files.items(): + (self.root / name).write_bytes(data) + await asyncio.sleep(0.2) + run.trace.content = "done" + + +def tracked(root: Path, *, track_files: bool = True, seed: int = 0) -> Environment: + """A workspace whose task writes ``setup.txt``, and ``seed`` more files, as it starts.""" + env = Environment("tracked") + env.workspace(root, track_files=track_files) + + @env.template() + async def write(): + (root / "setup.txt").write_text("from setup\n") + for index in range(seed): + (root / f"seed-{index:04d}-{'x' * 40}.txt").write_text("seed\n") + yield "write your files" + yield 1.0 + + return env + + +def exported(run: Run) -> list[tuple[str, dict[str, Any]]]: + """The run's file-tracking spans, in order, as (name, payload).""" + return [ + (span["name"], span["attributes"]["hud.payload"]) + for span in spans(run.trace_id) + if span["attributes"].get("hud.schema") == SCHEMA + ] + + +async def test_a_tracked_rollout_exports_its_setup_a_snapshot_its_diffs_and_deliverables( + tmp_path: Path, +) -> None: + root = tmp_path / "ws" + files = {"notes.md": b"# notes\n", "report.pdf": b"%PDF-1.4 report"} + + run = await rollout( + Task(env="tracked", id="write"), Writer(root, files), runtime=LocalRuntime(tracked(root)) + ) + + tracking = exported(run) + payloads = dict(tracking) + assert [name for name, _ in itertools.groupby(name for name, _ in tracking)] == [ + "filetracking.setup", + "filetracking.snapshot", + "filetracking.diff", + "filetracking.capture", + ] + assert [p["path"] for p in payloads["filetracking.setup"]["patches"]] == ["setup.txt"] + assert sorted(p["path"] for p in payloads["filetracking.diff"]["patches"]) == [ + "notes.md", + "report.pdf", + ] + assert [f["path"] for f in payloads["filetracking.capture"]["files"]] == ["report.pdf"] + + +async def test_a_tracker_without_setup_diffs_starts_the_trace_from_a_snapshot( + tmp_path: Path, +) -> None: + root = tmp_path / "ws" + root.mkdir() + env = Environment("tracked") + + @env.initialize + async def track() -> None: + tracker = FileTracker(root) + tracker.take_baseline() + server = await serve_file_tracking(tracker) + port = server.sockets[0].getsockname()[1] + env.add_capability( + Capability( + name="filetracking", protocol="filetracking/1", url=f"tcp://127.0.0.1:{port}" + ) + ) + + @env.template() + async def write(): + yield "write your files" + yield 1.0 + + run = await write().run(Writer(root, {"notes.md": b"# notes\n"})) + + names = [name for name, _ in itertools.groupby(name for name, _ in exported(run.runs[0]))] + assert names[:2] == ["filetracking.snapshot", "filetracking.diff"] + + +async def test_a_deliverable_too_large_to_capture_is_still_reported_as_changed( + tmp_path: Path, +) -> None: + root = tmp_path / "ws" + huge = {"huge.pdf": b"%PDF" + b"0" * (6 * 1024 * 1024)} + + run = await rollout( + Task(env="tracked", id="write"), Writer(root, huge), runtime=LocalRuntime(tracked(root)) + ) + + payloads = dict(exported(run)) + assert [p["path"] for p in payloads["filetracking.diff"]["patches"]] == ["huge.pdf"] + capture = payloads["filetracking.capture"] + assert (capture["files"], capture["files_skipped"]) == ([], 1) + + +async def test_a_large_workspace_snapshot_arrives_whole(tmp_path: Path) -> None: + root = tmp_path / "ws" + + run = await rollout( + Task(env="tracked", id="write"), + Writer(root, {}), + runtime=LocalRuntime(tracked(root, seed=1500)), + ) + + payloads = dict(exported(run)) + assert len(payloads["filetracking.snapshot"]["files"]) == 1501 + assert len(payloads["filetracking.setup"]["patches"]) == 1501 + + +async def test_an_untracked_workspace_exports_no_file_tracking(tmp_path: Path) -> None: + root = tmp_path / "ws" + + run = await rollout( + Task(env="tracked", id="write"), + Writer(root, {"notes.md": b"# notes\n"}), + runtime=LocalRuntime(tracked(root, track_files=False)), + ) + + assert (run.reward, exported(run)) == (1.0, []) + + +async def test_a_tracker_that_cannot_be_reached_exports_nothing_and_the_run_goes_on( + tmp_path: Path, +) -> None: + env = Environment("tracked") + env.add_capability( + Capability(name="filetracking", protocol="filetracking/1", url="tcp://127.0.0.1:9") + ) + + @env.template() + async def write(): + yield "write your files" + yield 1.0 + + (run,) = (await write().run(Writer(tmp_path, {"notes.md": b"# notes\n"}))).runs + + assert (run.reward, exported(run)) == (1.0, []) diff --git a/tests/telemetry/test_parent_trace.py b/tests/telemetry/test_parent_trace.py new file mode 100644 index 000000000..6a445c130 --- /dev/null +++ b/tests/telemetry/test_parent_trace.py @@ -0,0 +1,97 @@ +"""A rollout started inside another trace names that trace as its parent to everything it +reports to: the platform's trace record, the model gateway, and a hosted submission. Its +own trace is never its parent.""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING + +import pytest + +from hud import Environment +from hud.agents.openai_compatible import OpenAIChatAgent +from hud.agents.types import OpenAIChatConfig +from hud.eval import HostedRuntime, LocalRuntime, Task, rollout +from hud.telemetry.context import set_trace_context +from tests.fixtures.agents import ScriptedAgent +from tests.support import say + +if TYPE_CHECKING: + from tests.support import FakeServices, HudEnv, Models + +TRACE_ID = "b" * 32 +AMBIENTS = [("a" * 32, "a" * 32), (str(uuid.UUID(TRACE_ID)), None)] +AMBIENT_IDS = ["another-trace-is-the-parent", "its-own-trace-is-not-its-parent"] + + +def checked() -> Environment: + env = Environment("checked") + + @env.template() + async def check(): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + return env + + +@pytest.mark.parametrize(("ambient", "parent"), AMBIENTS, ids=AMBIENT_IDS) +async def test_the_trace_record_names_the_parent_trace( + ambient: str, parent: str | None, reporting: FakeServices +) -> None: + with set_trace_context(ambient): + await rollout( + Task(env="checked", id="check"), + ScriptedAgent("yes"), + runtime=LocalRuntime(checked()), + trace_id=TRACE_ID, + ) + + (entered,) = reporting.bodies("api", "POST", "/v2/trace/{id}/enter") + assert entered.get("parent_trace_id") == parent + + +@pytest.mark.parametrize(("ambient", "parent"), AMBIENTS, ids=AMBIENT_IDS) +async def test_the_model_gateway_is_told_the_parent_trace( + ambient: str, parent: str | None, models: Models, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([say("yes")]) + + with set_trace_context(ambient): + await rollout( + Task(env="checked", id="check"), + OpenAIChatAgent(OpenAIChatConfig(model="m")), + runtime=LocalRuntime(checked()), + trace_id=TRACE_ID, + ) + + (request,) = models.requests() + sent = request.headers.get("x-hud-parent-trace-id") + assert (uuid.UUID(sent).hex if sent else None) == parent + + +@pytest.mark.parametrize( + ("ambient", "parent"), + [*AMBIENTS, ("external-run-id", None)], + ids=[*AMBIENT_IDS, "an-id-that-is-not-a-trace-is-no-parent"], +) +async def test_a_hosted_submission_names_the_parent_trace( + ambient: str, parent: str | None, services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + services.route("api", "POST", "/v2/rollouts/submit", json={}) + services.route("api", "GET", "/v2/trace/{id}", json={"status": "completed", "reward": 1.0}) + agent = OpenAIChatAgent( + OpenAIChatConfig(model="m", api_key="secret", base_url="http://model.test") + ) + + with set_trace_context(ambient): + await HostedRuntime(poll_interval=0).run( + Task(env="checked", id="check"), agent, job_id="c" * 32, trace_id=TRACE_ID + ) + + (submitted,) = services.bodies("api", "POST", "/v2/rollouts/submit") + sent = submitted.get("parent_trace_id") + assert (uuid.UUID(sent).hex if sent else None) == parent diff --git a/tests/telemetry/test_platform_records.py b/tests/telemetry/test_platform_records.py new file mode 100644 index 000000000..fe4d11224 --- /dev/null +++ b/tests/telemetry/test_platform_records.py @@ -0,0 +1,269 @@ +"""What the platform records about an execution: one job for it, a trace entered for each +rollout under that job, and each rollout's outcome when it exits, sent only while +telemetry is on with an API key.""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING + +import pytest + +from hud import Environment +from hud.agents.base import Agent +from hud.agents.types import AgentConfig +from hud.eval import Job, LocalRuntime, Runtime, Task, Taskset, rollout +from hud.types import Step +from tests.fixtures.agents import ScriptedAgent +from tests.support import eventually + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from hud.eval import Run + from tests.support import FakeServices, HudEnv + +JOB_ENTER = "/v2/trace/job/{id}/enter" +TRACE_ENTER = "/v2/trace/{id}/enter" +TRACE_EXIT = "/v2/trace/{id}/exit" + + +def checked() -> Environment: + env = Environment("checked") + + @env.template() + async def check(n: int = 0): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + return env + + +async def test_a_taskset_reports_one_job_and_enters_each_rollout_under_it( + reporting: FakeServices, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level(logging.INFO, logger="hud.eval.job") + rows = [Task(env="checked", id="check", args={"n": n}) for n in (1, 2)] + + class Sampling(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "yes" + + agent = Sampling(AgentConfig(model="claude-test")) + job = await Taskset("pair", rows, taskset_id="ts-1").run( + agent, runtime=LocalRuntime(checked()), group=2 + ) + + (registered,) = reporting.requests("api", "POST", JOB_ENTER) + entered = {r.path.split("/")[3]: r.json for r in reporting.requests("api", "POST", TRACE_ENTER)} + assert registered.path == f"/v2/trace/job/{job.id}/enter" + assert registered.json == {"name": "pair (2 tasks) (2 times)", "group": 2, "taskset_id": "ts-1"} + assert f"job: {reporting.url('web')}/jobs/{uuid.UUID(job.id)}" in caplog.messages + assert set(entered) == {run.trace_id for run in job.runs} + assert {(e["job_id"], e["model"]) for e in entered.values()} == {(job.id, "claude-test")} + assert sorted((e["task_slug"], e["group_id"]) for e in entered.values()) == sorted( + (run.slug, run.group_id) for run in job.runs + ) + + +async def test_a_lone_rollout_reports_a_job_of_one_named_for_its_task( + reporting: FakeServices, +) -> None: + run = await rollout( + Task(env="checked", id="check"), ScriptedAgent("yes"), runtime=LocalRuntime(checked()) + ) + + (registered,) = reporting.requests("api", "POST", JOB_ENTER) + assert registered.json == {"name": "check", "group": 1} + assert registered.path == f"/v2/trace/job/{run.job_id}/enter" + + +async def test_a_rollout_given_a_job_reports_none_of_its_own(reporting: FakeServices) -> None: + job_id = "a" * 32 + + await rollout( + Task(env="checked", id="check"), + ScriptedAgent("yes"), + runtime=LocalRuntime(checked()), + job_id=job_id, + ) + + (entered,) = reporting.bodies("api", "POST", TRACE_ENTER) + assert (reporting.requests("api", "POST", JOB_ENTER), entered["job_id"]) == ([], job_id) + + +async def test_an_open_job_is_reported_once_for_every_call_it_spans( + reporting: FakeServices, +) -> None: + env = checked() + job = await Job.start("training", group=2) + + await Task(env="checked", id="check").run( + ScriptedAgent("yes"), runtime=LocalRuntime(env), job=job + ) + await Task(env="checked", id="check").run( + ScriptedAgent("yes"), runtime=LocalRuntime(env), job=job + ) + + assert reporting.bodies("api", "POST", JOB_ENTER) == [{"name": "training", "group": 2}] + assert {e["job_id"] for e in reporting.bodies("api", "POST", TRACE_ENTER)} == {job.id} + + +async def test_a_graded_rollout_reports_its_reward_evaluation_and_metadata( + reporting: FakeServices, +) -> None: + env = Environment("graded") + + @env.template() + async def check(): + yield "go" + yield { + "score": 0.5, + "subscores": [{"name": "format", "weight": 1.0, "value": 0.5, "info": {"seen": True}}], + } + + class Annotating(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "x" + run.trace.extra["turns"] = 3 + + await check().run(Annotating()) + + (exited,) = reporting.bodies("api", "POST", TRACE_EXIT) + assert exited == { + "status": "completed", + "reward": 0.5, + "evaluation_result": { + "score": 0.5, + "subscores": [{"name": "format", "weight": 1.0, "value": 0.5, "children": None}], + }, + "metadata": {"turns": 3}, + } + + +async def test_an_error_the_run_recovered_from_stays_off_its_report( + reporting: FakeServices, +) -> None: + class Recovers(Agent): + async def __call__(self, run: Run) -> None: + run.record(Step(source="system", error="model busy; asked again")) + run.trace.content = "yes" + + await Task(env="checked", id="check").run(Recovers(), runtime=LocalRuntime(checked())) + + (exited,) = reporting.bodies("api", "POST", TRACE_EXIT) + assert (exited["status"], "error" in exited) == ("completed", False) + + +async def test_a_rollout_whose_agent_raised_reports_its_error_beside_its_grade( + reporting: FakeServices, +) -> None: + class RaisesAfter(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "yes" + raise RuntimeError("agent exploded") + + await Task(env="checked", id="check").run(RaisesAfter(), runtime=LocalRuntime(checked())) + + (exited,) = reporting.bodies("api", "POST", TRACE_EXIT) + assert (exited["status"], exited["reward"], exited["error"]) == ( + "error", + 1.0, + "[agent loop] RuntimeError: agent exploded", + ) + + +async def test_a_rollout_whose_grading_failed_reports_no_evaluation( + reporting: FakeServices, +) -> None: + env = Environment("broken") + + @env.template() + async def grade_raises(): + yield "go" + raise RuntimeError("grader exploded") + yield 0.0 + + await grade_raises().run(ScriptedAgent("x")) + + (exited,) = reporting.bodies("api", "POST", TRACE_EXIT) + assert (exited["status"], exited["reward"], "evaluation_result" in exited) == ( + "error", + 0.0, + False, + ) + assert exited["error"].startswith("[grading] ") + + +async def test_a_rollout_that_never_launched_reports_where_it_failed( + reporting: FakeServices, +) -> None: + @asynccontextmanager + async def unavailable(task: Task) -> AsyncIterator[Runtime]: + raise RuntimeError("no substrate") + yield Runtime("tcp://127.0.0.1:1") + + await rollout(Task(env="checked", id="check"), ScriptedAgent("yes"), runtime=unavailable) + + (exited,) = reporting.bodies("api", "POST", TRACE_EXIT) + assert exited == { + "status": "error", + "reward": 0.0, + "error": "[provisioning] RuntimeError: no substrate", + } + + +async def test_a_timed_out_rollout_reports_why_it_stopped(reporting: FakeServices) -> None: + class Waits(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "yes" + await asyncio.sleep(30) + + await Task(env="checked", id="check").run( + Waits(AgentConfig(timeout_seconds=0.2)), runtime=LocalRuntime(checked()) + ) + + (exited,) = reporting.bodies("api", "POST", TRACE_EXIT) + assert (exited["status"], exited["stop_reason"], exited["reward"]) == ("error", "timeout", 1.0) + + +@pytest.mark.parametrize( + "variables", + [{"HUD_API_KEY": None}, {"HUD_TELEMETRY_ENABLED": "0"}], + ids=["telemetry-on-without-a-key", "a-key-with-telemetry-off"], +) +async def test_nothing_is_reported_unless_telemetry_is_on_with_a_key( + variables: dict[str, str | None], reporting: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(**variables) + + job = await Task(env="checked", id="check").run( + ScriptedAgent("yes"), runtime=LocalRuntime(checked()) + ) + + assert (job.reward, reporting.requests()) == (1.0, []) + + +async def test_a_cancelled_rollout_reports_its_start_but_no_outcome( + reporting: FakeServices, +) -> None: + prompted = asyncio.Event() + + class Waits(Agent): + async def __call__(self, run: Run) -> None: + prompted.set() + await asyncio.sleep(30) + + pending = asyncio.create_task( + rollout(Task(env="checked", id="check"), Waits(), runtime=LocalRuntime(checked())) + ) + await eventually(prompted.is_set) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + + assert len(reporting.requests("api", "POST", TRACE_ENTER)) == 1 + assert reporting.requests("api", "POST", TRACE_EXIT) == [] diff --git a/tests/telemetry/test_sink_failures.py b/tests/telemetry/test_sink_failures.py new file mode 100644 index 000000000..0c9dd79e0 --- /dev/null +++ b/tests/telemetry/test_sink_failures.py @@ -0,0 +1,100 @@ +"""A place a run reports to that fails never fails the run: the local span directory, the +HUD telemetry service, and the platform's job and trace records. A report retried until +it lands arrives intact; one given up on leaves a warning.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import pytest + +from hud import Environment +from hud.eval import LocalRuntime, Task +from hud.telemetry import flush +from tests.fixtures.agents import ScriptedAgent +from tests.support import Reply + +if TYPE_CHECKING: + from pathlib import Path + + from tests.support import FakeServices, HudEnv + + +def checked() -> Environment: + env = Environment("checked") + + @env.template() + async def check(): + answer = yield "say yes" + yield 1.0 if answer == "yes" else 0.0 + + return env + + +async def test_a_span_directory_that_cannot_be_created_does_not_fail_the_run( + reporting: FakeServices, hud_env: HudEnv, tmp_path: Path +) -> None: + (tmp_path / "a-file").write_text("") + hud_env.set(HUD_TELEMETRY_LOCAL_DIR=str(tmp_path / "a-file" / "spans")) + + job = await Task(env="checked", id="check").run( + ScriptedAgent("yes"), runtime=LocalRuntime(checked()) + ) + + assert (job.reward, job.runs[0].trace.status) == (1.0, "completed") + + +async def test_a_telemetry_service_that_is_down_does_not_fail_the_run( + reporting: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_TELEMETRY_URL="http://127.0.0.1:9") + + job = await Task(env="checked", id="check").run( + ScriptedAgent("yes"), runtime=LocalRuntime(checked()) + ) + assert flush(timeout=10) + + assert (job.reward, job.runs[0].trace.status) == (1.0, "completed") + + +@pytest.mark.parametrize( + ("service", "path", "statuses", "attempts", "gave_up"), + [ + ("telemetry", "/trace/{id}/telemetry-upload", [503, 503, 200], 3, False), + ("api", "/v2/trace/job/{id}/enter", [500, 200], 2, False), + ("api", "/v2/trace/job/{id}/enter", [500], 2, True), + ], + ids=[ + "span-uploads-answered-503-until-they-land", + "a-job-report-rejected-once", + "a-job-report-rejected-twice", + ], +) +async def test_a_rejected_report_is_retried_intact_and_never_fails_the_run( + service: str, + path: str, + statuses: list[int], + attempts: int, + gave_up: bool, + reporting: FakeServices, + caplog: pytest.LogCaptureFixture, +) -> None: + reporting.route(service, "POST", path, *(Reply(status=code, json={}) for code in statuses)) + + job = await Task(env="checked", id="check").run( + ScriptedAgent("yes"), runtime=LocalRuntime(checked()) + ) + assert flush(timeout=10) + + sent = reporting.requests(service, "POST", path) + warnings = [ + r.getMessage() + for r in caplog.records + if r.name == "hud.eval.job" and r.levelno == logging.WARNING + ] + assert (job.reward, job.runs[0].trace.status) == (1.0, "completed") + assert (len(sent), len({str(request.json) for request in sent})) == (attempts, 1) + assert [w.startswith("platform report /trace/job/") and "500" in w for w in warnings] == ( + [True] if gave_up else [] + ) diff --git a/tests/telemetry/test_spans.py b/tests/telemetry/test_spans.py new file mode 100644 index 000000000..4e03ea370 --- /dev/null +++ b/tests/telemetry/test_spans.py @@ -0,0 +1,313 @@ +"""Where a run's spans go: local span files and the HUD telemetry service, including +``@hud.instrument`` spans, and when an upload is flushed.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest +from dirty_equals import IsStr + +from hud import Environment, instrument +from hud.agents.base import Agent +from hud.agents.openai_compatible import OpenAIChatAgent +from hud.agents.types import OpenAIChatConfig +from hud.eval import LocalRuntime, Task, Taskset, rollout +from hud.telemetry import flush +from tests.fixtures.agents import ScriptedAgent +from tests.support import Reply, call, say, spans, steps + +if TYPE_CHECKING: + from hud.eval import Run + from tests.support import FakeServices, HudEnv, Models + +ISO = IsStr(regex=r"\d{4}-\d\d-\d\dT.*") +UPLOAD = "/trace/{id}/telemetry-upload" + + +def spans_env() -> Environment: + env = Environment("spans") + + @env.template() + async def answer(): + reply = yield "Reply with ok." + yield 1.0 if reply == "ok" else 0.0 + + return env + + +TASK = Task(env="spans", id="answer") + + +@pytest.mark.parametrize( + ("uploads", "local_dir", "written_to", "uploaded"), + [ + pytest.param("0", None, "home", False, id="uploads-off"), + pytest.param("0", "spans", "spans", False, id="uploads-off-with-dir"), + pytest.param("1", "spans", "spans", True, id="uploads-on-with-dir"), + pytest.param("1", None, None, True, id="uploads-on"), + ], +) +async def test_spans_are_written_where_the_configuration_says( + reporting: FakeServices, + hud_env: HudEnv, + tmp_path: Path, + uploads: str, + local_dir: str | None, + written_to: str | None, + uploaded: bool, +) -> None: + hud_env.set( + HUD_TELEMETRY_ENABLED=uploads, + HUD_TELEMETRY_LOCAL_DIR=str(tmp_path / local_dir) if local_dir else None, + ) + directories = {"home": hud_env.home / ".hud" / "spans", "spans": tmp_path / "spans"} + + job = await Taskset("t", [TASK]).run(ScriptedAgent("ok"), runtime=LocalRuntime(spans_env())) + + (run,) = job.runs + written = {name: spans(run.trace_id, path) != [] for name, path in directories.items()} + assert written == {name: name == written_to for name in directories} + assert flush(timeout=10) + assert bool(reporting.requests("telemetry", "POST", UPLOAD)) is uploaded + + +async def test_step_spans_carry_the_step_payload_and_timing() -> None: + run = await rollout(TASK, ScriptedAgent("ok"), runtime=LocalRuntime(spans_env())) + + exported = [span for span in spans(run.trace_id) if span["name"].startswith("step.")] + assert all(span["end_time"] == ISO for span in exported) + assert steps(run.trace_id)[1]["messages"][0]["content"]["text"] == "Reply with ok." + + +class Probe: + """An agent-side helper whose methods are instrumented.""" + + @instrument(name="probe.lookup") + async def lookup(self, items: list[int], limit: int = 3) -> dict[str, Any]: + return {"found": items[:limit]} + + @instrument(name="probe.quiet", record_args=False, record_result=False) + def quiet(self, secret: str) -> str: + return secret + + @instrument(name="probe.fail") + def fail(self, reason: str) -> None: + raise ValueError(reason) + + @instrument(name="probe.opaque") + def opaque(self, value: object) -> object: + return value + + +class Unprintable: + def __repr__(self) -> str: + raise RuntimeError("no repr") + + +class ProbingAgent(Agent): + """Calls each instrumented helper once, then answers.""" + + async def __call__(self, run: Run) -> None: + probe = Probe() + await probe.lookup(list(range(20))) + probe.quiet("hunter2") + with pytest.raises(ValueError, match="bad input"): + probe.fail("bad input") + probe.opaque(Unprintable()) + run.trace.content = "ok" + + +async def test_instrumented_calls_inside_a_run_record_their_arguments_and_outcome() -> None: + run = await rollout(TASK, ProbingAgent(), runtime=LocalRuntime(spans_env())) + + assert run.reward == 1.0 + probes = { + span["name"]: { + "status": span["status_code"], + "message": span["status_message"], + "events": {event["name"]: event["attributes"] for event in span["events"]}, + } + for span in spans(run.trace_id) + if span["name"].startswith("probe.") + } + assert probes == { + "probe.lookup": { + "status": "OK", + "message": None, + "events": { + "hud.request": { + "hud.payload": {"items": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "limit": 3} + }, + "hud.result": {"hud.payload": {"found": [0, 1, 2]}}, + }, + }, + "probe.quiet": {"status": "OK", "message": None, "events": {}}, + "probe.fail": { + "status": "ERROR", + "message": "ValueError: bad input", + "events": { + "hud.request": {"hud.payload": {"reason": "bad input"}}, + "exception": {"exception.message": "ValueError: bad input"}, + }, + }, + "probe.opaque": { + "status": "OK", + "message": None, + "events": { + "hud.request": {"hud.payload": {"value": ""}}, + "hud.result": {"hud.payload": ""}, + }, + }, + } + + +async def test_instrumented_calls_outside_a_run_export_nothing(hud_env: HudEnv) -> None: + probe = Probe() + + assert await probe.lookup([1, 2]) == {"found": [1, 2]} + assert not (hud_env.home / ".hud" / "spans").exists() + + +# ─── the HUD telemetry service ───────────────────────────────────────── + +ROOT = Path(__file__).parents[2] + + +def upload_env() -> Environment: + env = Environment("upload") + + @env.template() + async def answer(broken: bool = False): + reply = yield "Reply with ok." + if broken: + raise RuntimeError("grader broke") + yield 1.0 if reply == "ok" else 0.0 + + return env + + +def step_rows(spans: list[dict[str, Any]]) -> list[tuple[str, str, int]]: + steps = [span for span in spans if span["attributes"].get("hud.schema") == "hud.step.v1"] + return [ + (span["name"], span["status_code"], span["attributes"]["hud.payload"]["step_id"]) + for span in sorted(steps, key=lambda span: span["attributes"]["hud.payload"]["step_id"]) + ] + + +@pytest.mark.parametrize( + ("broken", "reward", "rows"), + [ + pytest.param( + False, + 1.0, + [ + ("step.task", "OK", 1), + ("step.user", "OK", 2), + ("step.agent", "OK", 3), + ("step.tool", "OK", 4), + ("step.agent", "OK", 5), + ("step.task", "OK", 6), + ], + id="graded", + ), + pytest.param( + True, + 0.0, + [ + ("step.task", "OK", 1), + ("step.user", "OK", 2), + ("step.agent", "OK", 3), + ("step.tool", "OK", 4), + ("step.agent", "OK", 5), + ("step.system", "ERROR", 6), + ], + id="grading-error", + ), + ], +) +async def test_every_rollout_step_is_uploaded_to_its_own_trace( + reporting: FakeServices, + models: Models, + broken: bool, + reward: float, + rows: list[tuple[str, str, int]], +) -> None: + models.script([call("missing_tool", x=1), say("ok")]) + agent = OpenAIChatAgent(OpenAIChatConfig(model="scripted", max_steps=4)) + task = Task(env="upload", id="answer", args={"broken": broken}) + + job = await Taskset("t", [task]).run(agent, runtime=LocalRuntime(upload_env()), group=2) + + assert [run.reward for run in job.runs] == [reward, reward] + assert flush(timeout=10) + uploaded: dict[str, list[dict[str, Any]]] = {} + for request in reporting.requests("telemetry", "POST", UPLOAD): + uploaded.setdefault(request.path.split("/")[2].replace("-", ""), []).extend( + request.json["telemetry"] + ) + assert sorted(uploaded) == sorted(str(run.trace_id).replace("-", "") for run in job.runs) + for trace_id, sent in uploaded.items(): + assert {span["attributes"]["hud.task_run_id"].replace("-", "") for span in sent} == { + trace_id + } + assert step_rows(sent) == rows + + +async def test_flush_returns_false_at_its_deadline_while_an_upload_hangs( + reporting: FakeServices, +) -> None: + reporting.route("telemetry", "POST", UPLOAD, Reply(json={}, delay=2.0)) + + run: Run = await rollout( + Task(env="upload", id="answer"), ScriptedAgent("ok"), runtime=LocalRuntime(upload_env()) + ) + + assert run.reward == 1.0 + assert flush(timeout=0.2) is False + assert flush(timeout=30) is True + assert reporting.requests("telemetry", "POST", UPLOAD) != [] + + +def test_spans_still_queued_at_interpreter_exit_are_uploaded( + reporting: FakeServices, tmp_path: Path +) -> None: + script = tmp_path / "rollout_then_exit.py" + script.write_text( + textwrap.dedent( + """ + import asyncio + from hud import Environment + from hud.eval import LocalRuntime, Task, rollout + from tests.fixtures.agents import ScriptedAgent + + env = Environment("upload") + + @env.template() + async def answer(): + reply = yield "Reply with ok." + yield 1.0 if reply == "ok" else 0.0 + + task = Task(env="upload", id="answer") + run = asyncio.run(rollout(task, ScriptedAgent("ok"), runtime=LocalRuntime(env))) + print(run.trace_id) + """ + ) + ) + + completed = subprocess.run( + [sys.executable, str(script)], + capture_output=True, + text=True, + timeout=120, + check=True, + cwd=ROOT, + ) + + trace_id = completed.stdout.strip().replace("-", "") + uploaded = {r.path.split("/")[2].replace("-", "") for r in reporting.requests("telemetry")} + assert uploaded == {trace_id} diff --git a/tests/test_import.py b/tests/test_import.py new file mode 100644 index 000000000..31cd0b608 --- /dev/null +++ b/tests/test_import.py @@ -0,0 +1,94 @@ +"""Importing the package: where unix sockets do not exist, and what warnings it leaves visible. + +Both are settled at import time, by the platform the standard library and the +dependencies see at their first import and by the order warning filters are +installed in, so each check runs in a fresh interpreter. + +Both routes out of a workspace are served on unix sockets, through a +``socketserver.ThreadingUnixStreamServer`` subclass, and a session runs on a pty. +Neither exists on Windows, so ``import hud`` must work there for the commands that +never go near a workspace. The interpreter imports the package once as the +platform it really is, then again with only the package's own modules dropped, +the platform reported as Windows, and the unix pieces taken away. + +``fastmcp`` imports ``authlib.jose``, which emits an ``AuthlibDeprecationWarning`` +on every CLI launch that the user can never act on. ``import hud`` hides that one +warning and no other, even with every warning shown; the warning is emitted the +way ``authlib.jose`` emits it, so the check holds whichever authlib is installed. +""" + +from __future__ import annotations + +import subprocess +import sys + +import pytest + +IMPORT_AS_WINDOWS = """ +import socketserver +import sys + +import hud + +for name in [n for n in sys.modules if n == "hud" or n.startswith("hud.")]: + del sys.modules[name] + +sys.platform = "win32" +sys.modules["pty"] = None # a Windows interpreter has no pty module to import +socketserver.__dict__.pop("ThreadingUnixStreamServer", None) # nor this server + +import hud + +print("IMPORTED") +""" + +SHOW_WARNING = """ +import sys +import warnings + +shown = [] +warnings.simplefilter("always") +warnings.showwarning = lambda message, *args, **kwargs: shown.append(str(message)) + +import hud +from authlib.deprecate import AuthlibDeprecationWarning, deprecate + +category, message = sys.argv[1:] +if category == "jose": + deprecate(message, version="2.0.0") +else: + warnings.warn({"authlib": AuthlibDeprecationWarning, "deprecation": DeprecationWarning, + "user": UserWarning}[category](message)) +print(shown) +""" + + +def run(code: str, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-c", code, *args], capture_output=True, text=True, timeout=120 + ) + + +def test_the_package_imports_where_unix_sockets_do_not_exist() -> None: + result = run(IMPORT_AS_WINDOWS) + + assert result.returncode == 0, f"stdout={result.stdout!r}\nstderr={result.stderr!r}" + assert "IMPORTED" in result.stdout + + +@pytest.mark.parametrize( + ("category", "message", "shown"), + [ + ("jose", "authlib.jose module is deprecated, please use joserfc instead.", False), + ("authlib", "authlib.oauth2 client is deprecated", True), + ("deprecation", "an unrelated deprecation", True), + ("user", "an unrelated user warning", True), + ], +) +def test_importing_hud_hides_only_the_authlib_jose_deprecation( + category: str, message: str, shown: bool +) -> None: + result = run(SHOW_WARNING, category, message) + + assert result.returncode == 0, result.stderr + assert (message in result.stdout) is shown diff --git a/hud/train/tests/__init__.py b/tests/train/__init__.py similarity index 100% rename from hud/train/tests/__init__.py rename to tests/train/__init__.py diff --git a/tests/train/test_training_batches.py b/tests/train/test_training_batches.py new file mode 100644 index 000000000..9258dba77 --- /dev/null +++ b/tests/train/test_training_batches.py @@ -0,0 +1,251 @@ +"""What a training batch sends to the HUD training service, and what it refuses to send.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +import pytest +import torch +from dirty_equals import IsStr + +from hud import Environment +from hud.agents import OpenAIChatAgent +from hud.agents.types import OpenAIChatConfig +from hud.eval import LocalRuntime, Run, Task, Taskset +from hud.train import DatumTensors, TrainingClient +from hud.utils.exceptions import HudRequestError +from tests.fixtures.agents import ScriptedAgent +from tests.support import Reply, Turn + +if TYPE_CHECKING: + from hud.agents.base import Agent + from tests.support import Models, Platform + +MODEL_ID = "00000000-0000-4000-8000-000000000001" +TRAIN = f"/v1/models/{MODEL_ID}/train" +TRACE_ID = IsStr(regex=r"[0-9a-f]{32}") + + +def _env() -> Environment: + env = Environment("train") + + @env.template() + async def answer(target: str = "ok"): + reply = yield "Reply with ok." + yield 1.0 if reply == target else 0.0 + + return env + + +@pytest.fixture +def trainable(platform: Platform) -> Platform: + """The platform, with ``owner/model`` in its catalog.""" + platform.add_model(id=MODEL_ID, model_name="owner/model") + return platform + + +def sampled(text: str) -> Turn: + """A reply from a trainable model: it carries its token ids and logprobs.""" + logprob = {"token": text, "logprob": -0.5, "bytes": [], "top_logprobs": []} + return Turn( + text=text, + extra={"prompt_token_ids": [1, 2, 3], "token_ids": [7], "logprobs": {"content": [logprob]}}, + ) + + +async def _runs(agent: Agent, *, tasks: list[Task] | None = None, group: int = 2) -> list[Run]: + job = await Taskset("t", tasks or [Task(env="train", id="answer")]).run( + agent, runtime=LocalRuntime(_env()), group=group, max_concurrent=1 + ) + return job.runs + + +async def test_a_step_sends_the_sampled_runs_then_one_optimizer_step( + trainable: Platform, models: Models +) -> None: + models.queue([sampled("ok"), sampled("no")]) + agent = OpenAIChatAgent( + OpenAIChatConfig( + model="owner/model", + max_steps=1, + completion_kwargs={"extra_body": {"return_token_ids": True}}, + ) + ) + runs = await _runs(agent) + + await TrainingClient("owner/model").step(runs, learning_rate=1e-5, group_size=2) + + training = trainable.training[MODEL_ID] + (sent,) = training.batches + sample = {"prompt_token_ids": [1, 2, 3], "output_token_ids": [7], "output_logprobs": [-0.5]} + assert [(i["samples"], i["reward"], i["trace_id"]) for i in sent["inputs"]] == [ + ([sample], 1.0, TRACE_ID), + ([sample], 0.0, TRACE_ID), + ] + assert [step["learning_rate"] for step in training.steps] == [1e-5] + + +async def test_runs_without_token_samples_train_by_trace_id(trainable: Platform) -> None: + runs = await _runs(ScriptedAgent("ok")) + + await TrainingClient("owner/model").forward_backward(runs, group_size=2) + + (batch,) = trainable.training[MODEL_ID].batches + assert batch["inputs"] == [str(run.trace_id) for run in runs] + + +async def test_the_model_slug_resolves_once_with_the_clients_own_key( + trainable: Platform, +) -> None: + runs = await _runs(ScriptedAgent("ok")) + trainable.keys = {"team-key"} + client = TrainingClient("owner/model", api_key="team-key") + + await client.forward_backward(runs, group_size=2) + await client.optim_step(learning_rate=1e-5) + + training = trainable.training[MODEL_ID] + assert (len(training.batches), len(training.steps)) == (1, 1) + assert len(trainable.services.requests("api", "GET", "/v2/models")) == 1 + + +async def test_an_optimizer_step_is_never_retried(trainable: Platform) -> None: + trainable.services.route("rl", "POST", f"{TRAIN}/optim-step", Reply(status=503)) + + with pytest.raises(HudRequestError) as raised: + await TrainingClient(MODEL_ID).optim_step(learning_rate=1e-5) + + assert raised.value.status_code == 503 + assert len(trainable.services.requests("rl", "POST", f"{TRAIN}/optim-step")) == 1 + + +async def test_a_custom_loss_sends_its_gradients_to_backward(trainable: Platform) -> None: + runs = await _runs(ScriptedAgent("ok")) + trainable.services.route( + "rl", + "POST", + f"{TRAIN}/forward", + json={ + "forward_id": "fwd-1", + "data": [ + { + "reward": reward, + "traj_idx": index, + "logprobs": [-0.5, -1.0], + "sampling_logprobs": [-0.5, -1.0], + "mask": [1.0, 1.0], + } + for index, reward in enumerate([1.0, 0.5]) + ], + }, + ) + + def loss( + data: list[DatumTensors], logprobs: list[torch.Tensor] + ) -> tuple[torch.Tensor, dict[str, float]]: + total = sum(-datum.reward * lp.sum() for datum, lp in zip(data, logprobs, strict=True)) + assert isinstance(total, torch.Tensor) + return total, {"loss": total.item()} + + await TrainingClient(MODEL_ID).forward_backward_custom(runs, loss, group_size=2) + + (forward,) = trainable.services.bodies("rl", "POST", f"{TRAIN}/forward") + (backward,) = trainable.training[MODEL_ID].batches + assert forward["inputs"] == [str(run.trace_id) for run in runs] + assert backward == { + "forward_id": "fwd-1", + "weights": [[1.0, 1.0], [0.5, 0.5]], + "metrics": {"loss": 2.25}, + } + + +async def _timed_out() -> list[Run]: + job = await Taskset("t", [Task(env="train", id="answer")]).run( + ScriptedAgent("ok", delay=5), runtime=LocalRuntime(_env()), rollout_timeout=0.2 + ) + return job.runs + + +@pytest.mark.parametrize( + ("runs", "group_size", "message"), + [ + pytest.param( + _timed_out, None, "timed-out runs require an explicit training reward", id="timeout" + ), + pytest.param( + lambda: _first(3, _runs(ScriptedAgent("ok"), group=2, tasks=_two_tasks())), + 2, + "3 trajectories do not divide evenly into groups of 2", + id="uneven", + ), + pytest.param( + lambda: _split_two_groups(_runs(ScriptedAgent("ok"), group=2, tasks=_three_tasks())), + 2, + "incomplete GRPO groups", + id="incomplete-group", + ), + pytest.param( + lambda: _single(Run.failed("never launched")), + None, + "run carries neither token-level samples nor a trace_id", + id="no-trace", + ), + ], +) +async def test_an_untrainable_batch_is_refused_before_anything_is_sent( + trainable: Platform, runs: Any, group_size: int | None, message: str +) -> None: + batch = await runs() + + with pytest.raises(ValueError, match=message): + await TrainingClient("owner/model").forward_backward(batch, group_size=group_size) + assert trainable.services.requests() == [] + + +def _two_tasks() -> list[Task]: + return [Task(env="train", id="answer", args={"target": target}) for target in ("ok", "no")] + + +def _three_tasks() -> list[Task]: + return [Task(env="train", id="answer", args={"target": t}) for t in ("ok", "no", "maybe")] + + +async def _first(count: int, runs: Any) -> list[Run]: + return (await runs)[:count] + + +async def _split_two_groups(runs: Any) -> list[Run]: + batch = await runs + return [batch[0], batch[2], batch[4], batch[5]] + + +async def _single(run: Run) -> list[Run]: + return [run] + + +@pytest.mark.parametrize( + ("answers", "order", "warns"), + [ + pytest.param(["ok", "ok", "ok", "ok"], [0, 1, 2, 3], True, id="no-spread"), + pytest.param(["ok", "no", "ok", "no"], [0, 1, 2, 3], False, id="spread"), + pytest.param(["ok", "ok", "ok", "ok"], [0, 2, 1, 3], True, id="interleaved-groups"), + ], +) +async def test_a_batch_with_no_reward_spread_warns_that_it_trains_nothing( + trainable: Platform, + caplog: pytest.LogCaptureFixture, + answers: list[str], + order: list[int], + warns: bool, +) -> None: + replies = iter(answers) + runs = await _runs(ScriptedAgent(lambda _: next(replies)), tasks=_two_tasks()) + + with caplog.at_level(logging.WARNING, logger="hud.train.client"): + await TrainingClient("owner/model").forward_backward( + [runs[index] for index in order], group_size=2 + ) + + assert ("accumulates no gradient" in caplog.text) is warns + assert len(trainable.training[MODEL_ID].batches) == 1