Rebuild the test suite as black-box scenarios

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
         run: uv python install ${{ matrix.python-version }}
 
       - name: Run tests
-        run: uv run --python ${{ matrix.python-version }} --all-extras pytest --cov --cov-report=''
+        run: uv run --python ${{ matrix.python-version }} --all-extras pytest -n auto --cov --cov-report=''
 
   lint-ruff:
     runs-on: ubuntu-latest
@@ -41,6 +41,9 @@ jobs:
           uv run --with=".[dev]" ruff format . --check
           uv run --with=".[dev]" ruff check .
 
+      - name: Check tests stay black-box
+        run: uv run --no-project python scripts/check_tests.py
+
   lint-ty:
     runs-on: ubuntu-latest
     steps:

.github/workflows/e2e.yml

@@ -0,0 +1,156 @@
+name: E2E
+
+# Tests marked `e2e` need something outside the test process. Each job selects
+# the ones its runner can provide; they skip themselves when it cannot.
+
+on:
+  pull_request:
+    paths:
+      - "hud/**"
+      - "tests/**"
+      - "environments/**"
+      - "pyproject.toml"
+      - ".github/workflows/e2e.yml"
+  push:
+    branches: [main]
+  schedule:
+    - cron: "17 7 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  docker:
+    name: Docker
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+
+      # Ubuntu 24.04 runners block unprivileged user namespaces through AppArmor;
+      # environments' bubblewrap sandboxes need them, as they have on HUD's hosts.
+      - name: Allow unprivileged user namespaces
+        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+
+      - name: Run Docker scenarios
+        run: >-
+          uv run --python 3.12 --all-extras
+          pytest -m "e2e and docker" -n 4 --timeout 1800 --durations 20
+
+  sandbox:
+    name: Sandbox
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      # Ubuntu 24.04 runners block unprivileged user namespaces through AppArmor;
+      # environments' bubblewrap sandboxes need them, as they have on HUD's hosts.
+      - name: Allow unprivileged user namespaces
+        run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+
+      # Environments run in containers with HUD's seccomp profile, so the
+      # sandbox scenarios run there too, as root, with bubblewrap installed.
+      - name: Run sandbox scenarios in a container
+        run: >-
+          docker run --rm
+          --security-opt seccomp=hud/eval/docker-seccomp.json
+          --security-opt systempaths=unconfined
+          --security-opt apparmor=unconfined
+          --volume "$PWD:/src" --workdir /src
+          --env UV_PROJECT_ENVIRONMENT=/opt/venv
+          python:3.12-bookworm
+          sh -c 'apt-get update -qq && apt-get install -y -qq bubblewrap openssh-client >/dev/null
+          && pip install -q "uv>=0.12.2"
+          && uv run --all-extras pytest -m "e2e and sandbox" -n 4 --timeout 600'
+
+  live:
+    name: Live
+    # Secrets are not available to pull requests from forks or Dependabot.
+    if: >-
+      github.event_name != 'pull_request' ||
+      (github.event.pull_request.head.repo.full_name == github.repository &&
+      github.event.pull_request.user.login != 'dependabot[bot]')
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      HUD_API_KEY: ${{ secrets.HUD_API_KEY }}
+      HUD_CI_DATA_FILE_ID: ${{ vars.HUD_CI_DATA_FILE_ID }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+
+      # The CLI agents' live scenarios run the real Claude Code and Codex CLIs.
+      - name: Install the agent CLIs
+        run: npm install --global @anthropic-ai/claude-code @openai/codex
+
+      - name: Run live scenarios
+        run: >-
+          uv run --python 3.12 --all-extras
+          pytest -m "e2e and live and not hosted" -n 4 --timeout 1200 --durations 20
+
+  modal:
+    name: Modal
+    # Runs real Modal sandboxes on the account in MODAL_TOKEN_ID / MODAL_TOKEN_SECRET;
+    # without them the scenarios skip themselves.
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
+      MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+
+      - name: Run Modal scenarios
+        run: >-
+          uv run --python 3.12 --all-extras
+          pytest -m "e2e and modal" -n 4 --timeout 1200 --durations 20
+
+  lerobot:
+    name: LeRobot
+    # Records robot rollouts with the real lerobot package (and torch), which no
+    # extra installs, so it runs on its own schedule.
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+
+      - name: Run LeRobot scenarios
+        run: >-
+          uv run --python 3.12 --all-extras --with 'lerobot[dataset]'
+          pytest -m "e2e and lerobot" -n 4 --timeout 600 --durations 20
+
+  cookbooks:
+    name: Cookbooks
+    # Cookbooks live outside the package, so SDK changes can break them silently.
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+
+      - name: Check cookbooks
+        run: >-
+          uv run --python 3.12 --all-extras
+          pytest tests/e2e/test_cookbooks.py -m e2e -n 4 --timeout 1200 --durations 20

.github/workflows/pre-release.yml

@@ -1,5 +1,8 @@
 name: Pre-release
 
+# The release gate (release.yml) looks up a successful run of this workflow on the
+# tagged commit, so keep its file name and trigger.
+
 on:
   workflow_dispatch:
 
@@ -15,8 +18,6 @@ jobs:
     concurrency:
       group: ci-example-${{ matrix.template }}
       cancel-in-progress: false
-    env:
-      EXAMPLE_NAME: ci-${{ matrix.template }}
     strategy:
       fail-fast: false
       matrix:
@@ -27,53 +28,30 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
-
-      - name: Create example environment
-        run: uv run hud init "$RUNNER_TEMP/$EXAMPLE_NAME" --template "${{ matrix.template }}"
-
-      - name: Install example with the SDK from this checkout
-        working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }}
-        run: |
-          uv build --wheel --no-create-gitignore --out-dir .hud-sdk "$GITHUB_WORKSPACE"
-          uv add --no-sync .hud-sdk/hud-*.whl
-          uv sync --all-extras
-          sed -i '/^RUN .*uv sync/i COPY .hud-sdk .hud-sdk' Dockerfile.hud
-          echo '!.hud-sdk/' >> .dockerignore
-          echo '!.hud-sdk/**' >> .dockerignore
-
-      - name: Run example checks
-        working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }}
-        run: |
-          uv run --no-sync hud task list --source .
-          if [ -d tests ]; then
-            uv run --no-sync ruff format . --check
-            uv run --no-sync ruff check .
-            uv run --no-sync pytest -q
-          fi
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
 
-      - name: Deploy example environment
-        working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }}
+      # The hosted test skips without credentials, which must not pass the gate.
+      - name: Require credentials
         env:
           HUD_API_KEY: ${{ secrets.HUD_API_KEY }}
-        run: uv run --no-sync hud deploy --no-env
+        run: test -n "$HUD_API_KEY"
 
-      - name: Run hosted task lifecycle
-        working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }}
+      # hud init, install this checkout's SDK, deploy, run one rollout on the HUD
+      # runtime with a live model, check its lifecycle, and read the trace back.
+      - name: Deploy and run the template on HUD
         env:
           HUD_API_KEY: ${{ secrets.HUD_API_KEY }}
           HUD_CI_DATA_FILE_ID: ${{ vars.HUD_CI_DATA_FILE_ID }}
-          HUD_TELEMETRY_LOCAL_DIR: ${{ runner.temp }}/traces-${{ matrix.template }}
-        run: |
-          uv run --no-sync "$GITHUB_WORKSPACE/scripts/example_lifecycle.py" prepare "${{ matrix.template }}"
-          timeout --signal=INT --kill-after=30s 15m uv run --no-sync hud eval .hud/ci-tasks.json claude \
-            --model claude-sonnet-4-6 --gateway --runtime hud --max-steps 10 --config max_tokens=2048 -y
-          uv run --no-sync "$GITHUB_WORKSPACE/scripts/example_lifecycle.py" check
+          HUD_E2E_TRACE_DIR: ${{ runner.temp }}/traces
+        run: >-
+          uv run --python 3.12 --all-extras
+          pytest -m "e2e and hosted" -p no:cacheprovider -rs
+          "tests/e2e/test_templates.py::test_a_deployed_template_runs_on_hud_and_its_trace_reads_back[${{ matrix.template }}]"
 
       - name: Save traces
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: example-traces-${{ matrix.template }}
-          path: ${{ runner.temp }}/traces-${{ matrix.template }}
+          path: ${{ runner.temp }}/traces/${{ matrix.template }}
           retention-days: 7

.gitignore

@@ -13,6 +13,8 @@ build/
 !docs/v6/build/
 *.egg-info/
 uv.lock
+# Harbor case fixtures carry a project's .env as test data.
+!tests/integrations/harbor/cases/**/.env
 
 # Media files
 *.jpg

AGENTS.md

@@ -16,7 +16,7 @@ adding local workarounds.
 - `CONTRIBUTING.md` for setup, test, lint, and type-check commands.
 - `pyproject.toml` for supported Python versions, dependencies, optional extras,
   ruff, ty, pytest, and coverage configuration.
-- Source files and colocated tests for exact behavior. Trust code and tests over
+- Source files and their tests for exact behavior. Trust code and tests over
   stale prose.
 - `cookbooks/` for runnable end-to-end examples (each is its own uv project).
 
@@ -31,6 +31,8 @@ inventory of current incidents.
 - `hud/cli/` is the Typer surface over the same modules.
 - `hud/integrations/` contains packaged adapters for external task formats.
 - `hud/_legacy.py` and `hud/patches/` quarantine v5 compatibility.
+- `tests/` mirrors the `hud` package layout: tests for `hud/cli/` live in
+  `tests/cli/`, tests for top-level modules in `tests/` itself.
 - `cookbooks/` contains standalone runnable examples outside the `hud` package.
 
 ## Working Style
@@ -53,14 +55,15 @@ Use the commands in `CONTRIBUTING.md` as the source of truth. Common commands:
 
 ```bash
 uv sync --extra dev
-uv run pytest -q
 uv run ruff format . --check
 uv run ruff check .
 uv run --extra dev --extra train ty check
+uv run python scripts/check_tests.py
+uv run pytest -n auto
 ```
 
-The shared pre-push hook lives in `.githooks/pre-push`, but agents should not
-change local git config unless explicitly asked.
+Locally, ruff and ty are the loop; CI runs the test suite. Run the tests you
+touched, and leave the full suite and the `e2e` lanes to CI.
 
 Tests run on Python 3.11 and 3.12 in CI. `pyproject.toml` currently supports
 Python `>=3.11, <3.13`.
@@ -119,27 +122,33 @@ Python `>=3.11, <3.13`.
 - Use `TYPE_CHECKING` imports for type-only imports that would otherwise add
   runtime dependency cost or cycles.
 
-## Testing Expectations
-
-- Add or update focused tests for behavior changes. Put tests near the module
-  they cover, following the existing `*/tests/` layout.
-- Test behavior and contracts, not private implementation details.
-- Regression tests should fail on the old behavior through the normal lifecycle
-  or public boundary. Do not manually seed private state such as internal maps,
-  caches, cursors, or prepared containers just to prove a changed line.
-- If a bug involves internal state, reach it through real setup and execution:
-  construction, configuration, preparation, run loop, provider response, tool
-  execution, or public API call.
-- Do not add hooks, helper methods, or abstraction layers only to make tests
-  easier. If a test needs that, reconsider the behavior boundary instead.
-- Test names should describe the observable behavior or contract, not the
-  private mechanism.
-- Mock external services, provider APIs, network, Docker, browser, and filesystem
-  boundaries as needed. Do not mock core logic just to make a test easy.
-- Mark tests that require `HUD_API_KEY`, network access, or deployed services as
-  integration tests.
-- Run the narrowest relevant tests first, then broader checks when the blast
-  radius is shared or user-facing.
+## Testing
+
+Tests are black-box: they drive the SDK through a public boundary and fake only
+what lies outside it. `tests/support` holds those fakes: the HUD
+services, scripted model providers, a `docker` executable, a VNC screen, and the
+real `hud` CLI run in a subprocess. `CONTRIBUTING.md` shows how to use it.
+
+- Enter through a public boundary: the `hud` CLI, an exported name, the control
+  channel or another wire protocol, a file format, or a process effect.
+- Fake only the outside world: HUD services, model providers, Docker, cloud SDKs,
+  the clock. Never patch a `hud` name, import a private one, or touch a private
+  attribute; `scripts/check_tests.py` enforces this in CI. Configure the SDK the
+  way a user does, through environment variables (`hud_env.set(...)`).
+- Never hand-build what a producer emits. Run the producer, so the test covers
+  the seam between it and its consumer.
+- Assert what a user or a peer observes: the reward, run and trace fields,
+  output and exit codes, files, what the platform recorded, wire frames. Prefer
+  fixture environments whose reward proves the path ran.
+- Write a scenario once per contract, as a table; a new case is a new row.
+  Assert the fields the case is about, never a recorded copy of a whole output:
+  no snapshots of argv, request bodies or rendered text.
+- Do not add tests to check your own change. Check it with a temporary script,
+  and extend a scenario only when the change adds or fixes a contract; a bug fix
+  lands with the scenario row that failed before it.
+- Tests needing something outside the process are marked `e2e` plus `docker`,
+  `sandbox`, `live` (a real HUD account) or `hosted`, and run only when selected.
+- Test names describe the observable behavior, not the mechanism.
 
 ## Operational Debugging