From 44b05ba586fedc67e2ea527c644ed912cf49119e Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:45:09 -0700 Subject: [PATCH 001/248] refactor: consolidate the package tests under one tests/ tree Tests lived in a tests/ package inside each hud subpackage, so the wheel shipped them and the sdist shipped the Harbor fixtures too. They now live in a top-level tests/ package that mirrors the hud layout: tests for hud/cli/ in tests/cli/, tests for top-level modules in tests/ itself. Paths that assumed the old depth now resolve from the module under test, pytest collects tests/, ty checks it alongside hud, and the build and coverage excludes for nested test directories are gone. --- AGENTS.md | 8 +++++--- pyproject.toml | 14 +++----------- .../claude/tools/tests => tests}/__init__.py | 0 {hud/agents/tests => tests/agents}/__init__.py | 0 .../tests => tests/agents/claude}/__init__.py | 0 .../agents/claude/tools}/__init__.py | 0 .../agents/claude/tools}/test_computer.py | 0 {hud/agents/tests => tests/agents}/cli_fakes.py | 0 .../tests => tests/agents/gemini}/__init__.py | 0 .../agents/gemini/tools}/__init__.py | 0 .../agents/gemini/tools}/test_computer.py | 0 .../eval/tests => tests/agents/openai}/__init__.py | 0 .../agents/openai/tools}/__init__.py | 0 .../agents/openai/tools}/test_computer.py | 0 .../agents/openai/tools}/test_strict_schema.py | 0 .../tests => tests/agents}/test_apply_patch.py | 0 {hud/agents/tests => tests/agents}/test_base.py | 0 .../tests => tests/agents}/test_claude_agent.py | 0 .../agents}/test_claude_cli_agent.py | 7 ++++--- .../tests => tests/agents}/test_codex_cli_agent.py | 7 ++++--- .../tests => tests/agents}/test_gemini_agent.py | 0 .../tests => tests/agents}/test_openai_agent.py | 0 .../agents}/test_openai_compatible_agent.py | 0 .../agents}/test_provider_native_tools.py | 0 .../agents}/test_robot_dataset_writer.py | 0 .../tests => tests/agents}/test_tool_agent.py | 0 {hud/agents/tests => tests/agents}/test_trace.py | 0 .../mcp-server => tests/capabilities}/__init__.py | 0 .../tests => tests/capabilities}/test_rfb.py | 0 .../tests => tests/capabilities}/test_ssh.py | 0 {hud/cli/tests => tests/cli}/__init__.py | 0 {hud/cli/tests => tests/cli}/test_cli.py | 0 {hud/cli/tests => tests/cli}/test_deploy.py | 0 {hud/cli/tests => tests/cli}/test_eval.py | 0 {hud/cli/tests => tests/cli}/test_init.py | 0 {hud/cli/tests => tests/cli}/test_jobs.py | 0 {hud/cli/tests => tests/cli}/test_main.py | 0 {hud/cli/tests => tests/cli}/test_project.py | 0 {hud/cli/tests => tests/cli}/test_qa.py | 0 {hud/cli/tests => tests/cli}/test_sync.py | 0 {hud/cli/tests => tests/cli}/test_task.py | 0 {hud/cli/tests => tests/cli}/test_trace.py | 0 {hud/clients/tests => tests/clients}/__init__.py | 0 .../tests => tests/clients}/test_connect.py | 0 {hud => tests}/conftest.py | 0 .../tests => tests/environment}/__init__.py | 0 .../tests => tests/environment}/conftest.py | 0 .../tests => tests/environment}/test_arguments.py | 0 .../environment}/test_capability_backing.py | 0 .../environment}/test_file_tracker.py | 0 .../environment}/test_file_tracking.py | 0 .../tests => tests/environment}/test_loader.py | 0 .../tests => tests/environment}/test_manifest.py | 0 .../tests => tests/environment}/test_platform.py | 0 .../environment}/test_process_connection.py | 4 ++-- .../environment}/test_robot_control.py | 0 .../environment}/test_robot_regressions.py | 0 .../tests => tests/environment}/test_server.py | 0 .../tests => tests/environment}/test_sessions.py | 0 .../tests => tests/environment}/test_tunnel.py | 0 .../tests => tests/environment}/test_workspace.py | 0 {hud/tests => tests/eval}/__init__.py | 0 {hud/eval/tests => tests/eval}/test_chat.py | 0 .../tests => tests/eval}/test_docker_provider.py | 0 .../eval}/test_file_tracking_observer.py | 0 {hud/eval/tests => tests/eval}/test_hosted.py | 0 {hud/eval/tests => tests/eval}/test_job.py | 0 .../tests => tests/eval}/test_local_runtime.py | 0 {hud/eval/tests => tests/eval}/test_rollout.py | 0 {hud/eval/tests => tests/eval}/test_shared.py | 0 {hud/eval/tests => tests/eval}/test_sync.py | 0 {hud/eval/tests => tests/eval}/test_task.py | 0 .../utils/tests => tests/integrations}/__init__.py | 0 tests/integrations/harbor/__init__.py | 0 .../integrations/harbor}/conftest.py | 0 .../tasks/agent-lifecycle/environment/Dockerfile | 0 .../harbor}/tasks/agent-lifecycle/instruction.md | 0 .../tasks/agent-lifecycle/solution/solve.sh | 0 .../harbor}/tasks/agent-lifecycle/task.toml | 0 .../harbor}/tasks/agent-lifecycle/tests/test.sh | 0 .../harbor}/tasks/hello-mcp/environment/Dockerfile | 0 .../hello-mcp/environment/docker-compose.yaml | 0 .../hello-mcp/environment/mcp-server/Dockerfile | 0 .../hello-mcp/environment/mcp-server/__init__.py | 0 .../hello-mcp/environment/mcp-server/server.py | 0 .../harbor}/tasks/hello-mcp/instruction.md | 0 .../harbor}/tasks/hello-mcp/solution/solve.sh | 0 .../integrations/harbor}/tasks/hello-mcp/task.toml | 0 .../harbor}/tasks/hello-mcp/tests/test.sh | 0 .../tasks/phase-boundary/environment/Dockerfile | 0 .../tasks/phase-boundary/environment/entrypoint.sh | 0 .../harbor}/tasks/phase-boundary/instruction.md | 0 .../harbor}/tasks/phase-boundary/solution/solve.sh | 0 .../harbor}/tasks/phase-boundary/task.toml | 0 .../harbor}/tasks/phase-boundary/tests/test.sh | 0 .../sidecar-reachability/environment/Dockerfile | 0 .../environment/docker-compose.yaml | 0 .../tasks/sidecar-reachability/instruction.md | 0 .../tasks/sidecar-reachability/solution/solve.sh | 0 .../harbor}/tasks/sidecar-reachability/task.toml | 0 .../tasks/sidecar-reachability/tests/Dockerfile | 0 .../tasks/sidecar-reachability/tests/test.sh | 0 .../verifier-lifecycle/environment/Dockerfile | 0 .../verifier-lifecycle/environment/entrypoint.sh | 0 .../tasks/verifier-lifecycle/instruction.md | 0 .../tasks/verifier-lifecycle/solution/solve.sh | 0 .../harbor}/tasks/verifier-lifecycle/task.toml | 0 .../harbor}/tasks/verifier-lifecycle/tests/test.sh | 0 .../integrations/harbor}/test_contract.py | 2 +- .../integrations/harbor}/test_harbor.py | 0 .../integrations/harbor}/test_integration.py | 2 +- {hud/patches/tests => tests/patches}/__init__.py | 0 .../tests => tests/patches}/test_warnings.py | 6 +++--- tests/telemetry/__init__.py | 0 .../tests => tests/telemetry}/test_exporter.py | 0 .../tests => tests/telemetry}/test_instrument.py | 0 .../tests => tests/telemetry}/test_robot_video.py | 0 {hud/tests => tests}/test_graders.py | 0 {hud/tests => tests}/test_init.py | 0 {hud/tests => tests}/test_init_module.py | 0 {hud/tests => tests}/test_robot.py | 0 {hud/tests => tests}/test_settings.py | 0 {hud/tests => tests}/test_trace.py | 0 {hud/tests => tests}/test_types.py | 0 {hud/tests => tests}/test_version.py | 0 {hud/train/tests => tests/train}/__init__.py | 0 {hud/train/tests => tests/train}/test_client.py | 0 tests/utils/__init__.py | 0 .../utils/tests => tests/utils}/test_exceptions.py | 0 {hud/utils/tests => tests/utils}/test_gateway.py | 0 {hud/utils/tests => tests/utils}/test_hints.py | 0 .../tests => tests/utils}/test_hud_console.py | 0 {hud/utils/tests => tests/utils}/test_naming.py | 0 {hud/utils/tests => tests/utils}/test_platform.py | 0 {hud/utils/tests => tests/utils}/test_process.py | 0 {hud/utils/tests => tests/utils}/test_requests.py | 0 .../tests => tests/utils}/test_serialization.py | 0 137 files changed, 23 insertions(+), 27 deletions(-) rename {hud/agents/claude/tools/tests => tests}/__init__.py (100%) rename {hud/agents/tests => tests/agents}/__init__.py (100%) rename {hud/agents/gemini/tools/tests => tests/agents/claude}/__init__.py (100%) rename {hud/agents/openai/tools/tests => tests/agents/claude/tools}/__init__.py (100%) rename {hud/agents/claude/tools/tests => tests/agents/claude/tools}/test_computer.py (100%) rename {hud/agents/tests => tests/agents}/cli_fakes.py (100%) rename {hud/capabilities/tests => tests/agents/gemini}/__init__.py (100%) rename {hud/environment/tests => tests/agents/gemini/tools}/__init__.py (100%) rename {hud/agents/gemini/tools/tests => tests/agents/gemini/tools}/test_computer.py (100%) rename {hud/eval/tests => tests/agents/openai}/__init__.py (100%) rename {hud/integrations/harbor/tests => tests/agents/openai/tools}/__init__.py (100%) rename {hud/agents/openai/tools/tests => tests/agents/openai/tools}/test_computer.py (100%) rename {hud/agents/openai/tools/tests => tests/agents/openai/tools}/test_strict_schema.py (100%) rename {hud/agents/tests => tests/agents}/test_apply_patch.py (100%) rename {hud/agents/tests => tests/agents}/test_base.py (100%) rename {hud/agents/tests => tests/agents}/test_claude_agent.py (100%) rename {hud/agents/tests => tests/agents}/test_claude_cli_agent.py (99%) rename {hud/agents/tests => tests/agents}/test_codex_cli_agent.py (98%) rename {hud/agents/tests => tests/agents}/test_gemini_agent.py (100%) rename {hud/agents/tests => tests/agents}/test_openai_agent.py (100%) rename {hud/agents/tests => tests/agents}/test_openai_compatible_agent.py (100%) rename {hud/agents/tests => tests/agents}/test_provider_native_tools.py (100%) rename {hud/agents/tests => tests/agents}/test_robot_dataset_writer.py (100%) rename {hud/agents/tests => tests/agents}/test_tool_agent.py (100%) rename {hud/agents/tests => tests/agents}/test_trace.py (100%) rename {hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server => tests/capabilities}/__init__.py (100%) rename {hud/capabilities/tests => tests/capabilities}/test_rfb.py (100%) rename {hud/capabilities/tests => tests/capabilities}/test_ssh.py (100%) rename {hud/cli/tests => tests/cli}/__init__.py (100%) rename {hud/cli/tests => tests/cli}/test_cli.py (100%) rename {hud/cli/tests => tests/cli}/test_deploy.py (100%) rename {hud/cli/tests => tests/cli}/test_eval.py (100%) rename {hud/cli/tests => tests/cli}/test_init.py (100%) rename {hud/cli/tests => tests/cli}/test_jobs.py (100%) rename {hud/cli/tests => tests/cli}/test_main.py (100%) rename {hud/cli/tests => tests/cli}/test_project.py (100%) rename {hud/cli/tests => tests/cli}/test_qa.py (100%) rename {hud/cli/tests => tests/cli}/test_sync.py (100%) rename {hud/cli/tests => tests/cli}/test_task.py (100%) rename {hud/cli/tests => tests/cli}/test_trace.py (100%) rename {hud/clients/tests => tests/clients}/__init__.py (100%) rename {hud/clients/tests => tests/clients}/test_connect.py (100%) rename {hud => tests}/conftest.py (100%) rename {hud/telemetry/tests => tests/environment}/__init__.py (100%) rename {hud/environment/tests => tests/environment}/conftest.py (100%) rename {hud/environment/tests => tests/environment}/test_arguments.py (100%) rename {hud/environment/tests => tests/environment}/test_capability_backing.py (100%) rename {hud/environment/tests => tests/environment}/test_file_tracker.py (100%) rename {hud/environment/tests => tests/environment}/test_file_tracking.py (100%) rename {hud/environment/tests => tests/environment}/test_loader.py (100%) rename {hud/environment/tests => tests/environment}/test_manifest.py (100%) rename {hud/environment/tests => tests/environment}/test_platform.py (100%) rename {hud/environment/tests => tests/environment}/test_process_connection.py (99%) rename {hud/environment/tests => tests/environment}/test_robot_control.py (100%) rename {hud/environment/tests => tests/environment}/test_robot_regressions.py (100%) rename {hud/environment/tests => tests/environment}/test_server.py (100%) rename {hud/environment/tests => tests/environment}/test_sessions.py (100%) rename {hud/environment/tests => tests/environment}/test_tunnel.py (100%) rename {hud/environment/tests => tests/environment}/test_workspace.py (100%) rename {hud/tests => tests/eval}/__init__.py (100%) rename {hud/eval/tests => tests/eval}/test_chat.py (100%) rename {hud/eval/tests => tests/eval}/test_docker_provider.py (100%) rename {hud/eval/tests => tests/eval}/test_file_tracking_observer.py (100%) rename {hud/eval/tests => tests/eval}/test_hosted.py (100%) rename {hud/eval/tests => tests/eval}/test_job.py (100%) rename {hud/eval/tests => tests/eval}/test_local_runtime.py (100%) rename {hud/eval/tests => tests/eval}/test_rollout.py (100%) rename {hud/eval/tests => tests/eval}/test_shared.py (100%) rename {hud/eval/tests => tests/eval}/test_sync.py (100%) rename {hud/eval/tests => tests/eval}/test_task.py (100%) rename {hud/utils/tests => tests/integrations}/__init__.py (100%) create mode 100644 tests/integrations/harbor/__init__.py rename {hud/integrations/harbor/tests => tests/integrations/harbor}/conftest.py (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/agent-lifecycle/environment/Dockerfile (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/agent-lifecycle/instruction.md (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/agent-lifecycle/solution/solve.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/agent-lifecycle/task.toml (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/agent-lifecycle/tests/test.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/hello-mcp/environment/Dockerfile (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/hello-mcp/environment/docker-compose.yaml (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/hello-mcp/environment/mcp-server/Dockerfile (100%) create mode 100644 tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/__init__.py rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/hello-mcp/environment/mcp-server/server.py (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/hello-mcp/instruction.md (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/hello-mcp/solution/solve.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/hello-mcp/task.toml (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/hello-mcp/tests/test.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/phase-boundary/environment/Dockerfile (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/phase-boundary/environment/entrypoint.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/phase-boundary/instruction.md (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/phase-boundary/solution/solve.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/phase-boundary/task.toml (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/phase-boundary/tests/test.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/sidecar-reachability/environment/Dockerfile (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/sidecar-reachability/environment/docker-compose.yaml (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/sidecar-reachability/instruction.md (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/sidecar-reachability/solution/solve.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/sidecar-reachability/task.toml (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/sidecar-reachability/tests/Dockerfile (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/sidecar-reachability/tests/test.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/verifier-lifecycle/environment/Dockerfile (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/verifier-lifecycle/environment/entrypoint.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/verifier-lifecycle/instruction.md (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/verifier-lifecycle/solution/solve.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/verifier-lifecycle/task.toml (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/tasks/verifier-lifecycle/tests/test.sh (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/test_contract.py (99%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/test_harbor.py (100%) rename {hud/integrations/harbor/tests => tests/integrations/harbor}/test_integration.py (99%) rename {hud/patches/tests => tests/patches}/__init__.py (100%) rename {hud/patches/tests => tests/patches}/test_warnings.py (96%) create mode 100644 tests/telemetry/__init__.py rename {hud/telemetry/tests => tests/telemetry}/test_exporter.py (100%) rename {hud/telemetry/tests => tests/telemetry}/test_instrument.py (100%) rename {hud/telemetry/tests => tests/telemetry}/test_robot_video.py (100%) rename {hud/tests => tests}/test_graders.py (100%) rename {hud/tests => tests}/test_init.py (100%) rename {hud/tests => tests}/test_init_module.py (100%) rename {hud/tests => tests}/test_robot.py (100%) rename {hud/tests => tests}/test_settings.py (100%) rename {hud/tests => tests}/test_trace.py (100%) rename {hud/tests => tests}/test_types.py (100%) rename {hud/tests => tests}/test_version.py (100%) rename {hud/train/tests => tests/train}/__init__.py (100%) rename {hud/train/tests => tests/train}/test_client.py (100%) create mode 100644 tests/utils/__init__.py rename {hud/utils/tests => tests/utils}/test_exceptions.py (100%) rename {hud/utils/tests => tests/utils}/test_gateway.py (100%) rename {hud/utils/tests => tests/utils}/test_hints.py (100%) rename {hud/utils/tests => tests/utils}/test_hud_console.py (100%) rename {hud/utils/tests => tests/utils}/test_naming.py (100%) rename {hud/utils/tests => tests/utils}/test_platform.py (100%) rename {hud/utils/tests => tests/utils}/test_process.py (100%) rename {hud/utils/tests => tests/utils}/test_requests.py (100%) rename {hud/utils/tests => tests/utils}/test_serialization.py (100%) diff --git a/AGENTS.md b/AGENTS.md index 3625e317b..bfca8c0b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ adding local workarounds. - `CONTRIBUTING.md` for setup, test, lint, and type-check commands. - `pyproject.toml` for supported Python versions, dependencies, optional extras, ruff, ty, pytest, and coverage configuration. -- Source files and colocated tests for exact behavior. Trust code and tests over +- Source files and their tests for exact behavior. Trust code and tests over stale prose. - `cookbooks/` for runnable end-to-end examples (each is its own uv project). @@ -31,6 +31,8 @@ inventory of current incidents. - `hud/cli/` is the Typer surface over the same modules. - `hud/integrations/` contains packaged adapters for external task formats. - `hud/_legacy.py` and `hud/patches/` quarantine v5 compatibility. +- `tests/` mirrors the `hud` package layout: tests for `hud/cli/` live in + `tests/cli/`, tests for top-level modules in `tests/` itself. - `cookbooks/` contains standalone runnable examples outside the `hud` package. ## Working Style @@ -121,8 +123,8 @@ Python `>=3.11, <3.13`. ## Testing Expectations -- Add or update focused tests for behavior changes. Put tests near the module - they cover, following the existing `*/tests/` layout. +- Add or update focused tests for behavior changes. Put them in the `tests/` + package that mirrors the module they cover. - Test behavior and contracts, not private implementation details. - Regression tests should fail on the old behavior through the normal lifecycle or public boundary. Do not manually seed private state such as internal maps, diff --git a/pyproject.toml b/pyproject.toml index b4ccdbfe5..070bc0e8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,6 @@ exclude = [ "cookbooks/", "environments/", "hud-python/", - "hud/integrations/harbor/tests/", "**/checkpoints/", "**/*.safetensors", "**/*.ckpt", @@ -96,7 +95,6 @@ include = [ "/pyproject.toml" ] exclude = [ - "*/tests/", "*/__pycache__/", "**/*.safetensors", "**/*.ckpt", @@ -128,7 +126,7 @@ bedrock = [ # Development dependencies - includes testing, linting, and automation tools dev = [ "hud[agents]", # Include agents for dev - "hud[robot]", # the robot end-to-end gates (hud/tests/test_robot.py) run the real wire + "hud[robot]", # the robot end-to-end gates (tests/test_robot.py) run the real wire "dotenv>=0.9.9", # Testing and linting "ruff==0.16.1", @@ -230,7 +228,7 @@ docstring-code-format = true runtime-evaluated-base-classes = ["pydantic.BaseModel"] [tool.ty.src] -include = ["hud"] +include = ["hud", "tests"] [tool.ty.rules] blanket-ignore-comment = "error" @@ -244,9 +242,6 @@ strict-equality-semantics = true [tool.coverage.run] source = ["hud"] -omit = [ - "*/tests/*", -] [tool.coverage.report] exclude_lines = [ @@ -263,14 +258,11 @@ exclude_lines = [ ] show_missing = true fail_under = 58 -omit = [ - "*/tests/*", -] [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" -testpaths = ["hud"] +testpaths = ["tests"] addopts = "-m 'not integration'" markers = [ "integration: marks tests as integration tests (require HUD_API_KEY, network access, or Docker)", diff --git a/hud/agents/claude/tools/tests/__init__.py b/tests/__init__.py similarity index 100% rename from hud/agents/claude/tools/tests/__init__.py rename to tests/__init__.py diff --git a/hud/agents/tests/__init__.py b/tests/agents/__init__.py similarity index 100% rename from hud/agents/tests/__init__.py rename to tests/agents/__init__.py diff --git a/hud/agents/gemini/tools/tests/__init__.py b/tests/agents/claude/__init__.py similarity index 100% rename from hud/agents/gemini/tools/tests/__init__.py rename to tests/agents/claude/__init__.py diff --git a/hud/agents/openai/tools/tests/__init__.py b/tests/agents/claude/tools/__init__.py similarity index 100% rename from hud/agents/openai/tools/tests/__init__.py rename to tests/agents/claude/tools/__init__.py diff --git a/hud/agents/claude/tools/tests/test_computer.py b/tests/agents/claude/tools/test_computer.py similarity index 100% rename from hud/agents/claude/tools/tests/test_computer.py rename to tests/agents/claude/tools/test_computer.py diff --git a/hud/agents/tests/cli_fakes.py b/tests/agents/cli_fakes.py similarity index 100% rename from hud/agents/tests/cli_fakes.py rename to tests/agents/cli_fakes.py diff --git a/hud/capabilities/tests/__init__.py b/tests/agents/gemini/__init__.py similarity index 100% rename from hud/capabilities/tests/__init__.py rename to tests/agents/gemini/__init__.py diff --git a/hud/environment/tests/__init__.py b/tests/agents/gemini/tools/__init__.py similarity index 100% rename from hud/environment/tests/__init__.py rename to tests/agents/gemini/tools/__init__.py diff --git a/hud/agents/gemini/tools/tests/test_computer.py b/tests/agents/gemini/tools/test_computer.py similarity index 100% rename from hud/agents/gemini/tools/tests/test_computer.py rename to tests/agents/gemini/tools/test_computer.py diff --git a/hud/eval/tests/__init__.py b/tests/agents/openai/__init__.py similarity index 100% rename from hud/eval/tests/__init__.py rename to tests/agents/openai/__init__.py diff --git a/hud/integrations/harbor/tests/__init__.py b/tests/agents/openai/tools/__init__.py similarity index 100% rename from hud/integrations/harbor/tests/__init__.py rename to tests/agents/openai/tools/__init__.py diff --git a/hud/agents/openai/tools/tests/test_computer.py b/tests/agents/openai/tools/test_computer.py similarity index 100% rename from hud/agents/openai/tools/tests/test_computer.py rename to tests/agents/openai/tools/test_computer.py diff --git a/hud/agents/openai/tools/tests/test_strict_schema.py b/tests/agents/openai/tools/test_strict_schema.py similarity index 100% rename from hud/agents/openai/tools/tests/test_strict_schema.py rename to tests/agents/openai/tools/test_strict_schema.py diff --git a/hud/agents/tests/test_apply_patch.py b/tests/agents/test_apply_patch.py similarity index 100% rename from hud/agents/tests/test_apply_patch.py rename to tests/agents/test_apply_patch.py diff --git a/hud/agents/tests/test_base.py b/tests/agents/test_base.py similarity index 100% rename from hud/agents/tests/test_base.py rename to tests/agents/test_base.py diff --git a/hud/agents/tests/test_claude_agent.py b/tests/agents/test_claude_agent.py similarity index 100% rename from hud/agents/tests/test_claude_agent.py rename to tests/agents/test_claude_agent.py diff --git a/hud/agents/tests/test_claude_cli_agent.py b/tests/agents/test_claude_cli_agent.py similarity index 99% rename from hud/agents/tests/test_claude_cli_agent.py rename to tests/agents/test_claude_cli_agent.py index a3cb5ac37..35f488b37 100644 --- a/hud/agents/tests/test_claude_cli_agent.py +++ b/tests/agents/test_claude_cli_agent.py @@ -27,9 +27,6 @@ from hud.agents.claude.sdk import computer_mcp from hud.agents.claude.sdk.agent import ClaudeCLIAgent from hud.agents.claude.sdk.events import ClaudeEvents -from hud.agents.tests.cli_fakes import FakeClient as _FakeClient -from hud.agents.tests.cli_fakes import FakeProcess as _FakeStreamProcess -from hud.agents.tests.cli_fakes import fake_run as _fake_run from hud.agents.types import AgentStep, ClaudeCLIConfig, ToolStep from hud.capabilities import Capability, Connection, SSHClient from hud.capabilities.rfb import WebPScreenshotEncoding @@ -37,6 +34,10 @@ from hud.telemetry.context import set_trace_context from hud.types import MCPToolResult +from .cli_fakes import FakeClient as _FakeClient +from .cli_fakes import FakeProcess as _FakeStreamProcess +from .cli_fakes import fake_run as _fake_run + @pytest.fixture(autouse=True) def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/hud/agents/tests/test_codex_cli_agent.py b/tests/agents/test_codex_cli_agent.py similarity index 98% rename from hud/agents/tests/test_codex_cli_agent.py rename to tests/agents/test_codex_cli_agent.py index 84c77b452..7b9bcb303 100644 --- a/hud/agents/tests/test_codex_cli_agent.py +++ b/tests/agents/test_codex_cli_agent.py @@ -13,15 +13,16 @@ from hud.agents.cli import resolve_executable from hud.agents.codex import CodexCLIAgent -from hud.agents.tests.cli_fakes import FakeClient as _FakeClient -from hud.agents.tests.cli_fakes import FakeProcess as _FakeProcess -from hud.agents.tests.cli_fakes import fake_run as _fake_run from hud.agents.types import AgentStep, CodexCLIConfig, ToolStep from hud.capabilities import Capability, Connection from hud.eval.runtime import RuntimeConfig, RuntimeResources from hud.settings import settings from hud.telemetry.context import set_trace_context +from .cli_fakes import FakeClient as _FakeClient +from .cli_fakes import FakeProcess as _FakeProcess +from .cli_fakes import fake_run as _fake_run + @pytest.fixture(autouse=True) def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/hud/agents/tests/test_gemini_agent.py b/tests/agents/test_gemini_agent.py similarity index 100% rename from hud/agents/tests/test_gemini_agent.py rename to tests/agents/test_gemini_agent.py diff --git a/hud/agents/tests/test_openai_agent.py b/tests/agents/test_openai_agent.py similarity index 100% rename from hud/agents/tests/test_openai_agent.py rename to tests/agents/test_openai_agent.py diff --git a/hud/agents/tests/test_openai_compatible_agent.py b/tests/agents/test_openai_compatible_agent.py similarity index 100% rename from hud/agents/tests/test_openai_compatible_agent.py rename to tests/agents/test_openai_compatible_agent.py diff --git a/hud/agents/tests/test_provider_native_tools.py b/tests/agents/test_provider_native_tools.py similarity index 100% rename from hud/agents/tests/test_provider_native_tools.py rename to tests/agents/test_provider_native_tools.py diff --git a/hud/agents/tests/test_robot_dataset_writer.py b/tests/agents/test_robot_dataset_writer.py similarity index 100% rename from hud/agents/tests/test_robot_dataset_writer.py rename to tests/agents/test_robot_dataset_writer.py diff --git a/hud/agents/tests/test_tool_agent.py b/tests/agents/test_tool_agent.py similarity index 100% rename from hud/agents/tests/test_tool_agent.py rename to tests/agents/test_tool_agent.py diff --git a/hud/agents/tests/test_trace.py b/tests/agents/test_trace.py similarity index 100% rename from hud/agents/tests/test_trace.py rename to tests/agents/test_trace.py diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/__init__.py b/tests/capabilities/__init__.py similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/__init__.py rename to tests/capabilities/__init__.py diff --git a/hud/capabilities/tests/test_rfb.py b/tests/capabilities/test_rfb.py similarity index 100% rename from hud/capabilities/tests/test_rfb.py rename to tests/capabilities/test_rfb.py diff --git a/hud/capabilities/tests/test_ssh.py b/tests/capabilities/test_ssh.py similarity index 100% rename from hud/capabilities/tests/test_ssh.py rename to tests/capabilities/test_ssh.py diff --git a/hud/cli/tests/__init__.py b/tests/cli/__init__.py similarity index 100% rename from hud/cli/tests/__init__.py rename to tests/cli/__init__.py diff --git a/hud/cli/tests/test_cli.py b/tests/cli/test_cli.py similarity index 100% rename from hud/cli/tests/test_cli.py rename to tests/cli/test_cli.py diff --git a/hud/cli/tests/test_deploy.py b/tests/cli/test_deploy.py similarity index 100% rename from hud/cli/tests/test_deploy.py rename to tests/cli/test_deploy.py diff --git a/hud/cli/tests/test_eval.py b/tests/cli/test_eval.py similarity index 100% rename from hud/cli/tests/test_eval.py rename to tests/cli/test_eval.py diff --git a/hud/cli/tests/test_init.py b/tests/cli/test_init.py similarity index 100% rename from hud/cli/tests/test_init.py rename to tests/cli/test_init.py diff --git a/hud/cli/tests/test_jobs.py b/tests/cli/test_jobs.py similarity index 100% rename from hud/cli/tests/test_jobs.py rename to tests/cli/test_jobs.py diff --git a/hud/cli/tests/test_main.py b/tests/cli/test_main.py similarity index 100% rename from hud/cli/tests/test_main.py rename to tests/cli/test_main.py diff --git a/hud/cli/tests/test_project.py b/tests/cli/test_project.py similarity index 100% rename from hud/cli/tests/test_project.py rename to tests/cli/test_project.py diff --git a/hud/cli/tests/test_qa.py b/tests/cli/test_qa.py similarity index 100% rename from hud/cli/tests/test_qa.py rename to tests/cli/test_qa.py diff --git a/hud/cli/tests/test_sync.py b/tests/cli/test_sync.py similarity index 100% rename from hud/cli/tests/test_sync.py rename to tests/cli/test_sync.py diff --git a/hud/cli/tests/test_task.py b/tests/cli/test_task.py similarity index 100% rename from hud/cli/tests/test_task.py rename to tests/cli/test_task.py diff --git a/hud/cli/tests/test_trace.py b/tests/cli/test_trace.py similarity index 100% rename from hud/cli/tests/test_trace.py rename to tests/cli/test_trace.py diff --git a/hud/clients/tests/__init__.py b/tests/clients/__init__.py similarity index 100% rename from hud/clients/tests/__init__.py rename to tests/clients/__init__.py diff --git a/hud/clients/tests/test_connect.py b/tests/clients/test_connect.py similarity index 100% rename from hud/clients/tests/test_connect.py rename to tests/clients/test_connect.py diff --git a/hud/conftest.py b/tests/conftest.py similarity index 100% rename from hud/conftest.py rename to tests/conftest.py diff --git a/hud/telemetry/tests/__init__.py b/tests/environment/__init__.py similarity index 100% rename from hud/telemetry/tests/__init__.py rename to tests/environment/__init__.py diff --git a/hud/environment/tests/conftest.py b/tests/environment/conftest.py similarity index 100% rename from hud/environment/tests/conftest.py rename to tests/environment/conftest.py diff --git a/hud/environment/tests/test_arguments.py b/tests/environment/test_arguments.py similarity index 100% rename from hud/environment/tests/test_arguments.py rename to tests/environment/test_arguments.py diff --git a/hud/environment/tests/test_capability_backing.py b/tests/environment/test_capability_backing.py similarity index 100% rename from hud/environment/tests/test_capability_backing.py rename to tests/environment/test_capability_backing.py diff --git a/hud/environment/tests/test_file_tracker.py b/tests/environment/test_file_tracker.py similarity index 100% rename from hud/environment/tests/test_file_tracker.py rename to tests/environment/test_file_tracker.py diff --git a/hud/environment/tests/test_file_tracking.py b/tests/environment/test_file_tracking.py similarity index 100% rename from hud/environment/tests/test_file_tracking.py rename to tests/environment/test_file_tracking.py diff --git a/hud/environment/tests/test_loader.py b/tests/environment/test_loader.py similarity index 100% rename from hud/environment/tests/test_loader.py rename to tests/environment/test_loader.py diff --git a/hud/environment/tests/test_manifest.py b/tests/environment/test_manifest.py similarity index 100% rename from hud/environment/tests/test_manifest.py rename to tests/environment/test_manifest.py diff --git a/hud/environment/tests/test_platform.py b/tests/environment/test_platform.py similarity index 100% rename from hud/environment/tests/test_platform.py rename to tests/environment/test_platform.py diff --git a/hud/environment/tests/test_process_connection.py b/tests/environment/test_process_connection.py similarity index 99% rename from hud/environment/tests/test_process_connection.py rename to tests/environment/test_process_connection.py index d98304272..3c023a451 100644 --- a/hud/environment/tests/test_process_connection.py +++ b/tests/environment/test_process_connection.py @@ -15,7 +15,7 @@ from hud.capabilities import Connection, SSHClient from hud.clients import connect -from hud.environment import Environment, Peer +from hud.environment import Environment, Peer, process_guard from hud.environment.egress import ANY_HOST, BRIDGE_PORT from hud.environment.process_guard import ProcessConnectionGuard, process_connections_supported from hud.environment.workspace import usable_bwrap @@ -26,7 +26,7 @@ reason="process connection guards are unavailable", ) -_GUARD_PATH = Path(__file__).parents[1] / "process_guard.py" +_GUARD_PATH = Path(process_guard.__file__) _PTRACE_SUPPORTED = ( sys.platform == "linux" and subprocess.run( diff --git a/hud/environment/tests/test_robot_control.py b/tests/environment/test_robot_control.py similarity index 100% rename from hud/environment/tests/test_robot_control.py rename to tests/environment/test_robot_control.py diff --git a/hud/environment/tests/test_robot_regressions.py b/tests/environment/test_robot_regressions.py similarity index 100% rename from hud/environment/tests/test_robot_regressions.py rename to tests/environment/test_robot_regressions.py diff --git a/hud/environment/tests/test_server.py b/tests/environment/test_server.py similarity index 100% rename from hud/environment/tests/test_server.py rename to tests/environment/test_server.py diff --git a/hud/environment/tests/test_sessions.py b/tests/environment/test_sessions.py similarity index 100% rename from hud/environment/tests/test_sessions.py rename to tests/environment/test_sessions.py diff --git a/hud/environment/tests/test_tunnel.py b/tests/environment/test_tunnel.py similarity index 100% rename from hud/environment/tests/test_tunnel.py rename to tests/environment/test_tunnel.py diff --git a/hud/environment/tests/test_workspace.py b/tests/environment/test_workspace.py similarity index 100% rename from hud/environment/tests/test_workspace.py rename to tests/environment/test_workspace.py diff --git a/hud/tests/__init__.py b/tests/eval/__init__.py similarity index 100% rename from hud/tests/__init__.py rename to tests/eval/__init__.py diff --git a/hud/eval/tests/test_chat.py b/tests/eval/test_chat.py similarity index 100% rename from hud/eval/tests/test_chat.py rename to tests/eval/test_chat.py diff --git a/hud/eval/tests/test_docker_provider.py b/tests/eval/test_docker_provider.py similarity index 100% rename from hud/eval/tests/test_docker_provider.py rename to tests/eval/test_docker_provider.py diff --git a/hud/eval/tests/test_file_tracking_observer.py b/tests/eval/test_file_tracking_observer.py similarity index 100% rename from hud/eval/tests/test_file_tracking_observer.py rename to tests/eval/test_file_tracking_observer.py diff --git a/hud/eval/tests/test_hosted.py b/tests/eval/test_hosted.py similarity index 100% rename from hud/eval/tests/test_hosted.py rename to tests/eval/test_hosted.py diff --git a/hud/eval/tests/test_job.py b/tests/eval/test_job.py similarity index 100% rename from hud/eval/tests/test_job.py rename to tests/eval/test_job.py diff --git a/hud/eval/tests/test_local_runtime.py b/tests/eval/test_local_runtime.py similarity index 100% rename from hud/eval/tests/test_local_runtime.py rename to tests/eval/test_local_runtime.py diff --git a/hud/eval/tests/test_rollout.py b/tests/eval/test_rollout.py similarity index 100% rename from hud/eval/tests/test_rollout.py rename to tests/eval/test_rollout.py diff --git a/hud/eval/tests/test_shared.py b/tests/eval/test_shared.py similarity index 100% rename from hud/eval/tests/test_shared.py rename to tests/eval/test_shared.py diff --git a/hud/eval/tests/test_sync.py b/tests/eval/test_sync.py similarity index 100% rename from hud/eval/tests/test_sync.py rename to tests/eval/test_sync.py diff --git a/hud/eval/tests/test_task.py b/tests/eval/test_task.py similarity index 100% rename from hud/eval/tests/test_task.py rename to tests/eval/test_task.py diff --git a/hud/utils/tests/__init__.py b/tests/integrations/__init__.py similarity index 100% rename from hud/utils/tests/__init__.py rename to tests/integrations/__init__.py diff --git a/tests/integrations/harbor/__init__.py b/tests/integrations/harbor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hud/integrations/harbor/tests/conftest.py b/tests/integrations/harbor/conftest.py similarity index 100% rename from hud/integrations/harbor/tests/conftest.py rename to tests/integrations/harbor/conftest.py diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/environment/Dockerfile b/tests/integrations/harbor/tasks/agent-lifecycle/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/environment/Dockerfile rename to tests/integrations/harbor/tasks/agent-lifecycle/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/instruction.md b/tests/integrations/harbor/tasks/agent-lifecycle/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/instruction.md rename to tests/integrations/harbor/tasks/agent-lifecycle/instruction.md diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/solution/solve.sh b/tests/integrations/harbor/tasks/agent-lifecycle/solution/solve.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/solution/solve.sh rename to tests/integrations/harbor/tasks/agent-lifecycle/solution/solve.sh diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/task.toml b/tests/integrations/harbor/tasks/agent-lifecycle/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/task.toml rename to tests/integrations/harbor/tasks/agent-lifecycle/task.toml diff --git a/hud/integrations/harbor/tests/tasks/agent-lifecycle/tests/test.sh b/tests/integrations/harbor/tasks/agent-lifecycle/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/agent-lifecycle/tests/test.sh rename to tests/integrations/harbor/tasks/agent-lifecycle/tests/test.sh diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/Dockerfile b/tests/integrations/harbor/tasks/hello-mcp/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/Dockerfile rename to tests/integrations/harbor/tasks/hello-mcp/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/hello-mcp/environment/docker-compose.yaml similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/docker-compose.yaml rename to tests/integrations/harbor/tasks/hello-mcp/environment/docker-compose.yaml diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/Dockerfile b/tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/Dockerfile rename to tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/Dockerfile diff --git a/tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/__init__.py b/tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/server.py b/tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/server.py similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/environment/mcp-server/server.py rename to tests/integrations/harbor/tasks/hello-mcp/environment/mcp-server/server.py diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/instruction.md b/tests/integrations/harbor/tasks/hello-mcp/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/instruction.md rename to tests/integrations/harbor/tasks/hello-mcp/instruction.md diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/solution/solve.sh b/tests/integrations/harbor/tasks/hello-mcp/solution/solve.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/solution/solve.sh rename to tests/integrations/harbor/tasks/hello-mcp/solution/solve.sh diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/task.toml b/tests/integrations/harbor/tasks/hello-mcp/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/task.toml rename to tests/integrations/harbor/tasks/hello-mcp/task.toml diff --git a/hud/integrations/harbor/tests/tasks/hello-mcp/tests/test.sh b/tests/integrations/harbor/tasks/hello-mcp/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/hello-mcp/tests/test.sh rename to tests/integrations/harbor/tasks/hello-mcp/tests/test.sh diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/environment/Dockerfile b/tests/integrations/harbor/tasks/phase-boundary/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/environment/Dockerfile rename to tests/integrations/harbor/tasks/phase-boundary/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/environment/entrypoint.sh b/tests/integrations/harbor/tasks/phase-boundary/environment/entrypoint.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/environment/entrypoint.sh rename to tests/integrations/harbor/tasks/phase-boundary/environment/entrypoint.sh diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/instruction.md b/tests/integrations/harbor/tasks/phase-boundary/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/instruction.md rename to tests/integrations/harbor/tasks/phase-boundary/instruction.md diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/solution/solve.sh b/tests/integrations/harbor/tasks/phase-boundary/solution/solve.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/solution/solve.sh rename to tests/integrations/harbor/tasks/phase-boundary/solution/solve.sh diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/task.toml b/tests/integrations/harbor/tasks/phase-boundary/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/task.toml rename to tests/integrations/harbor/tasks/phase-boundary/task.toml diff --git a/hud/integrations/harbor/tests/tasks/phase-boundary/tests/test.sh b/tests/integrations/harbor/tasks/phase-boundary/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/phase-boundary/tests/test.sh rename to tests/integrations/harbor/tasks/phase-boundary/tests/test.sh diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/Dockerfile b/tests/integrations/harbor/tasks/sidecar-reachability/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/Dockerfile rename to tests/integrations/harbor/tasks/sidecar-reachability/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/sidecar-reachability/environment/docker-compose.yaml similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/environment/docker-compose.yaml rename to tests/integrations/harbor/tasks/sidecar-reachability/environment/docker-compose.yaml diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/instruction.md b/tests/integrations/harbor/tasks/sidecar-reachability/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/instruction.md rename to tests/integrations/harbor/tasks/sidecar-reachability/instruction.md diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh b/tests/integrations/harbor/tasks/sidecar-reachability/solution/solve.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/solution/solve.sh rename to tests/integrations/harbor/tasks/sidecar-reachability/solution/solve.sh diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml b/tests/integrations/harbor/tasks/sidecar-reachability/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/task.toml rename to tests/integrations/harbor/tasks/sidecar-reachability/task.toml diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/Dockerfile b/tests/integrations/harbor/tasks/sidecar-reachability/tests/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/Dockerfile rename to tests/integrations/harbor/tasks/sidecar-reachability/tests/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/test.sh b/tests/integrations/harbor/tasks/sidecar-reachability/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/sidecar-reachability/tests/test.sh rename to tests/integrations/harbor/tasks/sidecar-reachability/tests/test.sh diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-lifecycle/environment/Dockerfile similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/environment/Dockerfile rename to tests/integrations/harbor/tasks/verifier-lifecycle/environment/Dockerfile diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/environment/entrypoint.sh b/tests/integrations/harbor/tasks/verifier-lifecycle/environment/entrypoint.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/environment/entrypoint.sh rename to tests/integrations/harbor/tasks/verifier-lifecycle/environment/entrypoint.sh diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/instruction.md b/tests/integrations/harbor/tasks/verifier-lifecycle/instruction.md similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/instruction.md rename to tests/integrations/harbor/tasks/verifier-lifecycle/instruction.md diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-lifecycle/solution/solve.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/solution/solve.sh rename to tests/integrations/harbor/tasks/verifier-lifecycle/solution/solve.sh diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/task.toml b/tests/integrations/harbor/tasks/verifier-lifecycle/task.toml similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/task.toml rename to tests/integrations/harbor/tasks/verifier-lifecycle/task.toml diff --git a/hud/integrations/harbor/tests/tasks/verifier-lifecycle/tests/test.sh b/tests/integrations/harbor/tasks/verifier-lifecycle/tests/test.sh similarity index 100% rename from hud/integrations/harbor/tests/tasks/verifier-lifecycle/tests/test.sh rename to tests/integrations/harbor/tasks/verifier-lifecycle/tests/test.sh diff --git a/hud/integrations/harbor/tests/test_contract.py b/tests/integrations/harbor/test_contract.py similarity index 99% rename from hud/integrations/harbor/tests/test_contract.py rename to tests/integrations/harbor/test_contract.py index 2e16c960f..32998c7e5 100644 --- a/hud/integrations/harbor/tests/test_contract.py +++ b/tests/integrations/harbor/test_contract.py @@ -1610,7 +1610,7 @@ def test_adapt_hashes_links_not_their_targets( def test_authored_runtime_assets_are_valid_source() -> None: - integration = Path(__file__).parents[1] + integration = Path(harbor.__file__).parent compile((integration / "env.py").read_text("utf-8"), "env.py", "exec") installer = (integration / "install.sh").read_text("utf-8") assert "python_version=3.12" in installer diff --git a/hud/integrations/harbor/tests/test_harbor.py b/tests/integrations/harbor/test_harbor.py similarity index 100% rename from hud/integrations/harbor/tests/test_harbor.py rename to tests/integrations/harbor/test_harbor.py diff --git a/hud/integrations/harbor/tests/test_integration.py b/tests/integrations/harbor/test_integration.py similarity index 99% rename from hud/integrations/harbor/tests/test_integration.py rename to tests/integrations/harbor/test_integration.py index 48ec0daf7..f97a0ce6b 100644 --- a/hud/integrations/harbor/tests/test_integration.py +++ b/tests/integrations/harbor/test_integration.py @@ -28,7 +28,7 @@ from hud.eval.run import Run TASKS = Path(__file__).parent / "tasks" -REPO = Path(__file__).resolve().parents[4] +REPO = Path(__file__).resolve().parents[3] pytestmark = [ pytest.mark.integration, diff --git a/hud/patches/tests/__init__.py b/tests/patches/__init__.py similarity index 100% rename from hud/patches/tests/__init__.py rename to tests/patches/__init__.py diff --git a/hud/patches/tests/test_warnings.py b/tests/patches/test_warnings.py similarity index 96% rename from hud/patches/tests/test_warnings.py rename to tests/patches/test_warnings.py index 5fca3356b..820b2c369 100644 --- a/hud/patches/tests/test_warnings.py +++ b/tests/patches/test_warnings.py @@ -25,7 +25,8 @@ import subprocess import sys -from pathlib import Path + +from hud.patches import warnings as warnings_module # Loads the suppression helper in a pristine interpreter without importing the rest # of ``hud`` (which would register unrelated filters). A custom showwarning() runs @@ -85,9 +86,8 @@ def _run_pristine(code: str) -> subprocess.CompletedProcess[str]: - warnings_module = Path(__file__).resolve().parents[1] / "warnings.py" return subprocess.run( - [sys.executable, "-c", code, str(warnings_module)], + [sys.executable, "-c", code, str(warnings_module.__file__)], capture_output=True, text=True, timeout=60, diff --git a/tests/telemetry/__init__.py b/tests/telemetry/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hud/telemetry/tests/test_exporter.py b/tests/telemetry/test_exporter.py similarity index 100% rename from hud/telemetry/tests/test_exporter.py rename to tests/telemetry/test_exporter.py diff --git a/hud/telemetry/tests/test_instrument.py b/tests/telemetry/test_instrument.py similarity index 100% rename from hud/telemetry/tests/test_instrument.py rename to tests/telemetry/test_instrument.py diff --git a/hud/telemetry/tests/test_robot_video.py b/tests/telemetry/test_robot_video.py similarity index 100% rename from hud/telemetry/tests/test_robot_video.py rename to tests/telemetry/test_robot_video.py diff --git a/hud/tests/test_graders.py b/tests/test_graders.py similarity index 100% rename from hud/tests/test_graders.py rename to tests/test_graders.py diff --git a/hud/tests/test_init.py b/tests/test_init.py similarity index 100% rename from hud/tests/test_init.py rename to tests/test_init.py diff --git a/hud/tests/test_init_module.py b/tests/test_init_module.py similarity index 100% rename from hud/tests/test_init_module.py rename to tests/test_init_module.py diff --git a/hud/tests/test_robot.py b/tests/test_robot.py similarity index 100% rename from hud/tests/test_robot.py rename to tests/test_robot.py diff --git a/hud/tests/test_settings.py b/tests/test_settings.py similarity index 100% rename from hud/tests/test_settings.py rename to tests/test_settings.py diff --git a/hud/tests/test_trace.py b/tests/test_trace.py similarity index 100% rename from hud/tests/test_trace.py rename to tests/test_trace.py diff --git a/hud/tests/test_types.py b/tests/test_types.py similarity index 100% rename from hud/tests/test_types.py rename to tests/test_types.py diff --git a/hud/tests/test_version.py b/tests/test_version.py similarity index 100% rename from hud/tests/test_version.py rename to tests/test_version.py diff --git a/hud/train/tests/__init__.py b/tests/train/__init__.py similarity index 100% rename from hud/train/tests/__init__.py rename to tests/train/__init__.py diff --git a/hud/train/tests/test_client.py b/tests/train/test_client.py similarity index 100% rename from hud/train/tests/test_client.py rename to tests/train/test_client.py diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hud/utils/tests/test_exceptions.py b/tests/utils/test_exceptions.py similarity index 100% rename from hud/utils/tests/test_exceptions.py rename to tests/utils/test_exceptions.py diff --git a/hud/utils/tests/test_gateway.py b/tests/utils/test_gateway.py similarity index 100% rename from hud/utils/tests/test_gateway.py rename to tests/utils/test_gateway.py diff --git a/hud/utils/tests/test_hints.py b/tests/utils/test_hints.py similarity index 100% rename from hud/utils/tests/test_hints.py rename to tests/utils/test_hints.py diff --git a/hud/utils/tests/test_hud_console.py b/tests/utils/test_hud_console.py similarity index 100% rename from hud/utils/tests/test_hud_console.py rename to tests/utils/test_hud_console.py diff --git a/hud/utils/tests/test_naming.py b/tests/utils/test_naming.py similarity index 100% rename from hud/utils/tests/test_naming.py rename to tests/utils/test_naming.py diff --git a/hud/utils/tests/test_platform.py b/tests/utils/test_platform.py similarity index 100% rename from hud/utils/tests/test_platform.py rename to tests/utils/test_platform.py diff --git a/hud/utils/tests/test_process.py b/tests/utils/test_process.py similarity index 100% rename from hud/utils/tests/test_process.py rename to tests/utils/test_process.py diff --git a/hud/utils/tests/test_requests.py b/tests/utils/test_requests.py similarity index 100% rename from hud/utils/tests/test_requests.py rename to tests/utils/test_requests.py diff --git a/hud/utils/tests/test_serialization.py b/tests/utils/test_serialization.py similarity index 100% rename from hud/utils/tests/test_serialization.py rename to tests/utils/test_serialization.py From 473ffa3336cdc481bed50f095e7df09773c88ca3 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:07:18 -0700 Subject: [PATCH 002/248] test: add a black-box harness and make tests black-box by policy Tests drive the SDK through its public boundaries and fake only what lies outside it. tests/harness provides those fakes: one local server standing in for the HUD API, telemetry, gateway, runtime and training services; scripted model providers speaking Chat Completions, Responses, Anthropic Messages and Gemini; the real hud CLI in a subprocess; a docker executable on PATH; and a VNC screen. Every test starts from an isolated configuration set through environment variables, with service URLs pointed at a closed port so nothing reaches a real deployment. AGENTS.md, CONTRIBUTING.md and a new REVIEW.md state the rules, and scripts/check_tests.py enforces them in CI. Tests needing Docker, a Linux sandbox, or a real HUD account are marked e2e and run in their own CI lanes. --- .github/workflows/ci.yml | 5 +- .github/workflows/e2e.yml | 83 ++++ AGENTS.md | 55 ++- CONTRIBUTING.md | 72 ++- REVIEW.md | 34 ++ pyproject.toml | 13 +- scripts/check_tests.py | 122 +++++ tests/conftest.py | 150 +++++- tests/harness/__init__.py | 45 ++ tests/harness/cli.py | 71 +++ tests/harness/config.py | 88 ++++ tests/harness/docker.py | 102 ++++ tests/harness/models.py | 451 ++++++++++++++++++ tests/harness/rfb.py | 108 +++++ tests/harness/scenario.py | 83 ++++ tests/harness/services.py | 298 ++++++++++++ tests/harness/spans.py | 38 ++ tests/harness/test_fakes.py | 249 ++++++++++ tests/integrations/harbor/test_integration.py | 3 +- 19 files changed, 2002 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/e2e.yml create mode 100644 REVIEW.md create mode 100644 scripts/check_tests.py create mode 100644 tests/harness/__init__.py create mode 100644 tests/harness/cli.py create mode 100644 tests/harness/config.py create mode 100644 tests/harness/docker.py create mode 100644 tests/harness/models.py create mode 100644 tests/harness/rfb.py create mode 100644 tests/harness/scenario.py create mode 100644 tests/harness/services.py create mode 100644 tests/harness/spans.py create mode 100644 tests/harness/test_fakes.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee2e8dd62..419c8c1d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: run: uv python install ${{ matrix.python-version }} - name: Run tests - run: uv run --python ${{ matrix.python-version }} --all-extras pytest --cov --cov-report='' + run: uv run --python ${{ matrix.python-version }} --all-extras pytest -n auto --cov --cov-report='' lint-ruff: runs-on: ubuntu-latest @@ -41,6 +41,9 @@ jobs: uv run --with=".[dev]" ruff format . --check uv run --with=".[dev]" ruff check . + - name: Check tests stay black-box + run: uv run --no-project python scripts/check_tests.py + lint-ty: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..44c403e8d --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,83 @@ +name: E2E + +# Tests marked `e2e` need something outside the test process. Each job selects +# the ones its runner can provide; they skip themselves when it cannot. + +on: + pull_request: + paths: + - "hud/**" + - "tests/**" + - "environments/**" + - "pyproject.toml" + - ".github/workflows/e2e.yml" + push: + branches: [main] + schedule: + - cron: "17 7 * * *" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + docker: + name: Docker + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + - name: Run Docker scenarios + run: >- + uv run --python 3.12 --all-extras + pytest -m "e2e and docker" -n 4 --timeout 1800 --durations 20 + + sandbox: + name: Sandbox + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + # Environments run in containers with HUD's seccomp profile, so the + # sandbox scenarios run there too, as root, with bubblewrap installed. + - name: Run sandbox scenarios in a container + run: >- + docker run --rm + --security-opt seccomp=hud/eval/docker-seccomp.json + --security-opt systempaths=unconfined + --security-opt apparmor=unconfined + --volume "$PWD:/src" --workdir /src + ghcr.io/astral-sh/uv:python3.12-bookworm + sh -c 'apt-get update -qq && apt-get install -y -qq bubblewrap openssh-client >/dev/null + && uv run --all-extras pytest -m "e2e and sandbox" -n 4 --timeout 600' + + live: + name: Live + # Secrets are not available to pull requests from forks or Dependabot. + if: >- + github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login != 'dependabot[bot]') + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + HUD_API_KEY: ${{ secrets.HUD_API_KEY }} + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + - name: Run live scenarios + run: >- + uv run --python 3.12 --all-extras + pytest -m "e2e and live and not hosted" -n 4 --timeout 1200 --durations 20 diff --git a/AGENTS.md b/AGENTS.md index bfca8c0b0..cf56d9b88 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,14 +55,15 @@ Use the commands in `CONTRIBUTING.md` as the source of truth. Common commands: ```bash uv sync --extra dev -uv run pytest -q uv run ruff format . --check uv run ruff check . uv run --extra dev --extra train ty check +uv run python scripts/check_tests.py +uv run pytest -n auto ``` -The shared pre-push hook lives in `.githooks/pre-push`, but agents should not -change local git config unless explicitly asked. +Locally, ruff and ty are the loop; CI runs the test suite. Run the tests you +touched, and leave the full suite and the `e2e` lanes to CI. Tests run on Python 3.11 and 3.12 in CI. `pyproject.toml` currently supports Python `>=3.11, <3.13`. @@ -121,27 +122,33 @@ Python `>=3.11, <3.13`. - Use `TYPE_CHECKING` imports for type-only imports that would otherwise add runtime dependency cost or cycles. -## Testing Expectations - -- Add or update focused tests for behavior changes. Put them in the `tests/` - package that mirrors the module they cover. -- Test behavior and contracts, not private implementation details. -- Regression tests should fail on the old behavior through the normal lifecycle - or public boundary. Do not manually seed private state such as internal maps, - caches, cursors, or prepared containers just to prove a changed line. -- If a bug involves internal state, reach it through real setup and execution: - construction, configuration, preparation, run loop, provider response, tool - execution, or public API call. -- Do not add hooks, helper methods, or abstraction layers only to make tests - easier. If a test needs that, reconsider the behavior boundary instead. -- Test names should describe the observable behavior or contract, not the - private mechanism. -- Mock external services, provider APIs, network, Docker, browser, and filesystem - boundaries as needed. Do not mock core logic just to make a test easy. -- Mark tests that require `HUD_API_KEY`, network access, or deployed services as - integration tests. -- Run the narrowest relevant tests first, then broader checks when the blast - radius is shared or user-facing. +## Testing + +Tests are black-box: they drive the SDK through a public boundary and fake only +what lies outside it. The harness in `tests/harness` holds those fakes: the HUD +services, scripted model providers, a `docker` executable, a VNC screen, and the +real `hud` CLI run in a subprocess. `CONTRIBUTING.md` shows how to use it. + +- Enter through a public boundary: the `hud` CLI, an exported name, the control + channel or another wire protocol, a file format, or a process effect. +- Fake only the outside world: HUD services, model providers, Docker, cloud SDKs, + the clock. Never patch a `hud` name, import a private one, or touch a private + attribute; `scripts/check_tests.py` enforces this in CI. Configure the SDK the + way a user does, through environment variables (`hud_env.set(...)`). +- Never hand-build what a producer emits. Run the producer, so the test covers + the seam between it and its consumer. +- Assert what a user or a peer observes: the reward, run and trace fields, + output and exit codes, files, request bodies, wire frames. Prefer fixture + environments whose reward proves the path ran. +- Write a scenario once per contract, as a table; a new case is a new row. Use + inline snapshots only where a whole output is the contract: request bodies the + platform parses, `--json` documents, docker and cloud-SDK transcripts. +- Do not add tests to check your own change. Check it with a temporary script, + and extend a scenario only when the change adds or fixes a contract; a bug fix + lands with the scenario row that failed before it. +- Tests needing something outside the process are marked `e2e` plus `docker`, + `sandbox`, `live` (a real HUD account) or `hosted`, and run only when selected. +- Test names describe the observable behavior, not the mechanism. ## Operational Debugging diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 92ea35ed1..7573bc633 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,31 +14,72 @@ We welcome contributions to the HUD SDK! This guide covers how to get started. ## Development Workflow -### Git Hooks - -Enable the shared pre-push hook (runs ruff, ty, pytest before each push): - -```bash -git config core.hooksPath .githooks -``` - ### Running Tests +CI runs the full suite; locally, run what you touched: + ```bash -uv run pytest -q +uv run pytest tests/cli -n auto # one area, in parallel +uv run pytest -n auto # everything that needs nothing outside the process ``` Tests run on Python 3.11 and 3.12 in CI. -Integration tests are excluded by default because they may use external -services, network access, or container builds. Run them explicitly: +Tests that need something outside the process are marked `e2e` and run only +when selected. They skip themselves when what they need is missing: ```bash -uv run pytest -m integration +uv run pytest -m "e2e and docker" # a running Docker daemon +uv run pytest -m "e2e and sandbox" # Linux, root and bubblewrap +uv run pytest -m "e2e and live" # a real HUD account: HUD_API_KEY; spends credits +uv run pytest -m "e2e and hosted" # deploys and runs on the HUD platform +``` + +### Writing Tests + +Tests are black-box: they drive the SDK through a public boundary and fake only +what lies outside it. `tests/harness` holds the fakes, and `tests/conftest.py` +provides them as fixtures: + +- `hud_env`: the SDK's configuration. Every test starts with a temporary home, + no credentials, and service URLs pointed at a closed port; `hud_env.set(...)` + changes environment variables the way a user would. +- `services`: a local stand-in for the HUD API, telemetry, gateway, runtime and + training services. Declare replies with `services.route(...)` and read what + the SDK sent with `services.requests(...)`; anything undeclared answers 404. +- `models`: scripted model providers behind the fake gateway, speaking OpenAI + Chat Completions and Responses, Anthropic Messages and Gemini. Script turns + with `models.script([call("bash", command="ls"), say("done")])`. +- `hud`: the real `hud` CLI in a subprocess. `hud("jobs", "list", "--json")` + returns the exit code, stdout and stderr. +- `fake_docker`: a `docker` executable on `PATH` that logs every invocation and + answers from rules. +- `ScriptedAgent`, `RecordingProvider`, `served`, `fake_screen`, and `steps` for + agents with fixed answers, observed placements, served environments, a VNC + screen, and the step spans a run exported. + +For example: + +```python +async def test_a_reporting_rollout_reaches_the_platform(services, hud_env): + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + services.route("api", "POST", "/v2/trace/{id}/exit", json={}) + + job = await Taskset("demo", [row]).run(ScriptedAgent("3"), runtime=LocalRuntime(env)) + + assert job.reward == 1.0 + assert services.bodies("api", "POST", "/v2/trace/{id}/exit") == snapshot(...) ``` -The Harbor integration tests require Docker and may take several minutes with -a cold build cache. +Rules, enforced by `scripts/check_tests.py` and in review: + +- Never patch a `hud` name, import a private one, or touch a private attribute. +- Never hand-build what a producer emits; run the producer. +- Write one scenario per contract as a table, and add a row for a new case. +- Use inline snapshots (`inline-snapshot`) only where a whole output is the + contract. Record them with `uv run pytest --inline-snapshot=create` and review + every change to one like any other diff. +- Do not commit a test to check your own change; use a temporary script. ### Code Quality @@ -46,6 +87,7 @@ a cold build cache. uv run ruff format . --check # Formatting uv run ruff check . # Linting uv run --extra dev --extra train ty check # Type checking +uv run python scripts/check_tests.py # Tests stay black-box ``` ## Code Style @@ -59,7 +101,7 @@ uv run --extra dev --extra train ty check # Type checking 1. **Branch naming**: `feature/description` or `fix/issue-number` 2. **Commits**: Use clear, descriptive messages -3. **Tests**: All CI checks must pass (ruff, ty, pytest) +3. **Tests**: All CI checks must pass (ruff, ty, the test policy check, pytest) 4. **Review**: Address feedback promptly ## Releases diff --git a/REVIEW.md b/REVIEW.md new file mode 100644 index 000000000..e7283faee --- /dev/null +++ b/REVIEW.md @@ -0,0 +1,34 @@ +# Reviewing hud-python + +Review adversarially: assume the change breaks something and find where. The +tests cover contracts at public boundaries, so the review is where a broken +seam, a missing scenario, or an unintended behavior change gets caught. + +## Behavior + +- Trace each changed contract from a real producer to a real consumer. Ask who + calls the code, with what inputs, and what they do with its output. +- Look for inputs a real producer emits that the change mishandles, and for + outputs the change emits that a real consumer cannot read. +- When you find a break, give the scenario row that fails because of it. + +## Tests + +The rules are under "Testing" in `AGENTS.md`. Treat each of these as a finding: + +- A test that patches a `hud` name, reads a private attribute, hand-builds what + a producer emits, or asserts that a mock was called the way it was set up. +- A bug fix without a scenario row that fails before the fix. +- A changed inline snapshot that the change does not explain. Every snapshot + change is a behavior change; one updated just to make a test pass hides one. +- A new test that only re-checks the author's change without adding a contract. + That belongs in a temporary script. +- A scenario that could pass with the code under test deleted. + +## Isolation and credentials + +For changes to `hud/environment`, the runtimes, egress, or Harbor, ask what an +agent inside the sandbox could now do: read the host's credentials, reach the +control channel, run code outside the sandbox, or keep a process alive past +teardown. Each new angle becomes a row in the sandbox scenarios, scored 1 only +when the attempt is denied. diff --git a/pyproject.toml b/pyproject.toml index 070bc0e8b..2fcf5be2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,6 +134,10 @@ dev = [ "pytest-asyncio", "pytest-mock", "pytest-cov", + "pytest-xdist", + "pytest-timeout", + "inline-snapshot", + "dirty-equals", "ty==0.0.67", ] @@ -263,7 +267,12 @@ fail_under = 58 asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" testpaths = ["tests"] -addopts = "-m 'not integration'" +addopts = "-m 'not e2e'" +timeout = 120 markers = [ - "integration: marks tests as integration tests (require HUD_API_KEY, network access, or Docker)", + "e2e: needs something outside the test process; runs only when selected with -m", + "docker: needs a running Docker daemon", + "sandbox: needs Linux, root, and bubblewrap", + "live: needs a real HUD account (HUD_API_KEY) and spends credits", + "hosted: deploys to and runs on the HUD platform", ] diff --git a/scripts/check_tests.py b/scripts/check_tests.py new file mode 100644 index 000000000..0f7cebfcd --- /dev/null +++ b/scripts/check_tests.py @@ -0,0 +1,122 @@ +"""Fail when a test reaches into hud internals. + +Tests drive the SDK through its public boundaries and fake only what lies outside +it (see "Testing" in AGENTS.md). This flags, in test modules outside the harness +(``tests/harness``, the one place allowed to configure the SDK): + +- patching a hud name: ``monkeypatch.setattr``, ``patch``, ``patch.object`` or + ``mocker.patch`` aimed at a ``"hud..."`` path or at anything imported from hud; +- importing a private name or module from hud; +- touching a private attribute of anything other than ``self`` or ``cls``. + +Usage: uv run python scripts/check_tests.py [path ...] +""" + +from __future__ import annotations + +import ast +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_PATHS = [ROOT / "tests", *sorted(ROOT.glob("environments/*/tests"))] +HARNESS = ROOT / "tests" / "harness" +PATCHERS = {"setattr", "delattr", "patch"} +PATCH_METHODS = {"object", "dict", "multiple"} + + +def _private(name: str) -> bool: + return name.startswith("_") and not (name.startswith("__") and name.endswith("__")) + + +def _is_patch(func: ast.expr) -> bool: + """``setattr``, ``patch`` and ``monkeypatch.setattr`` forms, and ``patch.object`` forms.""" + if isinstance(func, ast.Name): + return func.id in PATCHERS + if not isinstance(func, ast.Attribute): + return False + if func.attr in PATCHERS: + return True + owner = func.value + owner_name = owner.attr if isinstance(owner, ast.Attribute) else getattr(owner, "id", "") + return func.attr in PATCH_METHODS and owner_name == "patch" + + +def _root(node: ast.expr) -> ast.expr: + while isinstance(node, (ast.Attribute, ast.Subscript, ast.Call)): + node = node.func if isinstance(node, ast.Call) else node.value + return node + + +class Checker(ast.NodeVisitor): + def __init__(self, path: Path) -> None: + self.path = path + self.hud_names: set[str] = set() + self.problems: list[str] = [] + + def report(self, node: ast.AST, message: str) -> None: + line = getattr(node, "lineno", 0) + self.problems.append(f"{self.path.relative_to(ROOT)}:{line}: {message}") + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + if alias.name == "hud" or alias.name.startswith("hud."): + self.hud_names.add((alias.asname or alias.name).split(".")[0]) + if any(_private(part) for part in alias.name.split(".")): + self.report(node, f"imports private module {alias.name}") + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + module = node.module or "" + if module == "hud" or module.startswith("hud."): + private_module = any(_private(part) for part in module.split(".")) + for alias in node.names: + self.hud_names.add(alias.asname or alias.name) + if private_module or _private(alias.name): + self.report(node, f"imports private {module}.{alias.name}") + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + if _is_patch(node.func) and node.args: + target = node.args[0] + if isinstance(target, ast.Constant) and isinstance(target.value, str): + if target.value == "hud" or target.value.startswith("hud."): + self.report(node, f"patches {target.value}") + else: + root = _root(target) + if isinstance(root, ast.Name) and root.id in self.hud_names: + self.report(node, f"patches {ast.unparse(target)}, which comes from hud") + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute) -> None: + if _private(node.attr): + root = _root(node) + if not (isinstance(root, ast.Name) and root.id in {"self", "cls"}): + self.report(node, f"touches private attribute {ast.unparse(node)}") + self.generic_visit(node) + + +def check(path: Path) -> list[str]: + checker = Checker(path) + checker.visit(ast.parse(path.read_text("utf-8"), filename=str(path))) + return checker.problems + + +def main(argv: list[str]) -> int: + roots = [Path(arg).resolve() for arg in argv] or DEFAULT_PATHS + files = sorted( + file + for root in roots + for file in ([root] if root.is_file() else root.rglob("*.py")) + if HARNESS not in file.parents and "tasks" not in file.relative_to(ROOT).parts + ) + problems = [problem for file in files for problem in check(file)] + for problem in problems: + print(problem) + if problems: + print(f"\n{len(problems)} reach(es) into hud internals; see Testing in AGENTS.md.") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/conftest.py b/tests/conftest.py index ab9d4b705..20dbd14a8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,36 +1,136 @@ -"""Root test fixtures: isolate unit tests from developer settings. +"""Fixtures wiring the black-box harness (``tests/harness``) into every test. -Without this, any test that exercises ``Taskset.run`` (or other platform -reporting paths) makes real HTTP calls to the platform whenever the developer -has ``HUD_API_KEY`` configured — spamming the platform with fake jobs/traces -and stalling the suite on network retries. CI never catches it because CI has -no API key. +Every test runs against an isolated SDK configuration: a temporary home, no +developer credentials, telemetry uploads off, and every HUD service URL pointed +at a closed local port. A test that talks to HUD services asks for the +``services`` fixture, which points the SDK at the fake backend instead. + +Tests marked ``e2e`` need something outside the process (a Docker daemon, a +Linux sandbox, a real HUD account) and only run when selected with ``-m``; +they skip themselves when what they need is missing. """ from __future__ import annotations +import os +import shutil +import subprocess +import sys from pathlib import Path +from typing import TYPE_CHECKING import pytest +from dotenv import dotenv_values + +from tests.harness import FakeDocker, FakeServices, Hud, HudEnv, Models + +if TYPE_CHECKING: + from collections.abc import Iterator + +# The developer's or CI's own environment, captured before any test isolates it. +# Only ``live`` tests read credentials from it. +REAL_ENVIRONMENT = dict(os.environ) + +SERVICE_URL_VARIABLES = ( + "HUD_API_URL", + "HUD_TELEMETRY_URL", + "HUD_GATEWAY_URL", + "HUD_RUNTIME_URL", + "HUD_RL_URL", + "HUD_WEB_URL", +) + + +def real_hud_api_key() -> str | None: + """``HUD_API_KEY`` from the real environment or the real ``~/.hud/.env``.""" + if key := REAL_ENVIRONMENT.get("HUD_API_KEY"): + return key + home = REAL_ENVIRONMENT.get("HOME") + if home is None: + return None + return dotenv_values(Path(home) / ".hud" / ".env").get("HUD_API_KEY") @pytest.fixture(autouse=True) -def _isolate_hud_settings(request: pytest.FixtureRequest) -> None: - """Disable telemetry and the API key for unit tests. - - Tests marked ``integration`` keep the real settings (they require - ``HUD_API_KEY`` and network access by contract). - """ - if request.node.get_closest_marker("integration") is not None: - return - - from hud.settings import settings - - mp = pytest.MonkeyPatch() - request.addfinalizer(mp.undo) - mp.setattr(settings, "telemetry_enabled", False) - mp.setenv("HUD_CLI_ANALYTICS_ENABLED", "0") - mp.setattr(settings, "api_key", None) - mp.setattr(settings, "telemetry_local_dir", None) - home = request.getfixturevalue("tmp_path") - mp.setattr(Path, "home", lambda: home) +def hud_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> HudEnv: + """The SDK's configuration for this test; change it with ``hud_env.set(...)``.""" + return HudEnv(monkeypatch, tmp_path) + + +@pytest.fixture(scope="session") +def services_server() -> Iterator[FakeServices]: + server = FakeServices() + server.start() + yield server + server.stop() + + +@pytest.fixture +def services(services_server: FakeServices, hud_env: HudEnv) -> FakeServices: + """The fake HUD backend, with the SDK configured to reach it and no routes declared.""" + services_server.reset() + hud_env.set(**services_server.env()) + return services_server + + +@pytest.fixture +def models(services: FakeServices) -> Models: + """Scripted model providers behind the fake gateway.""" + return Models(services) + + +@pytest.fixture +def hud(tmp_path: Path) -> Hud: + """The real ``hud`` CLI, run in a subprocess from ``tmp_path / "work"``.""" + return Hud(tmp_path / "work") + + +@pytest.fixture +def fake_docker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FakeDocker: + return FakeDocker(tmp_path / "fake-docker", monkeypatch) + + +@pytest.fixture +def live(hud_env: HudEnv) -> HudEnv: + """Configure the SDK for a real HUD account: the real key and the real services.""" + key = real_hud_api_key() + if key is None: + pytest.skip("needs HUD_API_KEY") + hud_env.set( + HUD_API_KEY=key, + **{name: REAL_ENVIRONMENT.get(name) for name in SERVICE_URL_VARIABLES}, + ) + return hud_env + + +def _docker_available() -> bool: + if shutil.which("docker") is None: + return False + try: + probe = subprocess.run(["docker", "info"], capture_output=True, check=False, timeout=20) + except subprocess.TimeoutExpired: + return False + return probe.returncode == 0 + + +def _sandbox_available() -> bool: + return sys.platform == "linux" and os.geteuid() == 0 and shutil.which("bwrap") is not None + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Skip selected ``e2e`` tests whose daemon, sandbox, or credentials are missing.""" + del config + checks = { + "docker": (_docker_available, "needs a running Docker daemon"), + "sandbox": (_sandbox_available, "needs Linux, root, and bubblewrap"), + "live": (lambda: real_hud_api_key() is not None, "needs HUD_API_KEY"), + } + available: dict[str, bool] = {} + for item in items: + for mark, (check, reason) in checks.items(): + if mark not in item.keywords: + continue + if mark not in available: + available[mark] = check() + if not available[mark]: + item.add_marker(pytest.mark.skip(reason=reason)) diff --git a/tests/harness/__init__.py b/tests/harness/__init__.py new file mode 100644 index 000000000..dbb9a67d4 --- /dev/null +++ b/tests/harness/__init__.py @@ -0,0 +1,45 @@ +"""The black-box test harness: fakes of everything outside the SDK. + +Tests drive the SDK through its public boundaries (the ``hud`` CLI, exported +API, wire protocols, files) and fake only what lies outside it: the HUD +services, model providers, docker and VNC servers. The fixtures wiring these in +live in ``tests/conftest.py``. +""" + +from .cli import Hud, Result, scrub +from .config import HudEnv +from .docker import FakeDocker +from .models import ModelRequest, Models, ToolCall, Turn, call, calls, fail, say +from .rfb import FakeScreen, KeyEvent, PointerEvent, fake_screen +from .scenario import RecordingProvider, ScriptedAgent, served, task_row +from .services import FakeServices, Reply, Request +from .spans import spans, steps + +__all__ = [ + "FakeDocker", + "FakeScreen", + "FakeServices", + "Hud", + "HudEnv", + "KeyEvent", + "ModelRequest", + "Models", + "PointerEvent", + "RecordingProvider", + "Reply", + "Request", + "Result", + "ScriptedAgent", + "ToolCall", + "Turn", + "call", + "calls", + "fail", + "fake_screen", + "say", + "scrub", + "served", + "spans", + "steps", + "task_row", +] diff --git a/tests/harness/cli.py b/tests/harness/cli.py new file mode 100644 index 000000000..bbbd47441 --- /dev/null +++ b/tests/harness/cli.py @@ -0,0 +1,71 @@ +"""Run the real ``hud`` CLI in a subprocess, the way a user does.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from pathlib import Path + +UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + + +@dataclass(frozen=True) +class Result: + exit_code: int + stdout: str + stderr: str + + @property + def json(self) -> Any: + """The single JSON document a ``--json`` command prints on stdout.""" + return json.loads(self.stdout) + + def __str__(self) -> str: + return f"exit {self.exit_code}\n--- stdout\n{self.stdout}\n--- stderr\n{self.stderr}" + + +class Hud: + """``hud`` bound to a working directory. Use the ``hud`` fixture. + + The subprocess inherits this process's environment, which the ``hud_env`` + fixture has already isolated: a temporary home, no developer credentials, + and every service URL pointed at the fake services or a closed port. + """ + + def __init__(self, cwd: Path) -> None: + self.cwd = cwd + cwd.mkdir(parents=True, exist_ok=True) + + def __call__( + self, + *args: str, + cwd: Path | None = None, + input: str | None = None, + env: dict[str, str] | None = None, + timeout: float = 120, + ) -> Result: + completed = subprocess.run( + [sys.executable, "-m", "hud.cli", *args], + cwd=cwd or self.cwd, + env={**os.environ, "COLUMNS": "120", "NO_COLOR": "1", "TERM": "dumb", **(env or {})}, + input=input, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + return Result(completed.returncode, completed.stdout, completed.stderr) + + +def scrub(text: str, *paths: Path) -> str: + """Replace temporary paths and UUIDs so output compares across runs.""" + for index, path in enumerate(paths): + text = text.replace(str(path), f"") + return UUID.sub("", text) diff --git a/tests/harness/config.py b/tests/harness/config.py new file mode 100644 index 000000000..07c6dafe4 --- /dev/null +++ b/tests/harness/config.py @@ -0,0 +1,88 @@ +"""The configuration the SDK under test sees. + +The SDK reads its configuration from environment variables once, when +``hud.settings`` is imported. :class:`HudEnv` sets those variables for this +process and for every subprocess it starts, then rebuilds the settings object +from them, so an in-process rollout and a ``hud`` subprocess see the same +values. This is the only place tests touch ``hud.settings``. +""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +from hud.settings import Settings, settings +from hud.utils.gateway import list_gateway_models + +from .services import SERVICE_ENV + +if TYPE_CHECKING: + from pathlib import Path + + import pytest + +# Every service the SDK calls points at a closed local port until a test opts into +# the fake services, so nothing a test does can reach a real HUD deployment. The +# web app is only ever printed as a link, so it keeps its default. +CLOSED_PORT = "http://127.0.0.1:9" + +ISOLATED = { + "HUD_TELEMETRY_ENABLED": "0", + "HUD_CLI_ANALYTICS_ENABLED": "0", + "HUD_SKIP_VERSION_CHECK": "1", + **{var: f"{CLOSED_PORT}/{service}" for service, var in SERVICE_ENV.items() if service != "web"}, +} + +# Provider SDKs read these directly, bypassing hud.settings. +PROVIDER_ENV = ( + "OPENAI_BASE_URL", + "ANTHROPIC_BASE_URL", + "GOOGLE_API_KEY", + "GOOGLE_GENAI_USE_VERTEXAI", +) + + +def setting_variables() -> list[str]: + """Environment variable names the SDK settings read.""" + return [ + field.validation_alias + for field in Settings.model_fields.values() + if isinstance(field.validation_alias, str) + ] + + +class HudEnv: + """Isolated SDK configuration for one test: a fresh home and no developer credentials.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch, home: Path) -> None: + self._monkeypatch = monkeypatch + self.home = home + home.mkdir(parents=True, exist_ok=True) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + for name in (*setting_variables(), *PROVIDER_ENV): + monkeypatch.delenv(name, raising=False) + for name, value in ISOLATED.items(): + monkeypatch.setenv(name, value) + self._apply() + + def set(self, **values: str | None) -> None: + """Set variables (``None`` unsets one) and re-read the SDK settings from them.""" + for name, value in values.items(): + if value is None: + self._monkeypatch.delenv(name, raising=False) + else: + self._monkeypatch.setenv(name, value) + self._apply() + + def get(self, name: str) -> str | None: + return os.environ.get(name) + + def _apply(self) -> None: + fresh = Settings(_env_file=None) + for name in Settings.model_fields: + self._monkeypatch.setattr(settings, name, getattr(fresh, name)) + # The gateway model catalog is cached per process; a new configuration + # may point at a different catalog. + list_gateway_models.cache_clear() diff --git a/tests/harness/docker.py b/tests/harness/docker.py new file mode 100644 index 000000000..30cb38858 --- /dev/null +++ b/tests/harness/docker.py @@ -0,0 +1,102 @@ +"""A fake ``docker`` executable for runtime and Harbor scenarios. + +:class:`FakeDocker` puts a ``docker`` script first on ``PATH``. Every invocation +appends its argv, and the contents of each file it names with ``-f``/``--file``, +to a log; then it answers from the first rule whose regex matches the argv +joined by spaces. Tests add rules ahead of the defaults with :meth:`FakeDocker.on`. +""" + +from __future__ import annotations + +import json +import os +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from pathlib import Path + + import pytest + +SCRIPT = """#!{python} +import json, os, pathlib, re, sys, time + +state = pathlib.Path(os.environ["FAKE_DOCKER_STATE"]) +argv = sys.argv[1:] +files = {{}} +for flag, value in zip(argv, argv[1:]): + if flag in ("-f", "--file") and pathlib.Path(value).is_file(): + files[value] = pathlib.Path(value).read_text() +with (state / "calls.jsonl").open("a") as log: + log.write(json.dumps({{"argv": argv, "files": files}}) + "\\n") +joined = " ".join(argv) +for rule in json.loads((state / "rules.json").read_text()): + if re.search(rule["match"], joined): + time.sleep(rule["delay"]) + sys.stdout.write(rule["stdout"]) + sys.stderr.write(rule["stderr"]) + sys.exit(rule["exit"]) +""" + +DF_OUTPUT = ( + "Filesystem 1024-blocks Used Available Capacity Mounted on\n" + "overlay 104857600 0 104857600 0% /\n" +) + + +@dataclass(frozen=True) +class Call: + argv: list[str] + files: dict[str, str] + + @property + def command(self) -> str: + return " ".join(self.argv) + + +class FakeDocker: + """The fake ``docker`` on ``PATH``. Use the ``fake_docker`` fixture.""" + + def __init__(self, root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + if sys.platform == "win32": + raise RuntimeError("the fake docker executable is POSIX-only") + self._state = root / "state" + bin_dir = root / "bin" + self._state.mkdir(parents=True) + bin_dir.mkdir(parents=True) + executable = bin_dir / "docker" + executable.write_text(SCRIPT.format(python=sys.executable)) + executable.chmod(0o755) + (self._state / "calls.jsonl").touch() + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}") + monkeypatch.setenv("FAKE_DOCKER_STATE", str(self._state)) + self._rules: list[dict[str, Any]] = [] + self.on(r".", stdout="") + self.on(r"^volume create ", stdout="volume\n") + self.on(r"^exec .* df -Pk", stdout=DF_OUTPUT) + self.on(r"^run ", stdout="cid-1\n") + + def on( + self, + match: str, + *, + stdout: str = "", + stderr: str = "", + exit: int = 0, + delay: float = 0.0, + ) -> None: + """Answer invocations whose argv matches the regex ``match``; newest rules win.""" + self._rules.insert( + 0, {"match": match, "stdout": stdout, "stderr": stderr, "exit": exit, "delay": delay} + ) + (self._state / "rules.json").write_text(json.dumps(self._rules)) + + @property + def calls(self) -> list[Call]: + lines = (self._state / "calls.jsonl").read_text().splitlines() + return [Call(**json.loads(line)) for line in lines if line] + + def commands(self, prefix: str = "") -> list[str]: + """Invocations so far as command lines, those starting with ``prefix`` when given.""" + return [call.command for call in self.calls if call.command.startswith(prefix)] diff --git a/tests/harness/models.py b/tests/harness/models.py new file mode 100644 index 000000000..66d8e0e01 --- /dev/null +++ b/tests/harness/models.py @@ -0,0 +1,451 @@ +"""Scripted model providers behind the fake inference gateway. + +One provider-neutral script answers the four wire protocols hud's agents speak: + +- OpenAI Chat Completions: ``POST /chat/completions`` +- OpenAI Responses: ``POST /responses`` +- Anthropic Messages, streamed as server-sent events: ``POST /v1/messages`` +- Gemini: ``POST /v1beta/models/{model}:generateContent`` + +A script is a list of turns, or a function from :class:`ModelRequest` to a turn. +The turn a request gets is read from the conversation the request carries (how +many assistant turns it already holds), so one script serves many concurrent +rollouts. Agents reach the script through ``HUD_GATEWAY_URL`` or a ``base_url``. +""" + +from __future__ import annotations + +import itertools +import json +import threading +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +from .services import Reply + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Sequence + + from .services import FakeServices, Request + + +@dataclass(frozen=True) +class ToolCall: + name: str + arguments: dict[str, Any] = field(default_factory=dict) + id: str | None = None + + +@dataclass(frozen=True) +class Turn: + """One assistant turn. ``fail`` answers with an HTTP error instead, such as a 529 or 400.""" + + text: str | None = None + tool_calls: tuple[ToolCall, ...] = () + reasoning: str | None = None + truncated: bool = False + fail: Reply | None = None + + +def say(text: str, *, reasoning: str | None = None) -> Turn: + return Turn(text=text, reasoning=reasoning) + + +def call(name: str, /, **arguments: Any) -> Turn: + return Turn(tool_calls=(ToolCall(name, arguments),)) + + +def calls(*tool_calls: ToolCall, text: str | None = None) -> Turn: + return Turn(text=text, tool_calls=tool_calls) + + +def fail(status: int, message: str = "scripted provider error") -> Turn: + return Turn(fail=Reply(status=status, json={"error": {"type": "error", "message": message}})) + + +@dataclass(frozen=True) +class ModelRequest: + """A model request as the provider received it, with the turn it asks for.""" + + protocol: str + model: str + body: dict[str, Any] + turn: int + headers: dict[str, str] + + @property + def tools(self) -> list[str]: + """Names of the tools the request offers.""" + names: list[str] = [] + for tool in self.body.get("tools") or []: + if self.protocol == "gemini": + names.extend(decl["name"] for decl in tool.get("functionDeclarations", [])) + elif self.protocol == "chat": + names.append(tool["function"]["name"]) + elif "name" in tool: + names.append(tool["name"]) + return names + + @property + def tool_results(self) -> list[str]: + """Texts of the tool results this request returns to the model, oldest first.""" + if self.protocol == "chat": + return [ + _text(message.get("content")) + for message in self.body.get("messages", []) + if message.get("role") == "tool" + ] + if self.protocol == "anthropic": + return [ + _text(block.get("content")) + for message in self.body.get("messages", []) + for block in message.get("content") or [] + if isinstance(block, dict) and block.get("type") == "tool_result" + ] + if self.protocol == "responses": + return [ + _text(item.get("output")) + for item in _input_items(self.body) + if item.get("type", "").endswith("_call_output") + ] + return [ + json.dumps(part["functionResponse"].get("response")) + for content in self.body.get("contents", []) + for part in content.get("parts", []) + if "functionResponse" in part + ] + + @property + def prompt(self) -> str: + """Text of the first user turn.""" + if self.protocol in {"chat", "anthropic"}: + for message in self.body.get("messages", []): + if message.get("role") == "user": + return _text(message.get("content")) + if self.protocol == "responses": + for item in _input_items(self.body): + if item.get("role") == "user": + return _text(item.get("content")) + if self.protocol == "gemini": + for content in self.body.get("contents", []): + if content.get("role") == "user": + return "".join(part.get("text", "") for part in content.get("parts", [])) + return "" + + +class Models: + """The scripted providers on a :class:`FakeServices` gateway. Use the ``models`` fixture.""" + + def __init__(self, services: FakeServices) -> None: + self._services = services + self._turns: list[Turn] = [] + self._policy: Callable[[ModelRequest], Turn] | None = None + self._responses: dict[str, int] = {} + self._lock = threading.Lock() + self._ids = itertools.count(1) + services.route("gateway", "POST", "/chat/completions", handler=self._chat) + services.route("gateway", "POST", "/responses", handler=self._responses_api) + services.route("gateway", "POST", "/v1/messages", handler=self._anthropic) + services.route( + "gateway", "POST", "/v1beta/models/{model}:generateContent", handler=self._gemini + ) + + @property + def url(self) -> str: + return self._services.url("gateway") + + def script(self, turns: Sequence[Turn]) -> None: + """Answer the n-th assistant turn of every conversation with ``turns[n]``.""" + self._turns, self._policy = list(turns), None + + def respond(self, policy: Callable[[ModelRequest], Turn]) -> None: + """Answer every request with ``policy(request)``.""" + self._turns, self._policy = [], policy + + def requests(self) -> list[ModelRequest]: + """Every model request so far, decoded, in arrival order.""" + return [self._decode(request) for request in self._services.requests("gateway", "POST")] + + # ─── protocol endpoints ─────────────────────────────────────────── + + def _next(self, request: ModelRequest) -> Turn: + if self._policy is not None: + return self._policy(request) + if request.turn >= len(self._turns): + raise AssertionError( + f"model script has {len(self._turns)} turns; request asked for turn " + f"{request.turn + 1} ({request.protocol}, tool results {request.tool_results})" + ) + return self._turns[request.turn] + + def _id(self, prefix: str) -> str: + return f"{prefix}_{next(self._ids)}" + + def _decode(self, raw: Request) -> ModelRequest: + body = raw.json or {} + if raw.path == "/chat/completions": + protocol, model = "chat", body.get("model", "") + turn = sum(1 for m in body.get("messages", []) if m.get("role") == "assistant") + elif raw.path == "/v1/messages": + protocol, model = "anthropic", body.get("model", "") + turn = sum(1 for m in body.get("messages", []) if m.get("role") == "assistant") + elif raw.path == "/responses": + protocol, model = "responses", body.get("model", "") + previous = body.get("previous_response_id") + with self._lock: + turn = ( + self._responses[previous] + 1 + if previous is not None + else sum( + 1 + for item in _input_items(body) + if item.get("role") == "assistant" or item.get("type") == "function_call" + ) + ) + else: + protocol, model = "gemini", raw.params["model"] + turn = sum(1 for c in body.get("contents", []) if c.get("role") == "model") + return ModelRequest(protocol, model, body, turn, raw.headers) + + def _chat(self, raw: Request) -> Reply: + request = self._decode(raw) + body = request.body + turn = self._next(request) + if turn.fail is not None: + return turn.fail + tool_calls = [ + { + "id": tool.id or self._id("call"), + "type": "function", + "function": {"name": tool.name, "arguments": json.dumps(tool.arguments)}, + } + for tool in turn.tool_calls + ] + message: dict[str, Any] = {"role": "assistant", "content": turn.text} + if tool_calls: + message["tool_calls"] = tool_calls + if turn.reasoning is not None: + message["reasoning_content"] = turn.reasoning + finish = "length" if turn.truncated else ("tool_calls" if tool_calls else "stop") + return Reply( + json={ + "id": self._id("chatcmpl"), + "object": "chat.completion", + "created": 0, + "model": body.get("model", ""), + "choices": [ + {"index": 0, "message": message, "finish_reason": finish, "logprobs": None} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + + def _responses_api(self, raw: Request) -> Reply: + request = self._decode(raw) + body = request.body + turn = self._next(request) + if turn.fail is not None: + return turn.fail + response_id = self._id("resp") + with self._lock: + self._responses[response_id] = request.turn + output: list[dict[str, Any]] = [] + if turn.reasoning is not None: + output.append( + { + "type": "reasoning", + "id": self._id("rs"), + "summary": [{"type": "summary_text", "text": turn.reasoning}], + } + ) + if turn.text is not None: + output.append( + { + "type": "message", + "id": self._id("msg"), + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": turn.text, "annotations": []}], + } + ) + output.extend( + { + "type": "function_call", + "id": self._id("fc"), + "call_id": tool.id or self._id("call"), + "name": tool.name, + "arguments": json.dumps(tool.arguments), + "status": "completed", + } + for tool in turn.tool_calls + ) + return Reply( + json={ + "id": response_id, + "object": "response", + "created_at": 0, + "status": "incomplete" if turn.truncated else "completed", + "incomplete_details": {"reason": "max_output_tokens"} if turn.truncated else None, + "model": body.get("model", ""), + "output": output, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + ) + + def _anthropic(self, raw: Request) -> Reply: + request = self._decode(raw) + body = request.body + turn = self._next(request) + if turn.fail is not None: + return turn.fail + blocks: list[dict[str, Any]] = [] + if turn.reasoning is not None: + blocks.append({"type": "thinking", "thinking": turn.reasoning, "signature": "sig"}) + if turn.text is not None: + blocks.append({"type": "text", "text": turn.text}) + blocks.extend( + { + "type": "tool_use", + "id": tool.id or self._id("toolu"), + "name": tool.name, + "input": tool.arguments, + } + for tool in turn.tool_calls + ) + stop = "max_tokens" if turn.truncated else ("tool_use" if turn.tool_calls else "end_turn") + message = { + "id": self._id("msg"), + "type": "message", + "role": "assistant", + "model": body.get("model", ""), + "content": blocks, + "stop_reason": stop, + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + if not body.get("stream"): + return Reply(json=message) + return Reply(stream=_anthropic_events(message), content_type="text/event-stream") + + def _gemini(self, raw: Request) -> Reply: + turn = self._next(self._decode(raw)) + if turn.fail is not None: + return turn.fail + parts: list[dict[str, Any]] = [] + if turn.reasoning is not None: + parts.append({"text": turn.reasoning, "thought": True}) + if turn.text is not None: + parts.append({"text": turn.text}) + parts.extend( + {"functionCall": {"name": tool.name, "args": tool.arguments}} + for tool in turn.tool_calls + ) + return Reply( + json={ + "candidates": [ + { + "content": {"role": "model", "parts": parts}, + "finishReason": "MAX_TOKENS" if turn.truncated else "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 1, + "candidatesTokenCount": 1, + "totalTokenCount": 2, + }, + "modelVersion": raw.params["model"], + } + ) + + +def _anthropic_events(message: dict[str, Any]) -> Iterator[bytes]: + def event(kind: str, data: dict[str, Any]) -> bytes: + return f"event: {kind}\ndata: {json.dumps({'type': kind, **data})}\n\n".encode() + + start = {**message, "content": [], "stop_reason": None} + yield event("message_start", {"message": start}) + for index, block in enumerate(message["content"]): + if block["type"] == "text": + yield event( + "content_block_start", + {"index": index, "content_block": {"type": "text", "text": ""}}, + ) + yield event( + "content_block_delta", + {"index": index, "delta": {"type": "text_delta", "text": block["text"]}}, + ) + elif block["type"] == "thinking": + yield event( + "content_block_start", + { + "index": index, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + }, + ) + yield event( + "content_block_delta", + { + "index": index, + "delta": {"type": "thinking_delta", "thinking": block["thinking"]}, + }, + ) + yield event( + "content_block_delta", + { + "index": index, + "delta": {"type": "signature_delta", "signature": block["signature"]}, + }, + ) + else: + yield event( + "content_block_start", + {"index": index, "content_block": {**block, "input": {}}}, + ) + yield event( + "content_block_delta", + { + "index": index, + "delta": { + "type": "input_json_delta", + "partial_json": json.dumps(block["input"]), + }, + }, + ) + yield event("content_block_stop", {"index": index}) + yield event( + "message_delta", + { + "delta": {"stop_reason": message["stop_reason"], "stop_sequence": None}, + "usage": {"output_tokens": 1}, + }, + ) + yield event("message_stop", {}) + + +def _input_items(body: dict[str, Any]) -> list[dict[str, Any]]: + items = body.get("input", []) + if isinstance(items, str): + return [{"role": "user", "content": items}] + return [item for item in items if isinstance(item, dict)] + + +def _text(content: Any) -> str: + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join( + part.get("text", "") if isinstance(part, dict) else str(part) for part in content + ) + return json.dumps(content) diff --git a/tests/harness/rfb.py b/tests/harness/rfb.py new file mode 100644 index 000000000..aa7c0c089 --- /dev/null +++ b/tests/harness/rfb.py @@ -0,0 +1,108 @@ +"""A minimal RFB (VNC) server: a fixed framebuffer that records input events. + +It speaks RFB 3.8 with no authentication and Raw encoding, enough for +``asyncvnc`` and therefore ``hud.capabilities.RFBClient`` and the computer-use +tools built on it. +""" + +from __future__ import annotations + +import asyncio +import struct +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +@dataclass(frozen=True) +class KeyEvent: + key: int + down: bool + + +@dataclass(frozen=True) +class PointerEvent: + x: int + y: int + buttons: int + + +@dataclass +class FakeScreen: + """The server side of a VNC session. ``events`` holds every key and pointer event received.""" + + width: int = 64 + height: int = 48 + color: tuple[int, int, int] = (40, 120, 200) + events: list[KeyEvent | PointerEvent] = field(default_factory=list) + connections: int = 0 + port: int = 0 + + @property + def url(self) -> str: + return f"rfb://127.0.0.1:{self.port}" + + def key_events(self) -> list[KeyEvent]: + return [event for event in self.events if isinstance(event, KeyEvent)] + + def pointer_events(self) -> list[PointerEvent]: + return [event for event in self.events if isinstance(event, PointerEvent)] + + async def _session(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self.connections += 1 + try: + writer.write(b"RFB 003.008\n") + await reader.readexactly(12) + writer.write(b"\x01\x01") # one security type: None + await reader.readexactly(1) + writer.write(struct.pack(">I", 0)) + await reader.readexactly(1) # ClientInit + name = b"hud-fake-screen" + pixel_format = struct.pack(">BBBBHHHBBB3x", 32, 24, 0, 1, 255, 255, 255, 0, 8, 16) + writer.write(struct.pack(">HH", self.width, self.height) + pixel_format) + writer.write(struct.pack(">I", len(name)) + name) + await writer.drain() + while True: + (kind,) = await reader.readexactly(1) + if kind == 0: # SetPixelFormat + await reader.readexactly(19) + elif kind == 2: # SetEncodings + _, count = struct.unpack(">BH", await reader.readexactly(3)) + await reader.readexactly(4 * count) + elif kind == 3: # FramebufferUpdateRequest + await reader.readexactly(9) + pixel = bytes([*self.color, 255]) + writer.write(struct.pack(">BxH", 0, 1)) + writer.write(struct.pack(">HHHHi", 0, 0, self.width, self.height, 0)) + writer.write(pixel * (self.width * self.height)) + await writer.drain() + elif kind == 4: # KeyEvent + down, _, key = struct.unpack(">B2sI", await reader.readexactly(7)) + self.events.append(KeyEvent(key, bool(down))) + elif kind == 5: # PointerEvent + buttons, x, y = struct.unpack(">BHH", await reader.readexactly(5)) + self.events.append(PointerEvent(x, y, buttons)) + elif kind == 6: # ClientCutText + _, length = struct.unpack(">3sI", await reader.readexactly(7)) + await reader.readexactly(length) + else: + raise ValueError(f"unexpected RFB client message {kind}") + except (asyncio.IncompleteReadError, ConnectionError): + pass + finally: + writer.close() + + +@asynccontextmanager +async def fake_screen( + width: int = 64, height: int = 48, color: tuple[int, int, int] = (40, 120, 200) +) -> AsyncIterator[FakeScreen]: + """Serve a :class:`FakeScreen` on loopback for the duration of the block.""" + screen = FakeScreen(width, height, color) + server = await asyncio.start_server(screen._session, "127.0.0.1", 0) + screen.port = server.sockets[0].getsockname()[1] + async with server: + yield screen diff --git a/tests/harness/scenario.py b/tests/harness/scenario.py new file mode 100644 index 000000000..ac9794387 --- /dev/null +++ b/tests/harness/scenario.py @@ -0,0 +1,83 @@ +"""Agents, providers and serving helpers for scenarios built on the public API.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +from hud.agents.base import Agent +from hud.clients import connect +from hud.eval import LocalRuntime, Task + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + + from hud.clients import HudClient + from hud.environment import Environment + from hud.eval import Provider, Run, Runtime + + +class ScriptedAgent(Agent): + """Answers every run with ``answer``, or ``answer(prompt)`` when it is callable. + + ``delay`` waits before answering; ``fail_before`` raises instead of answering + and ``fail_after`` raises after the answer is recorded. + """ + + def __init__( + self, + answer: str | Callable[[str], str] = "", + *, + delay: float = 0.0, + fail_before: BaseException | None = None, + fail_after: BaseException | None = None, + ) -> None: + super().__init__() + self._answer: Callable[[str], str] = ( + (lambda _prompt: answer) if isinstance(answer, str) else answer + ) + self._delay = delay + self._fail_before = fail_before + self._fail_after = fail_after + self.prompts: list[str] = [] + + async def __call__(self, run: Run) -> None: + self.prompts.append(run.prompt_text) + if self._delay: + await asyncio.sleep(self._delay) + if self._fail_before is not None: + raise self._fail_before + run.trace.content = self._answer(run.prompt_text) + if self._fail_after is not None: + raise self._fail_after + + +class RecordingProvider: + """Places each row with ``inner`` and records when placements start and stop.""" + + def __init__(self, inner: Provider) -> None: + self._inner = inner + self.events: list[tuple[str, str]] = [] + + @asynccontextmanager + async def __call__(self, task: Task) -> AsyncIterator[Runtime]: + self.events.append(("start", task.slug)) + try: + async with self._inner(task) as runtime: + yield runtime + finally: + self.events.append(("stop", task.slug)) + + +@asynccontextmanager +async def served(env: Environment, *, row: Task | None = None) -> AsyncIterator[HudClient]: + """Serve ``env`` on loopback and yield a control-channel client connected to it.""" + task = row or Task(env=env.name, id="scenario") + async with LocalRuntime(env)(task) as runtime, connect(runtime) as client: + yield client + + +def task_row(env: Environment, template: str, **args: Any) -> Task: + """The data row for ``template`` on ``env``, as a taskset file would carry it.""" + return Task(env=env.name, id=template, args=args) diff --git a/tests/harness/services.py b/tests/harness/services.py new file mode 100644 index 000000000..a7a38dbd4 --- /dev/null +++ b/tests/harness/services.py @@ -0,0 +1,298 @@ +"""A local stand-in for every HUD service the SDK talks to. + +One HTTP and WebSocket server answers for the API, telemetry, inference gateway, +runtime gateway, training service and web app, each under its own path prefix. +Every request is recorded. Tests declare the replies they need with +:meth:`FakeServices.route`; anything undeclared answers 404, so a test notices +traffic it did not expect. + +The server runs on its own thread and event loop, so the SDK reaches it the +same way from an in-process rollout and from a ``hud`` subprocess. +""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import re +import socket +import threading +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any +from urllib.parse import parse_qs + +import uvicorn +from starlette.applications import Starlette +from starlette.responses import Response, StreamingResponse +from starlette.routing import Route, WebSocketRoute + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable, Iterable + + from starlette.requests import Request as StarletteRequest + from starlette.websockets import WebSocket + +SERVICE_ENV = { + "api": "HUD_API_URL", + "telemetry": "HUD_TELEMETRY_URL", + "gateway": "HUD_GATEWAY_URL", + "runtime": "HUD_RUNTIME_URL", + "rl": "HUD_RL_URL", + "web": "HUD_WEB_URL", +} + +METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"] + + +@dataclass(frozen=True) +class Request: + """One recorded request, addressed relative to its service's prefix.""" + + service: str + method: str + path: str + query: dict[str, list[str]] + headers: dict[str, str] + body: bytes + params: dict[str, str] = field(default_factory=dict) + + @property + def json(self) -> Any: + return json.loads(self.body) if self.body else None + + @property + def bearer(self) -> str | None: + value = self.headers.get("authorization", "") + return value.removeprefix("Bearer ") if value.startswith("Bearer ") else None + + +@dataclass +class Reply: + """What a route answers. ``stream`` sends server-sent events or other chunks as they come.""" + + status: int = 200 + json: Any = None + body: bytes | str | None = None + headers: dict[str, str] = field(default_factory=dict) + content_type: str | None = None + stream: Iterable[bytes] | None = None + delay: float = 0.0 + + +@dataclass +class _Route: + service: str + method: str + pattern: re.Pattern[str] + answer: Any + + +def _compile(path: str) -> re.Pattern[str]: + """``/trace/{id}`` matches one segment per name; ``{rest:path}`` matches the remainder.""" + regex = re.sub(r"\{(\w+):path\}", r"(?P<\1>.*)", path) + regex = re.sub(r"\{(\w+)\}", r"(?P<\1>[^/]+)", regex) + return re.compile(f"^{regex}$") + + +class FakeServices: + """The fake HUD backend. Use the ``services`` fixture rather than constructing one.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._routes: list[_Route] = [] + self._sockets: list[_Route] = [] + self._requests: list[Request] = [] + self._socket = socket.socket() + self._socket.bind(("127.0.0.1", 0)) + self.port: int = self._socket.getsockname()[1] + app = Starlette( + routes=[ + Route("/{service}/{rest:path}", self._http, methods=METHODS), + WebSocketRoute("/{service}/{rest:path}", self._websocket), + ] + ) + self._server = uvicorn.Server( + uvicorn.Config(app, log_level="error", lifespan="off", ws="websockets-sansio") + ) + self._thread = threading.Thread(target=self._serve, name="fake-hud-services", daemon=True) + + def _serve(self) -> None: + asyncio.run(self._server.serve(sockets=[self._socket])) + + def start(self) -> None: + self._thread.start() + while not self._server.started: + if not self._thread.is_alive(): + raise RuntimeError("fake HUD services failed to start") + threading.Event().wait(0.01) + + def stop(self) -> None: + self._server.should_exit = True + self._thread.join(timeout=10) + + def reset(self) -> None: + with self._lock: + self._routes.clear() + self._sockets.clear() + self._requests.clear() + + # ─── configuration ──────────────────────────────────────────────── + + def url(self, service: str) -> str: + if service not in SERVICE_ENV: + raise KeyError(f"unknown service {service!r}; one of {sorted(SERVICE_ENV)}") + return f"http://127.0.0.1:{self.port}/{service}" + + def env(self) -> dict[str, str]: + """``HUD_*_URL`` variables pointing every service here.""" + return {var: self.url(service) for service, var in SERVICE_ENV.items()} + + def route( + self, + service: str, + method: str, + path: str, + *replies: Reply, + handler: Callable[[Request], Reply | Awaitable[Reply]] | None = None, + json: Any = None, + status: int = 200, + ) -> None: + """Answer ``method path`` on ``service``. + + ``replies`` are served in order and the last repeats; ``handler`` computes a + reply from each :class:`Request`; with neither, the route answers ``json`` + with ``status``. Later routes win over earlier ones, so a test can override + a fixture's default. + """ + if handler is not None and replies: + raise ValueError("a route takes replies or a handler, not both") + answer = handler or _Sequence(list(replies) or [Reply(status=status, json=json)]) + with self._lock: + self._routes.insert(0, _Route(service, method.upper(), _compile(path), answer)) + + def websocket(self, service: str, path: str, handler: Callable[..., Awaitable[None]]) -> None: + """Serve a WebSocket at ``path``; ``handler(websocket, params)`` owns the session.""" + with self._lock: + self._sockets.insert(0, _Route(service, "WS", _compile(path), handler)) + + # ─── observation ────────────────────────────────────────────────── + + def requests( + self, service: str | None = None, method: str | None = None, path: str | None = None + ) -> list[Request]: + """Recorded requests, filtered; ``path`` accepts the same ``{name}`` patterns as routes.""" + pattern = _compile(path) if path is not None else None + with self._lock: + recorded = list(self._requests) + return [ + request + for request in recorded + if (service is None or request.service == service) + and (method is None or request.method == method.upper()) + and (pattern is None or pattern.match(request.path)) + ] + + def bodies(self, service: str, method: str, path: str) -> list[Any]: + return [request.json for request in self.requests(service, method, path)] + + # ─── serving ────────────────────────────────────────────────────── + + def _match( + self, routes: list[_Route], service: str, method: str, path: str + ) -> tuple[Any, dict[str, str]]: + with self._lock: + candidates = list(routes) + for route in candidates: + if ( + route.service == service + and route.method == method + and (found := route.pattern.match(path)) + ): + return route.answer, found.groupdict() + return None, {} + + async def _http(self, incoming: StarletteRequest) -> Response: + service = incoming.path_params["service"] + path = "/" + incoming.path_params["rest"] + answer, params = self._match(self._routes, service, incoming.method, path) + request = Request( + service=service, + method=incoming.method, + path=path, + query=parse_qs(incoming.url.query), + headers={key.lower(): value for key, value in incoming.headers.items()}, + body=await incoming.body(), + params=params, + ) + with self._lock: + self._requests.append(request) + if answer is None: + return Response( + json.dumps({"detail": f"no fake route for {incoming.method} {service}{path}"}), + status_code=404, + media_type="application/json", + ) + reply = answer(request) + if inspect.isawaitable(reply): + reply = await reply + if reply.delay: + await asyncio.sleep(reply.delay) + if reply.stream is not None: + return StreamingResponse( + _chunks(reply.stream), + status_code=reply.status, + headers=reply.headers, + media_type=reply.content_type or "text/event-stream", + ) + if reply.body is not None: + return Response( + reply.body, + status_code=reply.status, + headers=reply.headers, + media_type=reply.content_type or "application/octet-stream", + ) + return Response( + json.dumps(reply.json), + status_code=reply.status, + headers=reply.headers, + media_type="application/json", + ) + + async def _websocket(self, websocket: WebSocket) -> None: + service = websocket.path_params["service"] + path = "/" + websocket.path_params["rest"] + handler, params = self._match(self._sockets, service, "WS", path) + request = Request( + service=service, + method="WS", + path=path, + query=parse_qs(websocket.url.query), + headers={key.lower(): value for key, value in websocket.headers.items()}, + body=b"", + params=params, + ) + with self._lock: + self._requests.append(request) + if handler is None: + await websocket.close(code=4404) + return + await handler(websocket, params) + + +class _Sequence: + def __init__(self, replies: list[Reply]) -> None: + if not replies: + raise ValueError("a reply sequence needs at least one reply") + self._replies = replies + self._lock = threading.Lock() + + def __call__(self, request: Request) -> Reply: + del request + with self._lock: + return self._replies.pop(0) if len(self._replies) > 1 else self._replies[0] + + +async def _chunks(stream: Iterable[bytes]) -> AsyncIterator[bytes]: + for chunk in stream: + yield chunk diff --git a/tests/harness/spans.py b/tests/harness/spans.py new file mode 100644 index 000000000..47633fd52 --- /dev/null +++ b/tests/harness/spans.py @@ -0,0 +1,38 @@ +"""Read what the SDK's telemetry exported, from the local span files.""" + +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import Any + +from hud.settings import settings + +STEP_SCHEMA = "hud.step.v1" + + +def spans(trace_id: str | None, span_dir: Path | None = None) -> list[dict[str, Any]]: + """Every span exported for the run ``trace_id``, in start order. + + Spans land in ``/.jsonl``; the span directory + is the SDK's own (``HUD_TELEMETRY_LOCAL_DIR``, else ``~/.hud/spans`` while + uploads are off). + """ + if trace_id is None: + raise ValueError("the run has no trace id, so it exported no spans") + directory = span_dir or Path(settings.span_dir or "") + path = directory / f"{uuid.UUID(trace_id).hex}.jsonl" + if not path.exists(): + return [] + records = [json.loads(line) for line in path.read_text("utf-8").splitlines() if line.strip()] + return sorted(records, key=lambda span: span.get("start_time", "")) + + +def steps(trace_id: str | None, span_dir: Path | None = None) -> list[dict[str, Any]]: + """The step payloads of a run, in order.""" + return [ + span["attributes"]["hud.payload"] + for span in spans(trace_id, span_dir) + if span.get("attributes", {}).get("hud.schema") == STEP_SCHEMA + ] diff --git a/tests/harness/test_fakes.py b/tests/harness/test_fakes.py new file mode 100644 index 000000000..21e5488a9 --- /dev/null +++ b/tests/harness/test_fakes.py @@ -0,0 +1,249 @@ +"""The fakes speak the real protocols: each is driven here by the real SDK client or agent.""" + +from __future__ import annotations + +import io +import json +import subprocess +from typing import TYPE_CHECKING + +import httpx +from anthropic import AsyncAnthropic +from anthropic.types.beta import BetaMessageParam, BetaToolUseBlock +from dirty_equals import IsStr +from google import genai +from google.genai.types import HttpOptions +from inline_snapshot import snapshot +from openai import AsyncOpenAI +from openai.types.responses import ResponseFunctionToolCall +from PIL import Image + +from hud import Environment +from hud.agents.openai_compatible import OpenAIChatAgent +from hud.agents.types import OpenAIChatConfig +from hud.capabilities import Capability, RFBClient +from hud.eval import LocalRuntime, Task, Taskset, rollout +from tests.harness import KeyEvent, Reply, ScriptedAgent, call, fake_screen, say, steps + +if TYPE_CHECKING: + from pathlib import Path + + from tests.harness import FakeDocker, FakeServices, Hud, HudEnv, Models + +HEX_ID = IsStr(regex=r"[0-9a-f]{32}") + + +def test_undeclared_routes_answer_404_and_are_recorded(services: FakeServices) -> None: + response = httpx.get( + f"{services.url('api')}/v2/nothing?x=1", headers={"Authorization": "Bearer k"} + ) + + assert response.status_code == 404 + (request,) = services.requests("api", "GET", "/v2/nothing") + assert request.query == {"x": ["1"]} + assert request.bearer == "k" + + +def test_routes_serve_replies_in_order_then_repeat_the_last(services: FakeServices) -> None: + services.route("api", "GET", "/v2/items/{id}", Reply(status=502), Reply(json={"ok": True})) + url = f"{services.url('api')}/v2/items/7" + + assert [httpx.get(url).status_code for _ in range(3)] == [502, 200, 200] + assert services.requests("api", path="/v2/items/{id}")[0].params == {"id": "7"} + + +def _sums() -> Environment: + env = Environment("sums") + + @env.template() + async def add(a: int, b: int): + answer = yield f"add {a} {b}" + yield 1.0 if answer == str(a + b) else 0.0 + + return env + + +def _solve(prompt: str) -> str: + return str(sum(int(x) for x in prompt.split()[1:])) + + +async def test_a_reporting_rollout_reaches_the_fake_platform( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + for path in ("/v2/trace/job/{id}/enter", "/v2/trace/{id}/enter", "/v2/trace/{id}/exit"): + services.route("api", "POST", path, json={}) + services.route("telemetry", "POST", "/trace/{id}/telemetry-upload", json={}) + + job = await Taskset("demo", [Task(env="sums", id="add", args={"a": 1, "b": 2})]).run( + ScriptedAgent(_solve), runtime=LocalRuntime(_sums()) + ) + + assert job.reward == 1.0 + assert services.bodies("api", "POST", "/v2/trace/{id}/exit") == snapshot( + [{"status": "completed", "reward": 1.0, "evaluation_result": {"score": 1.0}}] + ) + assert services.bodies("api", "POST", "/v2/trace/{id}/enter") == snapshot( + [{"job_id": HEX_ID, "group_id": HEX_ID, "task_slug": "add-1744f53e", "model": "unknown"}] + ) + (upload,) = services.requests("telemetry", "POST", "/trace/{id}/telemetry-upload") + assert upload.bearer == "k" + assert [span["name"] for span in upload.json["telemetry"]] == snapshot( + ["step.task", "step.user", "step.task"] + ) + + +async def test_a_scripted_chat_model_drives_an_agent_through_a_workspace( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + workspace = tmp_path / "workspace" + report = workspace / "REPORT.md" + env = Environment("report") + env.workspace(workspace, guest_path=str(workspace)) + + @env.initialize + async def seed() -> None: + workspace.mkdir(parents=True, exist_ok=True) + + @env.template() + async def write_report(): + yield "Write PASS to REPORT.md." + yield 1.0 if report.exists() and report.read_text().strip() == "PASS" else 0.0 + + models.script([call("write", filePath=str(report), content="PASS"), say("done")]) + agent = OpenAIChatAgent(OpenAIChatConfig(model="scripted", max_steps=4)) + + run = await rollout(Task(env="report", id="write_report"), agent, runtime=LocalRuntime(env)) + + assert run.reward == 1.0 + first, second = models.requests() + assert first.protocol == "chat" + assert "write" in first.tools + assert second.tool_results != [] + assert first.headers["authorization"] == "Bearer k" + assert [step["source"] for step in steps(run.trace_id)] == snapshot( + ["task", "user", "agent", "tool", "agent", "task"] + ) + + +async def test_a_scripted_anthropic_turn_streams_through_the_anthropic_sdk(models: Models) -> None: + models.script([call("bash", command="ls"), say("done", reasoning="thinking it over")]) + client = AsyncAnthropic(base_url=models.url, api_key="k") + messages: list[BetaMessageParam] = [{"role": "user", "content": "go"}] + + async with client.beta.messages.stream(model="m", max_tokens=64, messages=messages) as stream: + first = await stream.get_final_message() + (tool_use,) = first.content + assert isinstance(tool_use, BetaToolUseBlock) + messages += [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": tool_use.id, + "name": tool_use.name, + "input": tool_use.input, + } + ], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": tool_use.id, "content": "a.txt"}], + }, + ] + async with client.beta.messages.stream(model="m", max_tokens=64, messages=messages) as stream: + second = await stream.get_final_message() + + assert first.stop_reason == "tool_use" + assert (tool_use.name, tool_use.input) == ("bash", {"command": "ls"}) + assert [block.type for block in second.content] == ["thinking", "text"] + assert models.requests()[1].tool_results == ["a.txt"] + + +async def test_scripted_responses_chain_turns_by_previous_response_id(models: Models) -> None: + models.script([call("lookup", q="x"), say("found it")]) + client = AsyncOpenAI(base_url=models.url, api_key="k") + + first = await client.responses.create(model="m", input="find x") + (function_call,) = first.output + assert isinstance(function_call, ResponseFunctionToolCall) + second = await client.responses.create( + model="m", + previous_response_id=first.id, + input=[{"type": "function_call_output", "call_id": function_call.call_id, "output": "x=1"}], + ) + + assert function_call.name == "lookup" + assert second.output_text == "found it" + assert models.requests()[1].tool_results == ["x=1"] + + +async def test_a_scripted_gemini_turn_parses_in_the_genai_sdk(models: Models) -> None: + models.script([call("click_at", x=10, y=20)]) + client = genai.Client( + api_key="k", http_options=HttpOptions(base_url=models.url, api_version="v1beta") + ) + + response = await client.aio.models.generate_content(model="gemini-test", contents="click it") + + (function_call,) = response.function_calls or [] + assert (function_call.name, function_call.args) == ("click_at", {"x": 10, "y": 20}) + assert models.requests()[0].model == "gemini-test" + + +def test_the_hud_cli_runs_against_the_fake_platform( + services: FakeServices, hud_env: HudEnv, hud: Hud +) -> None: + hud_env.set(HUD_API_KEY="k") + services.route("api", "GET", "/v2/jobs", json={"items": [{"id": "job-1", "name": "demo"}]}) + + result = hud("jobs", "list", "--json") + + assert result.exit_code == 0, result + assert result.json == [{"id": "job-1", "name": "demo"}] + (request,) = services.requests("api", "GET", "/v2/jobs") + assert request.query == {"limit": ["20"]} + assert request.bearer == "k" + + +def test_fake_docker_logs_invocations_and_answers_from_rules( + fake_docker: FakeDocker, tmp_path: Path +) -> None: + compose = tmp_path / "compose.json" + compose.write_text(json.dumps({"services": {}})) + fake_docker.on(r"^port cid-1", stdout="127.0.0.1:43210\n") + + run = subprocess.run(["docker", "run", "image"], capture_output=True, text=True, check=True) + port = subprocess.run( + ["docker", "port", "cid-1", "8765"], capture_output=True, text=True, check=True + ) + subprocess.run(["docker", "compose", "-f", str(compose), "up"], check=True) + + assert (run.stdout, port.stdout) == ("cid-1\n", "127.0.0.1:43210\n") + assert fake_docker.commands() == [ + "run image", + "port cid-1 8765", + f"compose -f {compose} up", + ] + assert fake_docker.calls[-1].files == {str(compose): '{"services": {}}'} + + +async def test_the_fake_screen_serves_frames_and_records_input() -> None: + async with fake_screen(width=8, height=6, color=(10, 20, 30)) as screen: + client = await RFBClient.connect(Capability.rfb(url=screen.url)) + try: + png, mime_type = await client.screenshot_png() + client.conn.keyboard.press("a") + client.conn.mouse.move(3, 4) + await client.drain() + finally: + await client.close() + + assert mime_type == "image/png" + image = Image.open(io.BytesIO(png)).convert("RGB") + assert image.size == (8, 6) + assert image.getpixel((0, 0)) == (10, 20, 30) + assert KeyEvent(ord("a"), True) in screen.key_events() + assert screen.pointer_events()[-1].x == 3 diff --git a/tests/integrations/harbor/test_integration.py b/tests/integrations/harbor/test_integration.py index f97a0ce6b..22452976b 100644 --- a/tests/integrations/harbor/test_integration.py +++ b/tests/integrations/harbor/test_integration.py @@ -31,7 +31,8 @@ REPO = Path(__file__).resolve().parents[3] pytestmark = [ - pytest.mark.integration, + pytest.mark.e2e, + pytest.mark.docker, pytest.mark.skipif(sys.platform == "win32", reason="adapted images are Linux containers"), ] From 4fec9728e70c8679aed0f2b75dc47fcc093f3af4 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:00:18 -0700 Subject: [PATCH 003/248] chore: accept any uv from 0.12.2 on The exact pin failed every other uv version outright, including the ones container images ship. The root lockfile is not committed, so the exact pin bought nothing beyond choosing CI's uv; a floor keeps the minimum the workflows rely on without blocking newer releases. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2fcf5be2d..0ebed58e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,7 +174,7 @@ train = [ [tool.uv] -required-version = "==0.12.2" +required-version = ">=0.12.2" [tool.ruff] target-version = "py311" From 452865e4188e21e17b1a22a31c62c1b0b64ae4e6 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:14 -0700 Subject: [PATCH 004/248] test: drive the control channel frame by frame Replace the server, manifest and argument tests, which reached into _TaskFactory.manifest_entry and the framing helpers, with one served environment spoken to in raw JSON-RPC. Every reply frame is compared with a golden copy: the tasks.list manifest, each grading and argument shape, the error replies for malformed requests, and how cancel, bye, grade, restart and disconnect close a template. Sessions and tunnels move to the same raw wire, so no test patches StreamWriter or reads a private endpoint; the workspace-binding session rows move to the sandbox lane. --- tests/environment/conftest.py | 92 +- tests/environment/test_arguments.py | 85 -- tests/environment/test_control_protocol.py | 1304 ++++++++++++++++++++ tests/environment/test_manifest.py | 114 -- tests/environment/test_server.py | 248 ---- tests/environment/test_sessions.py | 339 ++--- tests/environment/test_tunnel.py | 219 ++-- 7 files changed, 1685 insertions(+), 716 deletions(-) delete mode 100644 tests/environment/test_arguments.py create mode 100644 tests/environment/test_control_protocol.py delete mode 100644 tests/environment/test_manifest.py delete mode 100644 tests/environment/test_server.py diff --git a/tests/environment/conftest.py b/tests/environment/conftest.py index bb52f5b22..a125a97b2 100644 --- a/tests/environment/conftest.py +++ b/tests/environment/conftest.py @@ -1,28 +1,92 @@ -"""Harnesses for protocol-level environment tests. +"""Raw control-channel access for the environment scenarios. -Inline-defined envs have no source file to ``spawn``, so :func:`served` drives -the connect path against a loopback substrate served by this process. This is -a test harness, not an engine placement. +The control channel is newline-delimited JSON-RPC over TCP. :class:`Wire` speaks +it byte for byte, so a scenario sees exactly the frames a peer would. """ from __future__ import annotations +import asyncio +import contextlib +import json from contextlib import asynccontextmanager -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit -from hud.clients import connect -from hud.eval import LocalRuntime, Task +from dirty_equals import IsStr + +# The sandbox area's tests import ``served`` from here; it lives in the harness. +from tests.harness import served if TYPE_CHECKING: from collections.abc import AsyncIterator - from hud.clients import HudClient - from hud.environment import Environment +__all__ = ["FRAME_LIMIT", "SESSION_ID", "Wire", "encode", "served", "wire"] + +#: A control session id as the server mints it. +SESSION_ID = IsStr(regex=r"sess-[0-9a-f]{8}") + +#: The control channel's frame limit, in bytes, excluding the newline. +FRAME_LIMIT = 16 * 1024 * 1024 + + +def encode(frame: dict[str, Any]) -> bytes: + """One frame as the control channel carries it.""" + return json.dumps(frame, separators=(",", ":")).encode() + b"\n" + + +class Wire: + """One TCP connection to a served control channel.""" + + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self.reader = reader + self.writer = writer + self._ids = 0 + + async def send(self, method: str, params: dict[str, Any] | None = None) -> None: + self._ids += 1 + frame: dict[str, Any] = {"jsonrpc": "2.0", "id": self._ids, "method": method} + if params is not None: + frame["params"] = params + await self.write(encode(frame)) + + async def write(self, data: bytes) -> None: + self.writer.write(data) + await self.writer.drain() + + async def read(self) -> dict[str, Any] | None: + """The next reply frame, or ``None`` when the server hung up.""" + line = await asyncio.wait_for(self.reader.readline(), timeout=30) + return json.loads(line) if line else None + + async def call(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + await self.send(method, params) + reply = await self.read() + assert reply is not None, f"the server hung up on {method}" + return reply + + async def hang_up(self) -> None: + """Close the sending side and wait until the server has ended the connection.""" + self.writer.write_eof() + while await asyncio.wait_for(self.reader.read(65536), timeout=30): + pass + await self.close() + + async def close(self) -> None: + self.writer.close() + with contextlib.suppress(ConnectionError): + await self.writer.wait_closed() @asynccontextmanager -async def served(env: Environment) -> AsyncIterator[HudClient]: - """Serve *env* on a loopback substrate and yield a connected client.""" - task = Task(env=env.name, id="test") - async with LocalRuntime(env)(task) as runtime, connect(runtime) as client: - yield client +async def wire(url: str) -> AsyncIterator[Wire]: + """Open a raw connection to the control channel at ``tcp://host:port``.""" + address = urlsplit(url) + reader, writer = await asyncio.open_connection( + address.hostname, address.port, limit=2 * FRAME_LIMIT + ) + connection = Wire(reader, writer) + try: + yield connection + finally: + await connection.close() diff --git a/tests/environment/test_arguments.py b/tests/environment/test_arguments.py deleted file mode 100644 index c4424ae11..000000000 --- a/tests/environment/test_arguments.py +++ /dev/null @@ -1,85 +0,0 @@ -from __future__ import annotations - -from typing import Literal - -from pydantic import BaseModel - -from hud.environment import ( - DataFileArg, - DataFileRef, - DataFilesArg, - Environment, - GradingArg, - PromptArg, -) -from hud.eval import Run - -from .conftest import served - - -class _Attachment(DataFileRef): - expand: bool = False - - -class _Fixture(DataFileRef): - channel: Literal["mail", "calendar"] - - -class _Criterion(BaseModel): - requirement: str - weight: float = 1.0 - guidance: str | None = None - - -def test_argument_types_publish_editor_hints_and_model_schemas() -> None: - env = Environment("typed-args") - - @env.template() - async def task( - prompt: PromptArg, - attachment: DataFileArg[_Attachment], - fixtures: DataFilesArg[_Fixture], - criteria: GradingArg[_Criterion], - ): - yield prompt - yield 1.0 - - schema = task.manifest_entry()["args"] - properties = schema["properties"] - - assert properties["prompt"]["x-hud-hint"] == "prompt" - assert properties["attachment"]["x-hud-hint"] == "data-file" - assert properties["fixtures"]["x-hud-hint"] == "data-files" - assert properties["criteria"]["x-hud-hint"] == "grading" - assert properties["fixtures"]["items"]["$ref"] == "#/$defs/_Fixture" - assert properties["criteria"]["items"]["$ref"] == "#/$defs/_Criterion" - assert "channel" in schema["$defs"]["_Fixture"]["properties"] - assert "guidance" in schema["$defs"]["_Criterion"]["properties"] - - -async def test_typed_argument_values_reach_task_as_models() -> None: - env = Environment("typed-args") - - @env.template() - async def task( - attachments: DataFilesArg[_Attachment], - criteria: GradingArg[_Criterion], - ): - assert isinstance(attachments[0], _Attachment) - assert isinstance(criteria[0], _Criterion) - yield f"{attachments[0].path}:{criteria[0].requirement}" - yield 1.0 - - async with ( - served(env) as client, - Run( - client, - "task", - { - "attachments": [{"file_id": "file-1", "path": "brief.pdf"}], - "criteria": [{"requirement": "Answer the question", "weight": 2.0}], - }, - ) as run, - ): - assert run.prompt_text == "brief.pdf:Answer the question" - run.trace.content = "done" diff --git a/tests/environment/test_control_protocol.py b/tests/environment/test_control_protocol.py new file mode 100644 index 000000000..7c987992b --- /dev/null +++ b/tests/environment/test_control_protocol.py @@ -0,0 +1,1304 @@ +"""The control channel, frame by frame: what a peer sends and what the server answers. + +One environment carries a template per grading and argument shape. Each scenario +serves it on loopback, speaks raw JSON-RPC to it, and compares every reply frame +with a golden copy (session ids normalized). A ``guarded`` template appends to a +log when it starts and when its generator closes, so a scenario sees whether a +template ran and whether it was torn down. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Literal, cast + +import pytest +from dirty_equals import IsStr +from inline_snapshot import snapshot +from pydantic import BaseModel + +from hud.capabilities import Capability +from hud.environment import ( + DataFileArg, + DataFileRef, + DataFilesArg, + Environment, + GradingArg, + PromptArg, +) +from hud.eval import LocalRuntime, Task, rollout +from hud.graders import EvaluationResult, SubScore +from tests.harness import ScriptedAgent, steps + +from .conftest import FRAME_LIMIT, SESSION_ID, encode, wire + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + +Mode = Literal["upper", "lower"] + + +class Payload(BaseModel): + text: str + + +class Point(BaseModel): + x: int + y: int + + +class Attachment(DataFileRef): + expand: bool = False + + +class Criterion(BaseModel): + requirement: str + weight: float = 1.0 + + +def protocol_env(log: Path) -> Environment: + env = Environment("protocol", version="1.2.3") + + @env.initialize + async def publish_browser() -> None: + env.add_capability(Capability.cdp(url="ws://127.0.0.1:9/devtools/browser")) + + @env.template(description="Grades 1.0 when the answer is 42.") + async def plain(): + answer = yield "What is six times seven?" + yield 1.0 if answer == "42" else 0.0 + + @env.template() + async def rich(): + yield "go" + yield {"score": 0.5, "info": {"detail": "partial credit"}} + + @env.template() + async def model(): + yield "go" + yield EvaluationResult( + reward=0.75, + content="nice", + info={"max_tile": 256}, + subscores=[ + SubScore( + name="judge", + value=0.75, + info={"model": "judge-model"}, + children=[SubScore(name="criterion", value=1.0, info={"reason": "because"})], + ) + ], + ) + + @env.template() + async def bad_dict(): + yield "go" + yield {"reward": 1.0} + + @env.template() + async def stringy(): + yield "go" + yield "great job" + + @env.template() + async def no_grade(): + yield "go" + + @env.template() + async def typed(mode: Mode, payload: Payload, retries: int | None = None): + text = payload.text.upper() if mode == "upper" else payload.text.lower() + yield text + "!" * (retries or 0) + yield 1.0 + + @env.template(input=Point, returns=Point) + async def structured(): + answer = yield "Name a point." + content = answer.content + yield { + "score": 1.0 if isinstance(content, Point) else 0.0, + "content": content.model_dump() if isinstance(content, Point) else content, + "raw": answer.raw, + } + + @env.template() + async def files( + prompt: PromptArg, + attachment: DataFileArg[Attachment], + fixtures: DataFilesArg[Attachment], + criteria: GradingArg[Criterion], + ): + yield ( + f"{prompt} {type(attachment).__name__}:{attachment.path} " + f"{[type(f).__name__ for f in fixtures]} " + f"{type(criteria[0]).__name__}:{criteria[0].requirement}x{criteria[0].weight}" + ) + yield 1.0 + + @env.template() + async def defaults(difficulty: int, suite: str = "coding"): + yield f"{suite} at {difficulty}" + yield 1.0 + + @env.template() + async def flexible(n: int, **rest: str): + yield f"{n} {sorted(rest.items())}" + yield 1.0 + + @env.template() + async def loose(anything): + yield repr(anything) + yield 1.0 + + @env.template() + async def guarded(name: str): + with log.open("a") as out: + out.write(f"start {name}\n") + try: + yield f"holding {name}" + yield 1.0 + finally: + with log.open("a") as out: + out.write(f"closed {name}\n") + + @env.template() + async def lookup(): + settings: dict[str, str] = {} + yield settings["missing"] + + return env + + +def lines(log: Path) -> list[str]: + return log.read_text().splitlines() if log.exists() else [] + + +async def converse(env: Environment, frames: list[tuple[str, dict[str, Any] | None]]) -> list[Any]: + """Send ``frames`` on one connection; the replies, ``None`` once the server hangs up.""" + replies: list[Any] = [] + async with ( + LocalRuntime(env)(Task(env=env.name, id="protocol")) as runtime, + wire(runtime.url) as connection, + ): + for method, params in frames: + await connection.send(method, params) + replies.append(await connection.read()) + return replies + + +async def test_tasks_list_publishes_each_templates_argument_and_io_contract(tmp_path: Path) -> None: + (reply,) = await converse(protocol_env(tmp_path / "log"), [("tasks.list", None)]) + + assert {task["id"]: task for task in reply["result"]["tasks"]} == snapshot( + { + "plain": { + "id": "plain", + "description": "Grades 1.0 when the answer is 42.", + "args": {"properties": {}, "type": "object", "additionalProperties": False}, + }, + "rich": { + "id": "rich", + "description": "", + "args": {"properties": {}, "type": "object", "additionalProperties": False}, + }, + "model": { + "id": "model", + "description": "", + "args": {"properties": {}, "type": "object", "additionalProperties": False}, + }, + "bad_dict": { + "id": "bad_dict", + "description": "", + "args": {"properties": {}, "type": "object", "additionalProperties": False}, + }, + "stringy": { + "id": "stringy", + "description": "", + "args": {"properties": {}, "type": "object", "additionalProperties": False}, + }, + "no_grade": { + "id": "no_grade", + "description": "", + "args": {"properties": {}, "type": "object", "additionalProperties": False}, + }, + "typed": { + "id": "typed", + "description": "", + "args": { + "$defs": { + "Payload": { + "properties": {"text": {"title": "Text", "type": "string"}}, + "required": ["text"], + "title": "Payload", + "type": "object", + } + }, + "properties": { + "mode": {"enum": ["upper", "lower"], "title": "Mode", "type": "string"}, + "payload": {"$ref": "#/$defs/Payload"}, + "retries": { + "anyOf": [{"type": "integer"}, {"type": "null"}], + "default": None, + "title": "Retries", + }, + }, + "required": ["mode", "payload"], + "type": "object", + "additionalProperties": False, + }, + }, + "structured": { + "id": "structured", + "description": "", + "args": {"properties": {}, "type": "object", "additionalProperties": False}, + "input": { + "properties": { + "x": {"title": "X", "type": "integer"}, + "y": {"title": "Y", "type": "integer"}, + }, + "required": ["x", "y"], + "title": "Point", + "type": "object", + }, + "returns": { + "properties": { + "x": {"title": "X", "type": "integer"}, + "y": {"title": "Y", "type": "integer"}, + }, + "required": ["x", "y"], + "title": "Point", + "type": "object", + }, + }, + "files": { + "id": "files", + "description": "", + "args": { + "$defs": { + "Attachment": { + "additionalProperties": False, + "properties": { + "file_id": { + "description": "HUD data-file id", + "title": "File Id", + "type": "string", + }, + "path": { + "anyOf": [{"type": "string"}, {"type": "null"}], + "default": None, + "description": "Environment-owned destination path", + "title": "Path", + }, + "expand": {"default": False, "title": "Expand", "type": "boolean"}, + }, + "required": ["file_id"], + "title": "Attachment", + "type": "object", + }, + "Criterion": { + "properties": { + "requirement": {"title": "Requirement", "type": "string"}, + "weight": {"default": 1.0, "title": "Weight", "type": "number"}, + }, + "required": ["requirement"], + "title": "Criterion", + "type": "object", + }, + }, + "properties": { + "prompt": {"title": "Prompt", "type": "string", "x-hud-hint": "prompt"}, + "attachment": {"$ref": "#/$defs/Attachment", "x-hud-hint": "data-file"}, + "fixtures": { + "items": {"$ref": "#/$defs/Attachment"}, + "title": "Fixtures", + "type": "array", + "x-hud-hint": "data-files", + }, + "criteria": { + "items": {"$ref": "#/$defs/Criterion"}, + "title": "Criteria", + "type": "array", + "x-hud-hint": "grading", + }, + }, + "required": ["prompt", "attachment", "fixtures", "criteria"], + "type": "object", + "additionalProperties": False, + }, + }, + "defaults": { + "id": "defaults", + "description": "", + "args": { + "properties": { + "difficulty": {"title": "Difficulty", "type": "integer"}, + "suite": {"default": "coding", "title": "Suite", "type": "string"}, + }, + "required": ["difficulty"], + "type": "object", + "additionalProperties": False, + }, + }, + "flexible": { + "id": "flexible", + "description": "", + "args": { + "properties": {"n": {"title": "N", "type": "integer"}}, + "required": ["n"], + "type": "object", + "additionalProperties": True, + }, + }, + "loose": { + "id": "loose", + "description": "", + "args": { + "properties": {"anything": {"title": "Anything"}}, + "required": ["anything"], + "type": "object", + "additionalProperties": False, + }, + }, + "guarded": { + "id": "guarded", + "description": "", + "args": { + "properties": {"name": {"title": "Name", "type": "string"}}, + "required": ["name"], + "type": "object", + "additionalProperties": False, + }, + }, + "lookup": { + "id": "lookup", + "description": "", + "args": {"properties": {}, "type": "object", "additionalProperties": False}, + }, + } + ) + + +async def test_hello_names_the_session_and_publishes_hook_capabilities(tmp_path: Path) -> None: + replies = await converse(protocol_env(tmp_path / "log"), [("hello", {}), ("bye", None)]) + + assert replies == snapshot( + [ + { + "jsonrpc": "2.0", + "id": 1, + "result": { + "session_id": SESSION_ID, + "env": {"name": "protocol", "version": "1.2.3"}, + "bindings": [ + { + "name": "browser", + "protocol": "cdp/1.3", + "url": "ws://127.0.0.1:9/devtools/browser", + "params": {}, + } + ], + }, + }, + {"jsonrpc": "2.0", "id": 2, "result": {"goodbye": True}}, + ] + ) + + +@pytest.mark.parametrize( + ("template", "args", "answer", "expected"), + [ + pytest.param( + "plain", + {}, + "42", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "What is six times seven?"}}, + {"jsonrpc": "2.0", "id": 2, "result": {"score": 1.0}}, + ] + ), + id="number-right", + ), + pytest.param( + "plain", + {}, + "41", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "What is six times seven?"}}, + {"jsonrpc": "2.0", "id": 2, "result": {"score": 0.0}}, + ] + ), + id="number-wrong", + ), + pytest.param( + "rich", + {}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "go"}}, + { + "jsonrpc": "2.0", + "id": 2, + "result": {"score": 0.5, "info": {"detail": "partial credit"}}, + }, + ] + ), + id="score-dict-keeps-extras", + ), + pytest.param( + "model", + {}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "go"}}, + { + "jsonrpc": "2.0", + "id": 2, + "result": { + "done": True, + "content": "nice", + "info": {"max_tile": 256}, + "isError": False, + "subscores": [ + { + "name": "judge", + "weight": 1.0, + "value": 0.75, + "children": [ + { + "name": "criterion", + "weight": 1.0, + "value": 1.0, + "children": None, + "info": {"reason": "because"}, + } + ], + "info": {"model": "judge-model"}, + } + ], + "score": 0.75, + }, + }, + ] + ), + id="evaluation-result-whole-frame", + ), + pytest.param( + "bad_dict", + {}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "go"}}, + { + "jsonrpc": "2.0", + "id": 2, + "error": { + "code": -32000, + "message": ( + "task 'bad_dict' graded with a dict missing a numeric 'score' " + "(keys: ['reward'])" + ), + }, + }, + ] + ), + id="dict-without-score-fails-loudly", + ), + pytest.param( + "stringy", + {}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "go"}}, + { + "jsonrpc": "2.0", + "id": 2, + "error": { + "code": -32000, + "message": ( + "task graded with str: yield a number, an object with a numeric " + ".reward, or a dict containing a numeric 'score'" + ), + }, + }, + ] + ), + id="string-grade-fails-loudly", + ), + pytest.param( + "no_grade", + {}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "go"}}, + {"jsonrpc": "2.0", "id": 2, "result": {"score": 0.0}}, + ] + ), + id="no-grade-yield-grades-zero", + ), + pytest.param( + "typed", + {"mode": '"upper"', "payload": '{"text":"hello"}', "retries": "3"}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "HELLO!!!"}}, + {"jsonrpc": "2.0", "id": 2, "result": {"score": 1.0}}, + ] + ), + id="json-string-args-coerce-to-annotations", + ), + pytest.param( + "typed", + {"mode": "lower", "payload": {"text": "HeLLo"}}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "hello"}}, + {"jsonrpc": "2.0", "id": 2, "result": {"score": 1.0}}, + ] + ), + id="native-args-validate-against-annotations", + ), + pytest.param( + "structured", + {}, + '{"x": 1, "y": 2}', + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "Name a point."}}, + { + "jsonrpc": "2.0", + "id": 2, + "result": { + "score": 1.0, + "content": {"x": 1, "y": 2}, + "raw": '{"x": 1, "y": 2}', + }, + }, + ] + ), + id="returns-parses", + ), + pytest.param( + "structured", + {}, + "about (1, 2)", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "Name a point."}}, + { + "jsonrpc": "2.0", + "id": 2, + "result": {"score": 0.0, "content": "about (1, 2)", "raw": "about (1, 2)"}, + }, + ] + ), + id="returns-keeps-raw", + ), + pytest.param( + "files", + { + "prompt": "Review", + "attachment": {"file_id": "file-1", "path": "brief.pdf"}, + "fixtures": [{"file_id": "file-2"}], + "criteria": [{"requirement": "Answer it", "weight": 2.0}], + }, + "x", + snapshot( + [ + { + "jsonrpc": "2.0", + "id": 1, + "result": { + "prompt": ( + "Review Attachment:brief.pdf ['Attachment'] Criterion:Answer itx2.0" + ) + }, + }, + {"jsonrpc": "2.0", "id": 2, "result": {"score": 1.0}}, + ] + ), + id="data-file-and-grading-args-arrive-as-models", + ), + pytest.param( + "defaults", + {"difficulty": 3}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "coding at 3"}}, + {"jsonrpc": "2.0", "id": 2, "result": {"score": 1.0}}, + ] + ), + id="defaults-fill-in", + ), + pytest.param( + "flexible", + {"n": 1, "b": "2", "a": "1"}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "1 [('a', '1'), ('b', '2')]"}}, + {"jsonrpc": "2.0", "id": 2, "result": {"score": 1.0}}, + ] + ), + id="var-kwargs", + ), + pytest.param( + "loose", + {"anything": [1, {"a": None}]}, + "x", + snapshot( + [ + {"jsonrpc": "2.0", "id": 1, "result": {"prompt": "[1, {'a': None}]"}}, + {"jsonrpc": "2.0", "id": 2, "result": {"score": 1.0}}, + ] + ), + id="unannotated", + ), + ], +) +async def test_a_template_starts_and_grades_over_the_wire( + tmp_path: Path, template: str, args: dict[str, Any], answer: str, expected: Any +) -> None: + replies = await converse( + protocol_env(tmp_path / "log"), + [("tasks.start", {"id": template, "args": args}), ("tasks.grade", {"answer": answer})], + ) + + assert replies == expected + + +@pytest.mark.parametrize( + ("frames", "expected"), + [ + pytest.param( + [("tasks.start", {"id": "missing"})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": {"code": -32602, "message": "unknown task: 'missing'"}, + } + ], + [], + ) + ), + id="start-unknown-template", + ), + pytest.param( + [("tasks.start", {"id": 7})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": "tasks.start: 'id' must be a string", + }, + } + ], + [], + ) + ), + id="start-non-string-id", + ), + pytest.param( + [("tasks.start", {"id": "guarded", "args": ["x"]})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": "tasks.start: 'args' must be an object", + }, + } + ], + [], + ) + ), + id="start-list-args", + ), + pytest.param( + [("tasks.start", {"id": "guarded", "args": {"nmae": "x"}})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32000, + "message": ( + "task 'guarded': bad args ['nmae']: missing a required " + "argument: 'name'" + ), + }, + } + ], + [], + ) + ), + id="start-signature-mismatch", + ), + pytest.param( + [("tasks.grade", {"answer": "x"})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": {"code": -32600, "message": "no task in progress"}, + } + ], + [], + ) + ), + id="grade-without-start", + ), + pytest.param( + [("tasks.stop", {})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": {"code": -32601, "message": "method not found: tasks.stop"}, + } + ], + [], + ) + ), + id="unknown-method", + ), + pytest.param( + [("tasks.cancel", None), ("tunnel.open", {"capability": "browser"})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "result": {"cancelled": True}, + }, + { + "jsonrpc": "2.0", + "id": 2, + "error": {"code": -32601, "message": "method not found: tunnel.open"}, + }, + ], + [], + ) + ), + id="tunnel-open-mid-session", + ), + pytest.param( + [("hello", {"session_id": 5})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": "hello: 'session_id' must be a string", + }, + } + ], + [], + ) + ), + id="hello-non-string-session", + ), + pytest.param( + [("hello", {"session_id": "sess-00000000"})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32600, + "message": "unknown session: 'sess-00000000'", + }, + } + ], + [], + ) + ), + id="hello-unknown-session", + ), + pytest.param( + [("hello", {"workspace_routes": "ssh"})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": "hello: 'workspace_routes' must be a list", + }, + } + ], + [], + ) + ), + id="hello-routes-not-a-list", + ), + pytest.param( + [("hello", {"workspace_routes": [{"capability": "ssh", "host": "h", "port": "443"}]})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": "hello: workspace route port must be an integer", + }, + } + ], + [], + ) + ), + id="hello-route-port-not-an-int", + ), + pytest.param( + [("hello", {"connections": {"name": "x"}})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": "hello: 'connections' must be a list", + }, + } + ], + [], + ) + ), + id="hello-connections-object", + ), + pytest.param( + [("hello", {"connections": [{"name": "inference"}]})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": ( + "hello: connection name, capability, and url must be strings" + ), + }, + } + ], + [], + ) + ), + id="hello-connection-missing-fields", + ), + pytest.param( + [ + ( + "hello", + { + "connections": [ + { + "name": "inference", + "capability": "ssh", + "url": "https://inference.example", + "headers": {"Authorization": "Bearer t"}, + } + ] + }, + ) + ], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32000, + "message": "workspace capability 'ssh' does not exist", + }, + } + ], + [], + ) + ), + id="hello-connection-without-a-workspace", + ), + pytest.param( + [("hello", {"workspace_routes": [{"capability": "ssh", "host": "h", "port": 443}]})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32000, + "message": "workspace capability 'ssh' does not exist", + }, + } + ], + [], + ) + ), + id="hello-route-without-a-workspace", + ), + ], +) +async def test_a_bad_request_gets_an_error_reply_and_the_session_keeps_serving( + tmp_path: Path, frames: list[tuple[str, dict[str, Any] | None]], expected: Any +) -> None: + log = tmp_path / "log" + + replies = await converse(protocol_env(log), [*frames, ("tasks.list", None)]) + + *errors, listed = replies + assert (errors, lines(log)) == expected + assert "tasks" in listed["result"] + + +@pytest.mark.parametrize( + ("ending", "expected"), + [ + pytest.param( + [("tasks.cancel", None)], + snapshot( + ( + [{"jsonrpc": "2.0", "id": 2, "result": {"cancelled": True}}], + ["start first", "closed first"], + ["start first", "closed first"], + ) + ), + id="cancel", + ), + pytest.param( + [("bye", None)], + snapshot( + ( + [{"jsonrpc": "2.0", "id": 2, "result": {"goodbye": True}}], + ["start first", "closed first"], + ["start first", "closed first"], + ) + ), + id="bye", + ), + pytest.param( + [("tasks.grade", {"answer": "x"})], + snapshot( + ( + [{"jsonrpc": "2.0", "id": 2, "result": {"score": 1.0}}], + ["start first", "closed first"], + ["start first", "closed first"], + ) + ), + id="grade", + ), + pytest.param( + [("tasks.start", {"id": "guarded", "args": {"name": "second"}})], + snapshot( + ( + [{"jsonrpc": "2.0", "id": 2, "result": {"prompt": "holding second"}}], + ["start first", "closed first", "start second"], + ["start first", "closed first", "start second", "closed second"], + ) + ), + id="restart", + ), + pytest.param( + [], + snapshot(([], ["start first"], ["start first", "closed first"])), + id="disconnect-parks", + ), + ], +) +async def test_ending_a_session_closes_its_template( + tmp_path: Path, ending: list[tuple[str, dict[str, Any] | None]], expected: Any +) -> None: + log = tmp_path / "log" + env = protocol_env(log) + + async with LocalRuntime(env)(Task(env=env.name, id="protocol")) as runtime: + async with wire(runtime.url) as connection: + await connection.call("tasks.start", {"id": "guarded", "args": {"name": "first"}}) + replies = [await connection.call(method, params) for method, params in ending] + while_served = lines(log) + + assert (replies, while_served, lines(log)) == expected + + +def big_args_start(size: int) -> dict[str, Any]: + """A ``tasks.start`` frame of exactly ``size`` bytes, excluding the newline.""" + frame: dict[str, Any] = { + "jsonrpc": "2.0", + "id": 1, + "method": "tasks.start", + "params": {"id": "echo", "args": {"data": ""}}, + } + frame["params"]["args"]["data"] = "x" * (size - (len(encode(frame)) - 1)) + return frame + + +def oversized_env(kind: str, finished: list[str]) -> Environment: + env = Environment("oversized") + huge = "x" * FRAME_LIMIT + + @env.template() + async def echo(data: str): + yield data + yield {"score": 1.0, "detail": data} + + @env.template() + async def bloated(): + try: + if kind == "error": + raise ValueError(huge) + yield huge if kind == "prompt" else "ready" + yield {"score": 1.0, "detail": huge} + finally: + finished.append(kind) + + return env + + +@pytest.mark.parametrize( + ("size", "accepted"), + [(72483, True), (1024 * 1024, True), (FRAME_LIMIT, True), (FRAME_LIMIT + 1, False)], +) +async def test_a_request_up_to_the_frame_limit_round_trips(size: int, accepted: bool) -> None: + env = oversized_env("prompt", []) + frame = big_args_start(size) + data = frame["params"]["args"]["data"] + + async with ( + LocalRuntime(env)(Task(env=env.name, id="echo")) as runtime, + wire(runtime.url) as connection, + ): + await connection.write(encode(frame)) + started = await connection.read() + graded = await connection.call("tasks.grade", {"answer": "done"}) if accepted else None + + if accepted: + assert started is not None and started["result"]["prompt"] == data + assert graded is not None and graded["result"]["detail"] == data + else: + assert started == { + "jsonrpc": "2.0", + "id": None, + "error": { + "code": -32000, + "message": IsStr(regex=rf"Control-channel frame .* {FRAME_LIMIT} bytes.*"), + }, + } + + +@pytest.mark.parametrize("kind", ["prompt", "grade", "error"]) +async def test_an_oversized_reply_is_an_error_frame_and_releases_the_template(kind: str) -> None: + finished: list[str] = [] + env = oversized_env(kind, finished) + + async with LocalRuntime(env)(Task(env=env.name, id="bloated")) as runtime: + async with wire(runtime.url) as connection: + started = await connection.call("tasks.start", {"id": "bloated"}) + graded = None + if "result" in started: + graded = await connection.call("tasks.grade", {"answer": "done"}) + listed = await connection.call("tasks.list") + async with wire(runtime.url) as later: + regrade = await later.call("tasks.grade", {"answer": "done"}) + + error = graded if graded is not None else started + assert error["error"] == { + "code": -32000, + "message": IsStr( + regex=rf"Control-channel response is \d+ bytes; limit is {FRAME_LIMIT} bytes .*" + ), + } + assert finished == [kind] + assert [task["id"] for task in listed["result"]["tasks"]] == ["echo", "bloated"] + assert regrade["error"]["message"] == "no task in progress" + + +@pytest.mark.parametrize( + ("preface", "field", "serving"), + [ + pytest.param(False, "padding", False, id="oversized-first-frame-hangs-up"), + pytest.param(True, "padding", False, id="oversized-frame-mid-session-hangs-up"), + pytest.param(False, "id", True, id="reply-too-large-for-its-id-keeps-serving"), + ], +) +async def test_an_oversized_frame_gets_a_bounded_error_without_an_id( + preface: bool, field: str, serving: bool +) -> None: + frame: dict[str, Any] = {"jsonrpc": "2.0", "id": "", "method": "nope"} + frame[field] = "x" * (FRAME_LIMIT - (len(encode(frame)) - 1) if field == "id" else FRAME_LIMIT) + env = Environment("wire-limit") + + async with ( + LocalRuntime(env)(Task(env=env.name, id="unused")) as runtime, + wire(runtime.url) as connection, + ): + if preface: + await connection.call("hello", {}) + await connection.write(encode(frame)) + reply = await connection.read() + if serving: + await connection.send("tasks.list") + after = await connection.read() + + assert reply == { + "jsonrpc": "2.0", + "id": None, + "error": { + "code": -32000, + "message": IsStr(regex=rf"Control-channel .* {FRAME_LIMIT} bytes.*"), + }, + } + assert (after is not None and after["result"] == {"tasks": []}) is serving + + +async def test_an_evaluation_result_reaches_the_runs_evaluate_step(tmp_path: Path) -> None: + env = protocol_env(tmp_path / "log") + + run = await rollout( + Task(env="protocol", id="model"), ScriptedAgent("x"), runtime=LocalRuntime(env) + ) + + assert run.reward == 0.75 + assert steps(run.trace_id)[-1] == snapshot( + { + "step_id": 3, + "source": "task", + "messages": [], + "task_call": { + "phase": "evaluate", + "name": "model", + "arguments": {"answer": "x"}, + "result": { + "done": True, + "content": "nice", + "info": {"max_tile": 256}, + "isError": False, + "subscores": [ + { + "name": "judge", + "weight": 1.0, + "value": 0.75, + "children": [ + { + "name": "criterion", + "weight": 1.0, + "value": 1.0, + "children": None, + "info": {"reason": "because"}, + } + ], + "info": {"model": "judge-model"}, + } + ], + "score": 0.75, + }, + }, + "started_at": IsStr(), + "ended_at": IsStr(), + "extra": {}, + } + ) + + +def attach_template_to_a_plain_function(env: Environment) -> None: + def not_a_generator() -> str: + return "x" + + env.template()(cast("Any", not_a_generator)) + + +def register_a_template_twice(env: Environment) -> None: + @env.template(id="twice") + async def first(): + yield "a" + + @env.template(id="twice") + async def second(): + yield "b" + + +@pytest.mark.parametrize( + ("declare", "error", "message"), + [ + pytest.param( + attach_template_to_a_plain_function, TypeError, "must be an async generator function" + ), + pytest.param(register_a_template_twice, ValueError, "'twice' already registered"), + pytest.param( + lambda env: [env.workspace("a"), env.workspace("b")], + ValueError, + "workspace capability 'shell' is already attached", + ), + pytest.param( + lambda env: env.capability("missing"), KeyError, "unknown capability: 'missing'" + ), + pytest.param( + lambda env: env.add_capability(cast("Any", {"name": "x"})), + TypeError, + "expected Capability", + ), + pytest.param( + lambda env: env.add_capability( + Capability(name="db", protocol="tcp", url="", params={}) + ), + ValueError, + "capability 'db' has no url", + ), + ], +) +def test_an_authoring_mistake_raises_at_definition( + declare: Callable[[Environment], Any], error: type[Exception], message: str +) -> None: + with pytest.raises(error, match=message): + declare(Environment("authoring")) + + +async def test_publishing_a_capability_after_initialize_warns( + caplog: pytest.LogCaptureFixture, +) -> None: + env = Environment("late") + await env.start() + + with caplog.at_level(logging.WARNING, logger="hud.environment"): + env.add_capability(Capability.cdp(url="ws://127.0.0.1:9/late")) + await env.stop() + + assert "add_capability('browser') called after @env.initialize hooks" in caplog.text diff --git a/tests/environment/test_manifest.py b/tests/environment/test_manifest.py deleted file mode 100644 index 8b7c9e2ba..000000000 --- a/tests/environment/test_manifest.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Manifest entries: the task contract published over ``tasks.list``. - -``args`` describes the task function's parameters (what stored task args -must satisfy — validated platform-side at sync time); ``input``/``returns`` -are the agent's declared I/O types. -""" - -from __future__ import annotations - -from typing import Literal - -from pydantic import BaseModel - -from hud.environment import Environment - - -class _Point(BaseModel): - x: int - y: int - - -class _Payload(BaseModel): - text: str - - -_Mode = Literal["easy", "hard"] -_Retries = int | None - - -def test_args_schema_captures_params_defaults_and_required() -> None: - env = Environment("manifests") - - @env.template() - async def fix_bug(difficulty: int, suite: str = "coding"): - yield "go" - yield 1.0 - - entry = env.tasks["fix_bug"].manifest_entry() - - schema = entry["args"] - assert set(schema["properties"]) == {"difficulty", "suite"} - assert schema["properties"]["difficulty"]["type"] == "integer" - assert schema["properties"]["suite"]["default"] == "coding" - assert schema["required"] == ["difficulty"] - assert schema["additionalProperties"] is False - - -def test_args_schema_for_no_param_task_rejects_args() -> None: - env = Environment("manifests") - - @env.template() - async def bare(): - yield "go" - yield 1.0 - - schema = env.tasks["bare"].manifest_entry()["args"] - assert schema["properties"] == {} - assert schema["additionalProperties"] is False - - -def test_args_schema_var_keyword_allows_additional() -> None: - env = Environment("manifests") - - @env.template() - async def flexible(n: int, **rest: str): - yield "go" - yield 1.0 - - schema = env.tasks["flexible"].manifest_entry()["args"] - assert set(schema["properties"]) == {"n"} - assert schema["additionalProperties"] is True - - -def test_args_schema_unannotated_param_accepts_anything() -> None: - env = Environment("manifests") - - @env.template() - async def loose(anything): - yield "go" - yield 1.0 - - schema = env.tasks["loose"].manifest_entry()["args"] - assert schema["required"] == ["anything"] - assert "type" not in schema["properties"]["anything"] - - -def test_input_and_returns_schemas_still_published() -> None: - env = Environment("manifests") - - @env.template(input=_Point, returns=_Point) - async def typed(): - yield "go" - yield 1.0 - - entry = env.tasks["typed"].manifest_entry() - assert entry["input"]["properties"]["x"]["type"] == "integer" - assert entry["returns"]["properties"]["y"]["type"] == "integer" - assert entry["args"]["properties"] == {} - - -def test_args_schema_resolves_postponed_rich_annotations() -> None: - env = Environment("manifests") - - @env.template() - async def rich(mode: _Mode, payload: _Payload, retries: _Retries = None): - yield "go" - yield 1.0 - - assert callable(rich) - schema = env.tasks["rich"].manifest_entry()["args"] - assert schema["properties"]["mode"]["enum"] == ["easy", "hard"] - assert schema["properties"]["retries"]["default"] is None - assert "$ref" in schema["properties"]["payload"] - assert set(schema["required"]) == {"mode", "payload"} diff --git a/tests/environment/test_server.py b/tests/environment/test_server.py deleted file mode 100644 index 3dc34e1b5..000000000 --- a/tests/environment/test_server.py +++ /dev/null @@ -1,248 +0,0 @@ -"""The wire grade contract: ``tasks.grade`` frames carry a numeric ``score``. - -The server normalizes every grade yield to the canonical frame and fails -loudly on authoring bugs (a grade that is neither a number, a ``.reward`` -object, nor a ``{"score": ...}`` dict) instead of silently grading 0.0. -""" - -from __future__ import annotations - -import asyncio -import contextlib -from typing import Literal -from urllib.parse import urlsplit - -import pytest -from pydantic import BaseModel - -from hud.clients import HudProtocolError, connect -from hud.environment import Answer, Environment -from hud.environment.utils import FrameTooLargeError, encode_frame, read_frame, send_frame -from hud.eval import LocalRuntime, Run, Task -from hud.graders import EvaluationResult, SubScore - -from .conftest import served - - -class _Payload(BaseModel): - text: str - - -_Mode = Literal["upper", "lower"] - - -async def test_dict_grade_without_numeric_score_errors_loudly() -> None: - env = Environment("badgrade") - - @env.template() - async def reward_keyed(): - yield "go" - yield {"reward": 1.0} # wrong key: the wire grade frame is {"score": ...} - - async with served(env) as client: - with pytest.raises(HudProtocolError, match="score"): - async with Run(client, "reward_keyed", {}) as run: - run.trace.content = "x" - - -async def test_non_numeric_grade_errors_loudly() -> None: - env = Environment("badgrade") - - @env.template() - async def stringy(): - yield "go" - yield "great job" - - async with served(env) as client: - with pytest.raises(HudProtocolError, match="yield a number"): - async with Run(client, "stringy", {}) as run: - run.trace.content = "x" - - -async def test_score_dict_passes_through_with_extra_keys() -> None: - env = Environment("richgrade") - - @env.template() - async def rich(): - yield "go" - yield {"score": 0.5, "info": {"detail": "partial credit"}} - - async with served(env) as client: - async with Run(client, "rich", {}) as run: - run.trace.content = "x" - assert run.reward == 0.5 - assert run.grade.info == {"detail": "partial credit"} - - -async def test_evaluation_result_info_reaches_evaluate_step() -> None: - env = Environment("modelgrade") - - @env.template() - async def graded(): - yield "go" - yield EvaluationResult( - reward=0.75, - content="nice", - info={"max_tile": 256}, - subscores=[ - SubScore( - name="judge", - value=0.75, - children=[ - SubScore( - name="criterion", - value=1.0, - info={"reason": "because"}, - ) - ], - info={"model": "judge-model"}, - ) - ], - ) - - async with served(env) as client: - async with Run(client, "graded", {}) as run: - run.trace.content = "x" - assert run.reward == 0.75 - assert run.grade.info == {"max_tile": 256} - assert "info" not in run.evaluation["subscores"][0] - assert "info" not in run.evaluation["subscores"][0]["children"][0] - evaluate_step = run.trace.steps[-1] - assert evaluate_step.task_call is not None - assert evaluate_step.task_call.phase == "evaluate" - assert evaluate_step.task_call.result == { - "score": 0.75, - "done": True, - "content": "nice", - "info": {"max_tile": 256}, - "isError": False, - "subscores": [ - { - "name": "judge", - "weight": 1.0, - "value": 0.75, - "children": [ - { - "name": "criterion", - "weight": 1.0, - "value": 1.0, - "children": None, - "info": {"reason": "because"}, - } - ], - "info": {"model": "judge-model"}, - } - ], - } - - -def test_answer_holds_parsed_content_and_raw_string() -> None: - answer = Answer(content={"final": "42"}, raw='{"final": "42"}') - assert answer.content == {"final": "42"} - assert answer.raw == '{"final": "42"}' - - -async def test_start_coerces_postponed_rich_annotations() -> None: - env = Environment("coerce") - - @env.template() - async def typed(mode: _Mode, payload: _Payload, retries: int | None = None): - if mode == "upper": - prompt = payload.text.upper() - elif mode == "lower": - prompt = payload.text.lower() - else: - raise ValueError(f"unexpected mode: {mode!r}") - if retries is not None: - prompt += "!" * retries - yield prompt - yield 1.0 - - assert callable(typed) - async with served(env) as client: - async with Run( - client, - "typed", - { - "mode": '"upper"', - "payload": '{"text":"hello"}', - "retries": "3", - }, - ) as run: - run.trace.content = "x" - assert run.prompt == "HELLO!!!" - - -@pytest.mark.parametrize("frame_size", [72483, 1024 * 1024, 16777216, 16777217]) -async def test_task_frame_limit_and_retry(frame_size: int) -> None: - env = Environment("frame-limit") - - @env.template() - async def task(data: str): - yield data - yield {"score": 1.0, "detail": data} - - frame = { - "jsonrpc": "2.0", - "id": 2, - "method": "tasks.start", - "params": {"id": "task", "args": {"data": ""}}, - } - data = "x" * (frame_size - (len(encode_frame(frame)) - 1)) - async with served(env) as client: - if frame_size > 16777216: - with pytest.raises(FrameTooLargeError, match="16777217 bytes; limit is 16777216 bytes"): - await client.start_task("task", {"data": data}) - data = "small retry" - assert (await client.start_task("task", {"data": data}))["prompt"] == data - assert (await client.grade({"answer": "done"}))["detail"] == data - - -@pytest.mark.parametrize("result_kind", ["prompt", "grade", "error"]) -async def test_oversized_response_releases_task_and_reports_error(result_kind: str) -> None: - env = Environment("large-response") - released = asyncio.Event() - - @env.template() - async def task(): - try: - if result_kind == "error": - raise ValueError("x" * (16 * 1024 * 1024)) - yield "x" * (16 * 1024 * 1024) if result_kind == "prompt" else "ready" - yield {"score": 1.0, "detail": "x" * (16 * 1024 * 1024)} - finally: - released.set() - - async with LocalRuntime(env)(Task(env=env.name, id="task")) as runtime: - async with connect(runtime) as client: - with pytest.raises(HudProtocolError, match=r"response is .*limit is 16777216 bytes"): - await client.start_task("task") - await client.grade({"answer": "done"}) - assert released.is_set() - assert (await client.list_tasks())[0]["id"] == "task" - async with connect(runtime) as resumed: - with pytest.raises(HudProtocolError, match="no task in progress"): - await resumed.grade({"answer": "done"}) - - -@pytest.mark.parametrize("hello_first,large_id", [(False, False), (True, False), (False, True)]) -async def test_wire_size_errors_are_bounded(hello_first: bool, large_id: bool) -> None: - frame = {"jsonrpc": "2.0", "id": "", "method": "nope"} - field = "id" if large_id else "padding" - frame[field] = "x" * (16777216 - (len(encode_frame(frame)) - 1) if large_id else 16777216) - env = Environment("wire-limit") - async with LocalRuntime(env)(Task(env=env.name, id="unused")) as runtime: - address = urlsplit(runtime.url) - reader, writer = await asyncio.open_connection(address.hostname, address.port) - try: - if hello_first: - await send_frame(writer, {"jsonrpc": "2.0", "id": 1, "method": "hello"}) - assert await read_frame(reader) is not None - writer.write(encode_frame(frame)) - response = await asyncio.wait_for(read_frame(reader), timeout=5.0) - assert response is not None and response["id"] is None - assert "16777216 bytes" in response["error"]["message"] - finally: - writer.close() - with contextlib.suppress(ConnectionError): - await writer.wait_closed() diff --git a/tests/environment/test_sessions.py b/tests/environment/test_sessions.py index d5f660dee..6940194e2 100644 --- a/tests/environment/test_sessions.py +++ b/tests/environment/test_sessions.py @@ -1,31 +1,35 @@ -"""Control-channel sessions: each connection drives its own task. +"""Control sessions: each connection drives its own task, and a dropped one parks it. -Suspended tasks are keyed by session id: concurrent connections start and -grade independently, a dropped connection parks its task, and a later -connection grades a parked task by resuming the session (``hello`` with its -id) or — only when exactly one is parked — by a plain ``tasks.grade`` (the -split ``hud task start`` / ``hud task grade`` flow). +A session is keyed by the id ``hello`` returns. Concurrent connections start and +grade independently; a connection that drops leaves its task parked for a later +connection to grade, by resuming the session (``hello`` with its id) or, when +exactly one session is parked, by a plain ``tasks.grade`` (the split ``hud task +start`` / ``hud task grade`` flow). """ from __future__ import annotations -import asyncio -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest +from dirty_equals import IsStr from hud.capabilities import Connection from hud.clients import HudProtocolError, connect -from hud.environment import Environment, Workspace, WorkspaceRoute +from hud.environment import Environment, WorkspaceRoute from hud.eval import LocalRuntime, Task +from .conftest import Wire, wire + if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + from contextlib import AbstractAsyncContextManager from pathlib import Path -_SESSION = Task(env="sessions", id="echo") +ECHO = Task(env="sessions", id="echo") -def _env() -> Environment: +def echo_env() -> Environment: env = Environment("sessions") @env.template() @@ -36,161 +40,194 @@ async def echo(tag: str): return env -async def test_concurrent_sessions_grade_their_own_tasks() -> None: - async with ( - LocalRuntime(_env())(_SESSION) as runtime, - connect(runtime) as a, - connect(runtime) as b, - ): - await a.start_task("echo", {"tag": "a"}) - await b.start_task("echo", {"tag": "b"}) # must not disturb a's task - assert (await a.grade({"answer": "x"}))["tag"] == "a" - assert (await b.grade({"answer": "x"}))["tag"] == "b" - - -async def test_restart_replaces_only_the_sessions_own_task() -> None: - async with ( - LocalRuntime(_env())(_SESSION) as runtime, - connect(runtime) as a, - connect(runtime) as b, - ): - await b.start_task("echo", {"tag": "b"}) - await a.start_task("echo", {"tag": "first"}) - await a.start_task("echo", {"tag": "second"}) - assert (await a.grade({"answer": "x"}))["tag"] == "second" - assert (await b.grade({"answer": "x"}))["tag"] == "b" - - -@pytest.mark.parametrize("write_error", [None, ConnectionError, asyncio.CancelledError]) -async def test_disconnect_parks_the_task_for_a_later_connection( - monkeypatch: pytest.MonkeyPatch, write_error: type[BaseException] | None +def start(tag: str) -> tuple[str, dict[str, Any]]: + return "tasks.start", {"id": "echo", "args": {"tag": tag}} + + +GRADE = ("tasks.grade", {"answer": "x"}) + +# A step is (connection, action, frame). ``call`` sends the frame and reads the +# reply; ``drop`` sends it and hangs up without reading; ``close`` hangs up. A +# hang-up returns once the server has ended the connection. ``{"resume": "a"}`` +# in hello params stands for connection a's session id. +SCRIPTS: list[Any] = [ + pytest.param( + [ + ("a", "call", start("a")), + ("b", "call", start("b")), + ("a", "call", GRADE), + ("b", "call", GRADE), + ], + ["go a", "go b", {"score": 1.0, "tag": "a"}, {"score": 1.0, "tag": "b"}], + id="concurrent-sessions-grade-their-own-tasks", + ), + pytest.param( + [ + ("b", "call", start("b")), + ("a", "call", start("first")), + ("a", "call", start("second")), + ("a", "call", GRADE), + ("b", "call", GRADE), + ], + [ + "go b", + "go first", + "go second", + {"score": 1.0, "tag": "second"}, + {"score": 1.0, "tag": "b"}, + ], + id="restart-replaces-only-the-sessions-own-task", + ), + pytest.param( + [ + ("a", "drop", start("parked")), + ("b", "call", ("hello", {"resume": "a"})), + ("b", "call", GRADE), + ], + [None, "resumed a", {"score": 1.0, "tag": "parked"}], + id="drop-before-the-prompt-parks-the-task-for-resume", + ), + pytest.param( + [("a", "call", start("parked")), ("a", "close", None), ("b", "call", GRADE)], + ["go parked", None, {"score": 1.0, "tag": "parked"}], + id="a-plain-grade-adopts-the-only-parked-session", + ), + pytest.param( + [ + ("a", "call", start("one")), + ("a", "close", None), + ("b", "call", start("two")), + ("b", "close", None), + ("c", "call", GRADE), + ("c", "call", ("hello", {"resume": "b"})), + ("c", "call", GRADE), + ], + [ + "go one", + None, + "go two", + None, + {"error": IsStr(regex=r"2 parked sessions \(sess-\w+, sess-\w+\); resume one .*")}, + "resumed b", + {"score": 1.0, "tag": "two"}, + ], + id="several-parked-sessions-need-a-resume", + ), + pytest.param( + [("a", "call", start("a")), ("b", "call", ("hello", {"resume": "a"}))], + ["go a", {"error": IsStr(regex=r"session 'sess-\w+' has a live connection")}], + id="a-live-session-cannot-be-resumed", + ), +] + + +async def play(url: str, script: list[tuple[str, str, Any]]) -> list[Any]: + connections: dict[str, Wire] = {} + sessions: dict[str, str] = {} + contexts: list[AbstractAsyncContextManager[Wire]] = [] + seen: list[Any] = [] + try: + for name, action, frame in script: + if name not in connections: + context = wire(url) + contexts.append(context) + connections[name] = await context.__aenter__() + sessions[name] = (await connections[name].call("hello", {}))["result"]["session_id"] + connection = connections[name] + if action == "close": + await connection.hang_up() + seen.append(None) + continue + method, params = frame + if "resume" in params: + params = {"session_id": sessions[params["resume"]]} + if action == "drop": + await connection.send(method, params) + await connection.hang_up() + seen.append(None) + continue + reply = await connection.call(method, params) + seen.append(outcome(reply, sessions)) + finally: + for context in reversed(contexts): + await context.__aexit__(None, None, None) + return seen + + +def outcome(reply: dict[str, Any], sessions: dict[str, str]) -> Any: + if "error" in reply: + return {"error": reply["error"]["message"]} + result = reply["result"] + if "prompt" in result: + return result["prompt"] + if "session_id" in result: + owner = next(name for name, sid in sessions.items() if sid == result["session_id"]) + return f"resumed {owner}" + return result + + +@pytest.mark.parametrize(("script", "expected"), SCRIPTS) +async def test_each_session_keeps_its_own_task( + script: list[tuple[str, str, Any]], expected: list[Any] ) -> None: - real_write = asyncio.StreamWriter.write - - def write(writer: asyncio.StreamWriter, data: bytes) -> None: - if write_error is not None and b'"prompt":' in data: - writer.close() - raise write_error() - real_write(writer, data) - - monkeypatch.setattr(asyncio.StreamWriter, "write", write) - async with LocalRuntime(_env())(_SESSION) as runtime: - async with connect(runtime) as first: - assert first.manifest is not None - session_id = first.manifest.session_id - if write_error is None: - await first.start_task("echo", {"tag": "parked"}) - else: - with pytest.raises(EOFError): - await first.start_task("echo", {"tag": "parked"}) - async with connect(runtime) as later: - await later.hello(session_id=session_id) - assert (await later.grade({"answer": "x"}))["tag"] == "parked" - - -async def test_grade_with_multiple_parked_sessions_errors_loudly() -> None: - async with LocalRuntime(_env())(_SESSION) as runtime: - for tag in ("one", "two"): - async with connect(runtime) as client: - await client.start_task("echo", {"tag": tag}) - async with connect(runtime) as later: - with pytest.raises(HudProtocolError, match="parked sessions"): - await later.grade({"answer": "x"}) - - -async def test_hello_resumes_a_parked_session_by_id() -> None: - async with LocalRuntime(_env())(_SESSION) as runtime: - ids: dict[str, str] = {} - for tag in ("one", "two"): - async with connect(runtime) as client: - assert client.manifest is not None - ids[tag] = client.manifest.session_id - await client.start_task("echo", {"tag": tag}) - async with connect(runtime) as later: - await later.hello(session_id=ids["two"]) - assert (await later.grade({"answer": "x"}))["tag"] == "two" - - -async def test_hello_with_an_unknown_session_id_errors() -> None: - async with LocalRuntime(_env())(_SESSION) as runtime, connect(runtime) as client: - with pytest.raises(HudProtocolError, match="unknown session"): - await client.hello(session_id="sess-nope") - + async with LocalRuntime(echo_env())(ECHO) as runtime: + assert await play(runtime.url, script) == expected -async def test_hello_cannot_resume_a_live_session() -> None: - async with ( - LocalRuntime(_env())(_SESSION) as runtime, - connect(runtime) as a, - connect(runtime) as b, - ): - assert a.manifest is not None - await a.start_task("echo", {"tag": "a"}) - with pytest.raises(HudProtocolError, match="live connection"): - await b.hello(session_id=a.manifest.session_id) - -async def test_a_finished_session_releases_its_connection_for_the_next_rollout( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(Workspace, "supports_process_connections", property(lambda _self: True)) - env = _env() - workspace = env.workspace(tmp_path / "root") - first, second = ( - Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": f"Bearer {token}"}, - ) - for token in ("first-rollout", "rotated") +def inference(token: str) -> Connection: + return Connection( + name="inference", + capability="ssh", + url="https://inference.hud.so", + headers={"Authorization": f"Bearer {token}"}, ) - async with LocalRuntime(env)(_SESSION) as runtime: - async with connect(runtime, connections=[first]) as a: - await a.start_task("echo", {"tag": "a"}) - assert (await a.grade({"answer": "x"}))["tag"] == "a" - async with connect(runtime, connections=[second]) as b: - await b.start_task("echo", {"tag": "b"}) - assert [peer.name for peer in workspace.peers].count(second.host) == 1 - assert (await b.grade({"answer": "x"}))["tag"] == "b" +async def a_finished_session_releases_its_connection(env: Environment, root: Path) -> None: + workspace = env.workspace(root, network=False) + async with LocalRuntime(env)(ECHO) as runtime: + async with connect(runtime, connections=[inference("first")]) as first: + await first.start_task("echo", {"tag": "a"}) + await first.grade({"answer": "x"}) + async with connect(runtime, connections=[inference("rotated")]) as second: + await second.start_task("echo", {"tag": "b"}) + assert [peer.name for peer in workspace.peers].count("inference.hud.so") == 1 + assert (await second.grade({"answer": "x"}))["tag"] == "b" -async def test_a_live_session_keeps_its_connection_name_from_other_sessions( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(Workspace, "supports_process_connections", property(lambda _self: True)) - env = _env() - env.workspace(tmp_path / "root") - first, second = ( - Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": f"Bearer {token}"}, - ) - for token in ("first", "second") - ) - async with LocalRuntime(env)(_SESSION) as runtime, connect(runtime, connections=[first]): +async def a_live_session_keeps_its_connection_name(env: Environment, root: Path) -> None: + env.workspace(root, network=False) + async with LocalRuntime(env)(ECHO) as runtime, connect(runtime, connections=[inference("a")]): with pytest.raises(HudProtocolError, match="bound to another control session"): - async with connect(runtime, connections=[second]): + async with connect(runtime, connections=[inference("b")]): pass -async def test_a_finished_session_takes_its_workspace_route_with_it( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(Workspace, "bwrap_available", property(lambda _self: True)) - env = _env() - workspace = env.workspace(tmp_path / "root", network=False) +async def a_finished_session_takes_its_route_with_it(env: Environment, root: Path) -> None: + workspace = env.workspace(root, network=False) route = WorkspaceRoute("ssh", "inference.hud.so", 443) - - async with LocalRuntime(env)(_SESSION) as runtime: - async with connect(runtime, workspace_routes=[route]) as a: - await a.start_task("echo", {"tag": "a"}) + async with LocalRuntime(env)(ECHO) as runtime: + async with connect(runtime, workspace_routes=[route]) as first: + await first.start_task("echo", {"tag": "a"}) assert [peer.name for peer in workspace.peers] == ["inference.hud.so"] - await a.grade({"answer": "x"}) + await first.grade({"answer": "x"}) async with connect(runtime) as later: await later.start_task("echo", {"tag": "later"}) assert workspace.peers == () await later.grade({"answer": "x"}) + + +@pytest.mark.e2e +@pytest.mark.sandbox +@pytest.mark.parametrize( + "scenario", + [ + a_finished_session_releases_its_connection, + a_live_session_keeps_its_connection_name, + a_finished_session_takes_its_route_with_it, + ], +) +async def test_a_sessions_workspace_bindings_end_with_the_session( + tmp_path: Path, scenario: Callable[[Environment, Path], Awaitable[None]] +) -> None: + await scenario(echo_env(), tmp_path / "root") diff --git a/tests/environment/test_tunnel.py b/tests/environment/test_tunnel.py index aef7b8e86..93c0067a1 100644 --- a/tests/environment/test_tunnel.py +++ b/tests/environment/test_tunnel.py @@ -1,29 +1,26 @@ """Capability tunneling: environment daemons reached through the control port. -A capability address belongs to the environment's network. The raw manifest -retains that address for processes executing there, while the client binding -points at a local forwarder. Each connection to it becomes one fresh connection -to the control port, opened with a ``tunnel.open`` preface frame and spliced raw -to the daemon. The preface is transport-level routing — a connection is a stream -or a control session from its first frame, never upgraded mid-session. These -tests drive that path end to end against a served env fronting a real TCP echo -server. +A capability address belongs to the environment's network. The manifest keeps +that address for processes running there, while a client binding points at a +local forwarder. Each connection to the forwarder becomes one connection to the +control port, opened with a ``tunnel.open`` preface frame and then spliced raw to +the daemon. These scenarios front a TCP echo daemon with a served env. """ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING -from unittest.mock import Mock +from typing import TYPE_CHECKING, Any from urllib.parse import urlsplit import pytest from hud.capabilities import Capability from hud.environment import Environment -from hud.environment.utils import encode_frame, read_frame, send_frame +from hud.eval import LocalRuntime, Task +from tests.harness import served -from .conftest import served +from .conftest import FRAME_LIMIT, encode, wire if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -31,11 +28,11 @@ @pytest.fixture async def echo_port() -> AsyncIterator[int]: - """A substrate-side TCP daemon: echoes every byte back.""" + """A daemon on the environment's network that echoes every byte back.""" async def echo(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: try: - while data := await reader.read(1024): + while data := await reader.read(65536): writer.write(data) await writer.drain() finally: @@ -47,109 +44,123 @@ async def echo(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> No await server.wait_closed() -def _echo_env(port: int) -> Environment: - cap = Capability(name="echo", protocol="rfb/3.8", url=f"rfb://127.0.0.1:{port}", params={}) - return Environment("echo-env", capabilities=[cap]) +def echo_env(port: int) -> Environment: + return Environment( + "echo-env", + capabilities=[Capability(name="echo", protocol="rfb/3.8", url=f"rfb://127.0.0.1:{port}")], + ) -def test_mcp_capabilities_materialize_default_tunnel_ports() -> None: - assert Capability.mcp(url="http://tools.example/mcp").url == "http://tools.example:80/mcp" - assert Capability.mcp(url="https://tools.example/mcp").url == "https://tools.example:443/mcp" - assert Capability.mcp(url="ws://tools.example/mcp").url == "ws://tools.example:80/mcp" - assert Capability.mcp(url="wss://tools.example/mcp").url == "wss://tools.example:443/mcp" - assert ( - Capability.mcp(url="https://user:secret@tools.example/mcp").url - == "https://user:secret@tools.example:443/mcp" - ) +async def round_trip(host: str | None, port: int | None, payload: bytes) -> bytes: + reader, writer = await asyncio.open_connection(host, port) + writer.write(payload) + await writer.drain() + data = await reader.readexactly(len(payload)) + writer.close() + await writer.wait_closed() + return data + +@pytest.mark.parametrize( + ("url", "materialized"), + [ + ("http://tools.example/mcp", "http://tools.example:80/mcp"), + ("https://tools.example/mcp", "https://tools.example:443/mcp"), + ("ws://tools.example/mcp", "ws://tools.example:80/mcp"), + ("wss://tools.example/mcp", "wss://tools.example:443/mcp"), + ("https://user:secret@tools.example/mcp", "https://user:secret@tools.example:443/mcp"), + ], +) +def test_an_mcp_capability_names_the_port_a_tunnel_dials(url: str, materialized: str) -> None: + assert Capability.mcp(url=url).url == materialized -async def test_bytes_round_trip_through_the_forwarded_binding(echo_port: int) -> None: - async with served(_echo_env(echo_port)) as client: + +async def test_the_binding_forwards_bytes_while_the_manifest_keeps_the_env_address( + echo_port: int, +) -> None: + async with served(echo_env(echo_port)) as client: assert client.manifest is not None assert urlsplit(client.manifest.bindings[0].url).port == echo_port - parts = urlsplit(client.binding("echo").url) - assert parts.port != echo_port # the binding points at the forwarder + binding = urlsplit(client.binding("echo").url) - reader, writer = await asyncio.open_connection(parts.hostname, parts.port) - writer.write(b"ping through the tunnel") - await writer.drain() - assert await reader.readexactly(23) == b"ping through the tunnel" - writer.close() - await writer.wait_closed() + echoed = await round_trip(binding.hostname, binding.port, b"ping through the tunnel") + assert binding.port != echo_port + assert echoed == b"ping through the tunnel" -async def test_concurrent_tunnel_streams_do_not_interleave(echo_port: int) -> None: - async with served(_echo_env(echo_port)) as client: - parts = urlsplit(client.binding("echo").url) - - async def stream(payload: bytes) -> bytes: - reader, writer = await asyncio.open_connection(parts.hostname, parts.port) - writer.write(payload) - await writer.drain() - data = await reader.readexactly(len(payload)) - writer.close() - await writer.wait_closed() - return data - - payloads = [f"stream-{i}".encode() * 100 for i in range(8)] - assert await asyncio.gather(*(stream(p) for p in payloads)) == payloads +async def test_concurrent_tunnel_streams_do_not_interleave(echo_port: int) -> None: + payloads = [f"stream-{i}".encode() * 100 for i in range(8)] -async def test_tunnel_open_for_an_unknown_capability_returns_an_error_frame( - echo_port: int, -) -> None: - async with served(_echo_env(echo_port)) as client: - assert client._endpoint is not None - reader, writer = await asyncio.open_connection(*client._endpoint) - await send_frame( - writer, - {"jsonrpc": "2.0", "id": 1, "method": "tunnel.open", "params": {"capability": "nope"}}, - ) - opened = await read_frame(reader) - assert opened is not None and "error" in opened - assert "nope" in opened["error"]["message"] - writer.close() - await writer.wait_closed() - - -async def test_tunnel_open_mid_session_is_not_a_method(echo_port: int) -> None: - """A control session never mutates into a stream: tunnel.open is a preface only.""" - async with served(_echo_env(echo_port)) as client: - assert client._endpoint is not None - reader, writer = await asyncio.open_connection(*client._endpoint) - await send_frame(writer, {"jsonrpc": "2.0", "id": 1, "method": "tasks.list"}) - assert await read_frame(reader) is not None # a control session is established - await send_frame( - writer, - {"jsonrpc": "2.0", "id": 2, "method": "tunnel.open", "params": {"capability": "echo"}}, + async with served(echo_env(echo_port)) as client: + binding = urlsplit(client.binding("echo").url) + echoed = await asyncio.gather( + *(round_trip(binding.hostname, binding.port, payload) for payload in payloads) ) - opened = await read_frame(reader) - assert opened is not None and "error" in opened - assert opened["error"]["code"] == -32601 # method not found - writer.close() - await writer.wait_closed() + assert echoed == payloads -async def test_closing_the_client_tears_down_its_forwarders(echo_port: int) -> None: - async with served(_echo_env(echo_port)) as client: - parts = urlsplit(client.binding("echo").url) + +async def test_closing_the_client_closes_its_forwarders(echo_port: int) -> None: + async with served(echo_env(echo_port)) as client: + binding = urlsplit(client.binding("echo").url) with pytest.raises(OSError): - _, writer = await asyncio.open_connection(parts.hostname, parts.port) - writer.close() - - -@pytest.mark.parametrize("frame_size", [70000, 16 * 1024 * 1024]) -async def test_buffered_frame_preserves_newline_boundary_and_backpressure(frame_size: int) -> None: - reader = asyncio.StreamReader() - transport = Mock(spec=asyncio.Transport) - reader.set_transport(transport) - frame = {"padding": ""} - frame["padding"] = "x" * (frame_size - (len(encode_frame(frame)) - 1)) - reader.feed_data(encode_frame(frame) + b"raw bytes") - assert await asyncio.wait_for(read_frame(reader, max_bytes=16 * 1024 * 1024), 1) == frame - assert await reader.readexactly(9) == b"raw bytes" - - transport.pause_reading.reset_mock() - reader.feed_data(b"x" * (128 * 1024 + 1)) - transport.pause_reading.assert_called_once() + await asyncio.open_connection(binding.hostname, binding.port) + + +@pytest.mark.parametrize("preface_size", [128, 70000, FRAME_LIMIT]) +async def test_raw_bytes_sent_with_the_preface_reach_the_daemon( + echo_port: int, preface_size: int +) -> None: + preface: dict[str, Any] = { + "jsonrpc": "2.0", + "id": 1, + "method": "tunnel.open", + "params": {"capability": "echo"}, + "padding": "", + } + preface["padding"] = "x" * (preface_size - (len(encode(preface)) - 1)) + env = echo_env(echo_port) + + async with ( + LocalRuntime(env)(Task(env=env.name, id="tunnel")) as runtime, + wire(runtime.url) as stream, + ): + await stream.write(encode(preface) + b"raw bytes") + opened = await stream.read() + echoed = await asyncio.wait_for(stream.reader.readexactly(9), timeout=30) + + assert opened == {"jsonrpc": "2.0", "id": 1, "result": {"capability": "echo"}} + assert echoed == b"raw bytes" + + +@pytest.mark.parametrize( + ("params", "error"), + [ + pytest.param( + {"capability": "nope"}, + {"code": -32000, "message": "\"unknown capability: 'nope'\""}, + id="unknown-capability", + ), + pytest.param( + {"capability": 7}, + {"code": -32602, "message": "tunnel.open: 'capability' must be a string"}, + id="non-string-capability", + ), + ], +) +async def test_a_refused_tunnel_gets_one_error_frame_and_is_closed( + echo_port: int, params: dict[str, Any], error: dict[str, Any] +) -> None: + env = echo_env(echo_port) + + async with ( + LocalRuntime(env)(Task(env=env.name, id="tunnel")) as runtime, + wire(runtime.url) as stream, + ): + refused = await stream.call("tunnel.open", params) + after = await stream.read() + + assert refused == {"jsonrpc": "2.0", "id": 1, "error": error} + assert after is None From 2ce5c071f3f28c9b4749d26585836ab50ec4f34b Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:14 -0700 Subject: [PATCH 005/248] fix(environment): reject a falsy non-object tasks.start args tasks.start read its args as `params.get("args") or {}`, so an empty list, zero or an empty string slipped past the object check as no args and surfaced as a confusing signature error (or ran a no-arg template). Only an absent or null args now means none. --- hud/environment/server.py | 4 +++- tests/environment/test_control_protocol.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/hud/environment/server.py b/hud/environment/server.py index a7c1bb6c2..784fd5ffd 100644 --- a/hud/environment/server.py +++ b/hud/environment/server.py @@ -409,7 +409,9 @@ async def error_to(msg_id: int | str | None, code: int, message: str) -> None: if not isinstance(task_id, str): await error_to(msg_id, -32602, "tasks.start: 'id' must be a string") continue - args = params.get("args") or {} + args = params.get("args") + if args is None: + args = {} if not isinstance(args, dict): await error_to(msg_id, -32602, "tasks.start: 'args' must be an object") continue diff --git a/tests/environment/test_control_protocol.py b/tests/environment/test_control_protocol.py index 7c987992b..e03615421 100644 --- a/tests/environment/test_control_protocol.py +++ b/tests/environment/test_control_protocol.py @@ -733,6 +733,25 @@ async def test_a_template_starts_and_grades_over_the_wire( ), id="start-list-args", ), + pytest.param( + [("tasks.start", {"id": "guarded", "args": []})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": "tasks.start: 'args' must be an object", + }, + } + ], + [], + ) + ), + id="start-empty-list-args", + ), pytest.param( [("tasks.start", {"id": "guarded", "args": {"nmae": "x"}})], snapshot( From cb431227d9ad886a646a1cef64b993c68f970266 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 006/248] fix(environment): report a template's KeyError as its own error tasks.start caught KeyError around the whole start, so a template that raised KeyError before its prompt (a missing dict key, an env lookup) was answered with "unknown task" naming the template that exists. The template id is now checked before starting, and anything the template raises reaches the peer as the error it is. --- hud/environment/server.py | 5 ++--- tests/environment/test_control_protocol.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/hud/environment/server.py b/hud/environment/server.py index 784fd5ffd..244d096a4 100644 --- a/hud/environment/server.py +++ b/hud/environment/server.py @@ -415,11 +415,10 @@ async def error_to(msg_id: int | str | None, code: int, message: str) -> None: if not isinstance(args, dict): await error_to(msg_id, -32602, "tasks.start: 'args' must be an object") continue - try: - prompt = await self.start(session_id, task_id, args) - except KeyError: + if task_id not in env.tasks: await error_to(msg_id, -32602, f"unknown task: {task_id!r}") continue + prompt = await self.start(session_id, task_id, args) try: await reply_to(msg_id, prompt) except FrameTooLargeError: diff --git a/tests/environment/test_control_protocol.py b/tests/environment/test_control_protocol.py index e03615421..17d4982a8 100644 --- a/tests/environment/test_control_protocol.py +++ b/tests/environment/test_control_protocol.py @@ -774,6 +774,22 @@ async def test_a_template_starts_and_grades_over_the_wire( ), id="start-signature-mismatch", ), + pytest.param( + [("tasks.start", {"id": "lookup"})], + snapshot( + ( + [ + { + "jsonrpc": "2.0", + "id": 1, + "error": {"code": -32000, "message": "'missing'"}, + } + ], + [], + ) + ), + id="start-template-raises-keyerror", + ), pytest.param( [("tasks.grade", {"answer": "x"})], snapshot( From 5b068099ab473ac902223ddb8d7f59dbdfed3ac9 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 007/248] test: drive file tracking over the filetracking/1 wire The tracker tests constructed FileTracker and patched its private size caps. Serve a tracking workspace over a fixture tree instead and compare each reply on the wire: the manifest without build output, VCS metadata, ignored files or secrets; agent edits with redacted secret placeholders and the oversized-file placeholder; the setup layer, advance, and a flush that captures deliverables, skips secret-shaped ones and flags a capture cap. An unknown method gets an error and the connection keeps serving. --- tests/environment/test_file_tracker.py | 343 --------------- tests/environment/test_file_tracking.py | 540 +++++++++++++++++++++--- 2 files changed, 470 insertions(+), 413 deletions(-) delete mode 100644 tests/environment/test_file_tracker.py diff --git a/tests/environment/test_file_tracker.py b/tests/environment/test_file_tracker.py deleted file mode 100644 index 6da1c48bf..000000000 --- a/tests/environment/test_file_tracker.py +++ /dev/null @@ -1,343 +0,0 @@ -"""FileTracker: snapshot diffing, excludes, gitignore, and the secrets deny-list.""" - -from __future__ import annotations - -import base64 -from typing import TYPE_CHECKING - -from hud.environment.file_tracker import FileTracker - -if TYPE_CHECKING: - from pathlib import Path - - import pytest - - -def test_modified_file_produces_a_unified_diff(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("line1\nline2\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "a.txt").write_text("line1\nCHANGED\n") - diff = tracker.take_snapshot() - - assert diff.files_changed == 1 - patch = diff.patches[0] - assert patch.rel_path == "a.txt" - assert patch.status == "modified" - assert "CHANGED" in patch.patch - - -def test_added_and_deleted_files(tmp_path: Path) -> None: - (tmp_path / "keep.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "new.txt").write_text("hello\n") - (tmp_path / "keep.txt").unlink() - diff = tracker.take_snapshot() - - by_path = {p.rel_path: p for p in diff.patches} - assert by_path["new.txt"].status == "added" - assert by_path["new.txt"].size_before == 0 - assert by_path["keep.txt"].status == "deleted" - - -def test_no_changes_yields_empty_diff(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - diff = tracker.take_snapshot() - - assert diff.files_changed == 0 - assert diff.patches == [] - - -def test_exclude_patterns_skip_build_output(tmp_path: Path) -> None: - (tmp_path / "node_modules").mkdir() - (tmp_path / "node_modules" / "dep.js").write_text("module.exports = 1;\n") - go_cache = tmp_path / ".tmp" / "go-build123" - go_cache.mkdir(parents=True) - (go_cache / "_pkg_.a").write_text("compiled\n") - (tmp_path / "go-build.go").write_text("package main\n") - (tmp_path / "src.py").write_text("x = 1\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest_paths = {entry["path"] for entry in tracker.current_manifest()} - assert "go-build.go" in manifest_paths - assert "src.py" in manifest_paths - assert not any(p.startswith("node_modules/") for p in manifest_paths) - assert not any(p.startswith(".tmp/") for p in manifest_paths) - - -def test_gitignore_is_honored(tmp_path: Path) -> None: - (tmp_path / ".gitignore").write_text("ignored.txt\n") - (tmp_path / "ignored.txt").write_text("secret-ish\n") - (tmp_path / "tracked.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest_paths = {entry["path"] for entry in tracker.current_manifest()} - assert "tracked.txt" in manifest_paths - assert "ignored.txt" not in manifest_paths - - -def test_vcs_metadata_dirs_are_excluded(tmp_path: Path) -> None: - git_dir = tmp_path / ".git" - git_dir.mkdir() - (git_dir / "config").write_text('[remote "origin"]\nurl = https://token@example.com/repo.git\n') - (tmp_path / "tracked.txt").write_text("x\n") - - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest_paths = {entry["path"] for entry in tracker.current_manifest()} - assert "tracked.txt" in manifest_paths - assert not any(path == ".git" or path.startswith(".git/") for path in manifest_paths) - - (git_dir / "config").write_text( - '[remote "origin"]\nurl = https://new-token@example.com/repo.git\n' - ) - diff = tracker.take_snapshot() - - assert diff.files_changed == 0 - assert diff.patches == [] - - -def test_secret_files_are_tracked_but_content_is_never_emitted(tmp_path: Path) -> None: - (tmp_path / ".env").write_text("API_KEY=supersecretvalue\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - assert tracker.current_manifest() == [] - - (tmp_path / ".env").write_text("API_KEY=supersecretvalue\nDB_PASSWORD=hunter2\n") - diff = tracker.take_snapshot() - - assert diff.files_changed == 1 - patch = diff.patches[0] - assert patch.rel_path == ".env" - assert patch.status == "modified" - # The change is detected, but the content is redacted — never in the patch. - assert "redacted" in patch.patch.lower() - assert "supersecretvalue" not in patch.patch - assert "hunter2" not in patch.patch - - -def test_vcs_config_files_are_redacted(tmp_path: Path) -> None: - (tmp_path / ".gitmodules").write_text( - '[submodule "private"]\nurl = https://token@example.com/private.git\n' - ) - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / ".gitmodules").write_text( - '[submodule "private"]\nurl = https://new-token@example.com/private.git\n' - ) - diff = tracker.take_snapshot() - - assert diff.files_changed == 1 - patch = diff.patches[0] - assert patch.rel_path == ".gitmodules" - assert "redacted" in patch.patch.lower() - assert "token@example.com" not in patch.patch - assert "new-token@example.com" not in patch.patch - - -def test_capture_changed_deliverables_since_setup_snapshot(tmp_path: Path) -> None: - (tmp_path / "setup.xlsx").write_bytes(b"setup workbook") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - tracker.take_setup_snapshot() - - (tmp_path / "analysis.py").write_text("print('tracked by diff')\n") - (tmp_path / "notes.txt").write_text("tracked by diff\n") - (tmp_path / "deliverable.xlsx").write_bytes(b"workbook bytes") - (tmp_path / "report.html").write_text("report\n") - - payload = tracker.flush_changes()["capture"] - - assert payload["files_changed"] == 4 - assert payload["files_eligible"] == 2 - assert payload["files_captured"] == 2 - assert payload["files_skipped"] == 0 - files = {file["path"]: file for file in payload["files"]} - - xlsx = files["deliverable.xlsx"] - assert xlsx["status"] == "added" - assert ( - xlsx["content_type"] == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ) - assert len(xlsx["content_hash"]) == 64 - assert base64.b64decode(xlsx["file"]["data"]) == b"workbook bytes" - assert xlsx["file"]["type"] == "file" - assert xlsx["file"]["media_type"] == xlsx["content_type"] - - html = files["report.html"] - assert html["status"] == "added" - assert html["content_type"] == "text/html" - assert base64.b64decode(html["file"]["data"]) == b"report\n" - assert html["file"]["media_type"] == "text/html" - - -def test_capture_skips_secret_shaped_files(tmp_path: Path) -> None: - (tmp_path / ".env.xlsx").write_bytes(b"API_KEY=before") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / ".env.xlsx").write_bytes(b"API_KEY=after") - payload = tracker.flush_changes()["capture"] - - assert payload["files_changed"] == 1 - assert payload["files_eligible"] == 0 - assert payload["files_captured"] == 0 - assert payload["files"] == [] - - -def test_capture_marks_cap_skips_as_truncated( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(FileTracker, "_MAX_CAPTURE_FILE_BYTES", 4) - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "large.pdf").write_bytes(b"12345") - payload = tracker.flush_changes()["capture"] - - assert payload["files_eligible"] == 1 - assert payload["files_captured"] == 0 - assert payload["files_skipped"] == 1 - assert payload["truncated"] is True - - -def test_per_file_diff_cap_emits_a_placeholder( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(FileTracker, "_MAX_DIFF_FILE_BYTES", 4) - (tmp_path / "big.txt").write_text("aaaaaaaa\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "big.txt").write_text("bbbbbbbb\n") - diff = tracker.take_snapshot() - - assert diff.files_changed == 1 - assert "too large to diff" in diff.patches[0].patch - - -def test_budget_skipped_files_stay_pending_until_emitted( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - (tmp_path / "a.txt").write_text("a1\n") - (tmp_path / "b.txt").write_text("b1\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - (tmp_path / "a.txt").write_text("a2\n") - (tmp_path / "b.txt").write_text("b2\n") - - # Cap so small no patch fits: both changes are skipped this poll. - monkeypatch.setattr(FileTracker, "_MAX_DIFF_BYTES", 1) - first = tracker.take_snapshot() - assert first.truncated - assert first.patches == [] - assert set(first.skipped_paths) == {"a.txt", "b.txt"} - - # Next poll with headroom: the skipped changes must not be lost — the - # baseline kept them pending, so they re-diff now. - monkeypatch.setattr(FileTracker, "_MAX_DIFF_BYTES", 50 * 1024 * 1024) - second = tracker.take_snapshot() - by_path = {p.rel_path: p for p in second.patches} - assert set(by_path) == {"a.txt", "b.txt"} - assert "a2" in by_path["a.txt"].patch - - -def test_nested_gitignore_is_not_honored_or_leaked(tmp_path: Path) -> None: - # Root has no .gitignore; a nested package does. With root-only honoring the - # nested rule must neither take effect locally nor leak tree-wide (the old - # basename match would have excluded "data.txt" everywhere). - pkg = tmp_path / "pkg" - pkg.mkdir() - (pkg / ".gitignore").write_text("data.txt\n") - (pkg / "data.txt").write_text("local\n") - (tmp_path / "data.txt").write_text("root\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest_paths = {entry["path"] for entry in tracker.current_manifest()} - assert "data.txt" in manifest_paths - assert "pkg/data.txt" in manifest_paths - - -def test_manifest_carries_paths_and_hashes(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - manifest = tracker.current_manifest() - - assert len(manifest) == 1 - entry = manifest[0] - assert entry["path"] == "a.txt" - assert entry["size"] == (tmp_path / "a.txt").stat().st_size - assert len(entry["content_hash"]) == 64 # sha256 hex - - -def test_setup_snapshot_returns_changes_and_resets_agent_baseline(tmp_path: Path) -> None: - source = tmp_path / "source.py" - source.write_text("before\n") - tracker = FileTracker(tmp_path) - initial = tracker.take_baseline() - - source.write_text("after\n") - (tmp_path / "generated.txt").write_text("setup\n") - setup = tracker.take_setup_snapshot() - - patches = {patch.rel_path: patch for patch in setup.patches} - assert set(patches) == {"generated.txt", "source.py"} - assert patches["source.py"].content_hash_before == initial.files["source.py"].content_hash - assert patches["source.py"].content_hash_after == tracker.current_manifest()[1]["content_hash"] - assert tracker.take_snapshot().files_changed == 0 - - -def test_setup_snapshot_keeps_complete_agent_baseline_when_diff_is_capped( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - tracker = FileTracker(tmp_path) - tracker.take_baseline() - (tmp_path / "generated.txt").write_text("setup\n") - monkeypatch.setattr(FileTracker, "_MAX_DIFF_BYTES", 0) - - setup = tracker.take_setup_snapshot() - - assert setup.truncated is True - assert setup.patches == [] - assert tracker.take_snapshot().files_changed == 0 - - -def test_to_dict_shape_matches_wire_contract(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("1\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - (tmp_path / "a.txt").write_text("2\n") - - payload = tracker.take_snapshot().to_dict() - - assert set(payload) >= { - "snapshot_timestamp", - "scan_duration_ms", - "files_scanned", - "files_changed", - "patches", - } - assert set(payload["patches"][0]) == { - "path", - "status", - "patch", - "size_before", - "size_after", - "content_hash_before", - "content_hash_after", - } diff --git a/tests/environment/test_file_tracking.py b/tests/environment/test_file_tracking.py index 181d20011..ec4d195ca 100644 --- a/tests/environment/test_file_tracking.py +++ b/tests/environment/test_file_tracking.py @@ -1,82 +1,482 @@ -"""filetracking/1 wire roundtrip: serve a FileTracker, drive the wire directly.""" +"""``filetracking/1``: what a served workspace reports about its files, reply by reply. + +A workspace with ``track_files=True`` publishes a ``filetracking`` capability. +These scenarios serve one over a fixture tree holding build output, VCS +metadata, ignored files and credentials, edit the tree the way an agent would, +and compare each reply on the wire with a golden copy (timestamps normalized). +""" from __future__ import annotations -import asyncio import base64 +import hashlib +import re from typing import TYPE_CHECKING, Any -from hud.environment.file_tracker import FileTracker, serve_file_tracking -from hud.environment.utils import read_frame, send_frame +import pytest +from inline_snapshot import snapshot + +from hud.environment import Environment, Workspace +from tests.harness import served + +from .conftest import wire if TYPE_CHECKING: from pathlib import Path +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +#: Larger than the per-file diff limit (1 MiB). +BIG_TEXT = 1024 * 1024 + 1 +#: Larger than the per-file capture limit (5 MiB). +BIG_PDF = 5 * 1024 * 1024 + 1 + +TREE = { + "a.txt": "line1\nline2\n", + "keep.txt": "keep\n", + "src.py": "x = 1\n", + "go-build.go": "package main\n", + "data.txt": "root data\n", + "pkg/data.txt": "package data\n", + "pkg/.gitignore": "data.txt\n", + ".gitignore": "ignored.txt\n", + "ignored.txt": "ignored\n", + "node_modules/dep.js": "module.exports = 1;\n", + ".tmp/go-build123/_pkg_.a": "compiled\n", + ".git/config": '[remote "origin"]\nurl = https://token@example.com/repo.git\n', + ".gitmodules": '[submodule "private"]\nurl = https://token@example.com/private.git\n', + ".env": "API_KEY=supersecretvalue\n", + ".env.xlsx": "API_KEY=before-setup", + "setup.xlsx": "setup workbook", + "big.txt": "a" * BIG_TEXT, +} + +#: An agent's edits: modified, deleted, added, secret, excluded and oversized files. +EDITS = { + "a.txt": "line1\nCHANGED\n", + "keep.txt": None, + "new.txt": "hello\n", + ".env": "API_KEY=supersecretvalue\nDB_PASSWORD=hunter2\n", + ".gitmodules": '[submodule "private"]\nurl = https://new-token@example.com/private.git\n', + ".git/config": '[remote "origin"]\nurl = https://new-token@example.com/repo.git\n', + "ignored.txt": "still ignored, longer\n", + "node_modules/dep.js": "module.exports = 2; // edited\n", + "big.txt": "b" * (BIG_TEXT + 1), +} + +#: A rollout's deliverables, written after setup. +DELIVERABLES = { + "analysis.py": "print('tracked by diff')\n", + "deliverable.xlsx": "workbook bytes", + "report.html": "report\n", + ".env.xlsx": "API_KEY=after", + "large.pdf": "%" * BIG_PDF, +} + + +def write(root: Path, files: dict[str, str | None]) -> None: + for name, content in files.items(): + path = root / name + if content is None: + path.unlink() + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + +def normalized(value: Any) -> Any: + """Replace clocks and mtimes, which change between runs, and content hashes.""" + if isinstance(value, dict): + return { + key: "" + if key in {"snapshot_timestamp", "scan_duration_ms"} + else normalized(item) + for key, item in value.items() + } + if isinstance(value, list): + return [normalized(item) for item in value] + if isinstance(value, str): + value = re.sub(r"^(redacted|overlimit):(\d+):\d+$", r"\1:\2:", value) + value = re.sub(r"^[0-9a-f]{64}$", "", value) + if len(value) > 200: + return f"<{len(value)} chars>" + return value + + +async def test_file_tracking_reports_setup_agent_edits_and_deliverables(tmp_path: Path) -> None: + root = tmp_path / "root" + write(root, TREE) + env = Environment("tracked") + env.workspace(root, track_files=True) + + async with served(env) as client, wire(client.binding("filetracking").url) as tracking: + initial = await tracking.call("snapshot") + unchanged = await tracking.call("diff") + write(root, EDITS) + edited = await tracking.call("diff") + repeated = await tracking.call("diff") + write(root, {"generated.txt": "setup\n", "src.py": "x = 2 # setup\n"}) + setup = await tracking.call("setup") + after_setup = await tracking.call("diff") + write(root, {"legacy.txt": "legacy\n"}) + advanced = await tracking.call("advance") + after_advance = await tracking.call("diff") + write(root, DELIVERABLES) + flushed = await tracking.call("flush") -async def _call( - reader: asyncio.StreamReader, - writer: asyncio.StreamWriter, - method: str, -) -> dict[str, Any]: - await send_frame( - writer, - {"jsonrpc": "2.0", "id": 1, "method": method, "params": {}}, + assert normalized(initial) == snapshot( + { + "jsonrpc": "2.0", + "id": 1, + "result": { + "files": [ + {"path": ".gitignore", "size": 12, "content_hash": ""}, + {"path": "a.txt", "size": 12, "content_hash": ""}, + { + "path": "big.txt", + "size": 1048577, + "content_hash": "overlimit:1048577:", + }, + {"path": "data.txt", "size": 10, "content_hash": ""}, + {"path": "go-build.go", "size": 13, "content_hash": ""}, + {"path": "keep.txt", "size": 5, "content_hash": ""}, + {"path": "pkg/.gitignore", "size": 9, "content_hash": ""}, + {"path": "pkg/data.txt", "size": 13, "content_hash": ""}, + {"path": "setup.xlsx", "size": 14, "content_hash": ""}, + {"path": "src.py", "size": 6, "content_hash": ""}, + ], + "files_scanned": 10, + }, + } ) - reply = await read_frame(reader) - assert reply is not None - assert "error" not in reply - result = reply.get("result") - assert isinstance(result, dict) - return result - - -async def test_diff_snapshot_setup_roundtrip(tmp_path: Path) -> None: - (tmp_path / "a.txt").write_text("x\n") - tracker = FileTracker(tmp_path) - tracker.take_baseline() - - server = await serve_file_tracking(tracker) - host, port = server.sockets[0].getsockname()[:2] - reader, writer = await asyncio.open_connection(host, port) - try: - # Nothing changed yet. - assert (await _call(reader, writer, "diff"))["files_changed"] == 0 - - # An edit shows up as a diff on the next pull. - (tmp_path / "a.txt").write_text("x\ny\n") - diff = await _call(reader, writer, "diff") - assert diff["files_changed"] == 1 - assert diff["patches"][0]["path"] == "a.txt" - - # diff() advanced the baseline, so a re-pull is empty. - assert (await _call(reader, writer, "diff"))["files_changed"] == 0 - - # snapshot() returns the full manifest. - snapshot = await _call(reader, writer, "snapshot") - assert any(entry["path"] == "a.txt" for entry in snapshot["files"]) - - # advance() remains available to clients that predate setup diffs. - (tmp_path / "legacy.txt").write_text("legacy\n") - assert await _call(reader, writer, "advance") == {"advanced": True} - assert (await _call(reader, writer, "diff"))["files_changed"] == 0 - - # setup() emits scenario changes and starts a clean agent-edit layer. - (tmp_path / "b.txt").write_text("z\n") - setup = await _call(reader, writer, "setup") - assert setup["files_changed"] == 1 - assert setup["patches"][0]["path"] == "b.txt" - assert (await _call(reader, writer, "diff"))["files_changed"] == 0 - - # flush() returns the trailing diff plus bounded changed deliverables. - (tmp_path / "report.xlsx").write_bytes(b"spreadsheet") - flush = await _call(reader, writer, "flush") - assert flush["diff"]["files_changed"] == 1 - capture = flush["capture"] - assert capture["files_captured"] == 1 - assert capture["files"][0]["path"] == "report.xlsx" - assert base64.b64decode(capture["files"][0]["file"]["data"]) == b"spreadsheet" - finally: - writer.close() - await writer.wait_closed() - server.close() - await server.wait_closed() + assert normalized([unchanged, repeated, after_setup, after_advance]) == snapshot( + [ + { + "jsonrpc": "2.0", + "id": 2, + "result": { + "snapshot_timestamp": "", + "scan_duration_ms": "", + "files_scanned": 13, + "files_changed": 0, + "patches": [], + }, + }, + { + "jsonrpc": "2.0", + "id": 4, + "result": { + "snapshot_timestamp": "", + "scan_duration_ms": "", + "files_scanned": 13, + "files_changed": 0, + "patches": [], + }, + }, + { + "jsonrpc": "2.0", + "id": 6, + "result": { + "snapshot_timestamp": "", + "scan_duration_ms": "", + "files_scanned": 14, + "files_changed": 0, + "patches": [], + }, + }, + { + "jsonrpc": "2.0", + "id": 8, + "result": { + "snapshot_timestamp": "", + "scan_duration_ms": "", + "files_scanned": 15, + "files_changed": 0, + "patches": [], + }, + }, + ] + ) + assert normalized(edited) == snapshot( + { + "jsonrpc": "2.0", + "id": 3, + "result": { + "snapshot_timestamp": "", + "scan_duration_ms": "", + "files_scanned": 13, + "files_changed": 6, + "patches": [ + { + "path": "keep.txt", + "status": "deleted", + "patch": """\ +--- a/keep.txt ++++ b/keep.txt +@@ -1 +0,0 @@ +-keep\ +""", + "size_before": 5, + "size_after": 0, + "content_hash_before": "", + "content_hash_after": None, + }, + { + "path": "new.txt", + "status": "added", + "patch": """\ +--- a/new.txt ++++ b/new.txt +@@ -0,0 +1 @@ ++hello\ +""", + "size_before": 0, + "size_after": 6, + "content_hash_before": None, + "content_hash_after": "", + }, + { + "path": "a.txt", + "status": "modified", + "patch": """\ +--- a/a.txt ++++ b/a.txt +@@ -1,2 +1,2 @@ + line1 +-line2 ++CHANGED\ +""", + "size_before": 12, + "size_after": 14, + "content_hash_before": "", + "content_hash_after": "", + }, + { + "path": ".env", + "status": "modified", + "patch": "Secret file changed (content redacted): .env\n", + "size_before": 25, + "size_after": 45, + "content_hash_before": "redacted:25:", + "content_hash_after": "redacted:45:", + }, + { + "path": ".gitmodules", + "status": "modified", + "patch": "Secret file changed (content redacted): .gitmodules\n", + "size_before": 66, + "size_after": 70, + "content_hash_before": "redacted:66:", + "content_hash_after": "redacted:70:", + }, + { + "path": "big.txt", + "status": "modified", + "patch": "File too large to diff (1048577 -> 1048578 bytes): big.txt\n", + "size_before": 1048577, + "size_after": 1048578, + "content_hash_before": "overlimit:1048577:", + "content_hash_after": "overlimit:1048578:", + }, + ], + }, + } + ) + assert normalized(setup) == snapshot( + { + "jsonrpc": "2.0", + "id": 5, + "result": { + "snapshot_timestamp": "", + "scan_duration_ms": "", + "files_scanned": 14, + "files_changed": 2, + "patches": [ + { + "path": "generated.txt", + "status": "added", + "patch": """\ +--- a/generated.txt ++++ b/generated.txt +@@ -0,0 +1 @@ ++setup\ +""", + "size_before": 0, + "size_after": 6, + "content_hash_before": None, + "content_hash_after": "", + }, + { + "path": "src.py", + "status": "modified", + "patch": """\ +--- a/src.py ++++ b/src.py +@@ -1 +1 @@ +-x = 1 ++x = 2 # setup\ +""", + "size_before": 6, + "size_after": 15, + "content_hash_before": "", + "content_hash_after": "", + }, + ], + }, + } + ) + assert advanced == snapshot({"jsonrpc": "2.0", "id": 7, "result": {"advanced": True}}) + assert normalized(flushed) == snapshot( + { + "jsonrpc": "2.0", + "id": 9, + "result": { + "diff": { + "snapshot_timestamp": "", + "scan_duration_ms": "", + "files_scanned": 19, + "files_changed": 5, + "patches": [ + { + "path": "deliverable.xlsx", + "status": "added", + "patch": """\ +--- a/deliverable.xlsx ++++ b/deliverable.xlsx +@@ -0,0 +1 @@ ++workbook bytes\ +""", + "size_before": 0, + "size_after": 14, + "content_hash_before": None, + "content_hash_after": "", + }, + { + "path": ".env.xlsx", + "status": "modified", + "patch": "Secret file changed (content redacted): .env.xlsx\n", + "size_before": 20, + "size_after": 13, + "content_hash_before": "redacted:20:", + "content_hash_after": "redacted:13:", + }, + { + "path": "analysis.py", + "status": "added", + "patch": """\ +--- a/analysis.py ++++ b/analysis.py +@@ -0,0 +1 @@ ++print('tracked by diff')\ +""", + "size_before": 0, + "size_after": 25, + "content_hash_before": None, + "content_hash_after": "", + }, + { + "path": "report.html", + "status": "added", + "patch": """\ +--- a/report.html ++++ b/report.html +@@ -0,0 +1 @@ ++report\ +""", + "size_before": 0, + "size_after": 33, + "content_hash_before": None, + "content_hash_after": "", + }, + { + "path": "large.pdf", + "status": "added", + "patch": "File too large to diff (0 -> 5242881 bytes): large.pdf\n", + "size_before": 0, + "size_after": 5242881, + "content_hash_before": None, + "content_hash_after": "overlimit:5242881:", + }, + ], + }, + "capture": { + "snapshot_timestamp": "", + "scan_duration_ms": "", + "files_scanned": 19, + "files_changed": 5, + "files_eligible": 3, + "files_captured": 2, + "files_skipped": 1, + "files": [ + { + "path": "deliverable.xlsx", + "size": 14, + "content_hash": "", + "content_type": XLSX, + "file": { + "type": "file", + "data": "d29ya2Jvb2sgYnl0ZXM=", + "media_type": XLSX, + }, + "status": "added", + }, + { + "path": "report.html", + "size": 33, + "content_hash": "", + "content_type": "text/html", + "file": { + "type": "file", + "data": "PGh0bWw+PGJvZHk+cmVwb3J0PC9ib2R5PjwvaHRtbD4K", + "media_type": "text/html", + }, + "status": "added", + }, + ], + "truncated": True, + }, + }, + } + ) + manifest = {entry["path"]: entry["content_hash"] for entry in initial["result"]["files"]} + assert manifest["a.txt"] == hashlib.sha256(b"line1\nline2\n").hexdigest() + captured = { + file["path"]: base64.b64decode(file["file"]["data"]) + for file in flushed["result"]["capture"]["files"] + } + assert captured == { + "deliverable.xlsx": b"workbook bytes", + "report.html": b"report\n", + } + + +async def test_a_bad_request_gets_an_error_and_the_connection_keeps_serving( + tmp_path: Path, +) -> None: + root = tmp_path / "root" + write(root, {"a.txt": "one\n"}) + env = Environment("tracked") + env.workspace(root, track_files=True) + + async with served(env) as client, wire(client.binding("filetracking").url) as tracking: + refused = await tracking.call("rewind") + write(root, {"a.txt": "one\ntwo\n"}) + diff = await tracking.call("diff") + + assert refused == snapshot( + { + "jsonrpc": "2.0", + "id": 1, + "error": {"code": -32000, "message": "unknown filetracking method: 'rewind'"}, + } + ) + assert [patch["path"] for patch in diff["result"]["patches"]] == ["a.txt"] + + +@pytest.mark.parametrize("track_files", [True, False]) +def test_the_file_tracking_capability_needs_a_started_tracking_workspace( + tmp_path: Path, track_files: bool +) -> None: + workspace = Workspace(tmp_path, track_files=track_files) + + with pytest.raises(RuntimeError, match=r"file tracking not started"): + workspace.file_tracking_capability() From f59f1f616d5b011d601b1f8fc871547abf3f2ed4 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 008/248] test: resolve environment references from one fixture tree Collapse the loader tests into one table over a fixture tree holding every source shape a project has, each row resolving a reference to an environment name or the error a user sees. Add the serving process to the same tree: python -m hud.environment.server and hud serve announce their port, answer hello, and run the environment's shutdown hook when stopped; a reference that does not load exits with the load error. --- tests/environment/test_loader.py | 554 ++++++++++++++++--------------- 1 file changed, 294 insertions(+), 260 deletions(-) diff --git a/tests/environment/test_loader.py b/tests/environment/test_loader.py index f3ec4b22b..e85a8fad3 100644 --- a/tests/environment/test_loader.py +++ b/tests/environment/test_loader.py @@ -1,309 +1,343 @@ -"""``load_environment``: resolve env references — source paths, modules, factories.""" +"""Environment references: what ``load_environment`` and the serving process resolve. + +A fixture tree holds every shape of source a project has: an ``env.py``, a +directory of modules, a package with a factory beside its ``env.py``, packages +with relative imports and re-exports, a module shadowing a stdlib name, and task +modules that export rows. Each row resolves one reference to an environment name +or to the error a user sees. The serving rows run the real serving process on +the same tree: it announces its port, answers ``hello``, and runs the +environment's shutdown hook on SIGTERM. +""" from __future__ import annotations -import importlib +import asyncio import json +import os +import signal import sys -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Any import pytest from hud.environment import load_environment +from .conftest import wire + if TYPE_CHECKING: from collections.abc import Iterator - from pathlib import Path - -def test_load_environment_selects_by_attr_or_env_name(tmp_path) -> None: - module = tmp_path / "envs.py" - module.write_text( - """ -from hud import Environment -first = Environment("env-one") -second = Environment("env-two") -""".strip(), - encoding="utf-8", +def declare(name: str, variable: str = "env") -> str: + """Source declaring Environment ``name``, whose shutdown hook leaves a marker in cwd.""" + return ( + "from pathlib import Path\n" + "from hud import Environment\n" + f"{variable} = Environment({name!r})\n" + f"@{variable}.shutdown\n" + f"async def _stopped_{variable}():\n" + f" Path('stopped-{name}').write_text('stopped')\n" ) - assert load_environment(module, name="first").name == "env-one" - assert load_environment(module, name="env-two").name == "env-two" - with pytest.raises(ValueError, match="multiple Environments"): - load_environment(module) - with pytest.raises(ValueError, match="no Environment named 'missing'"): - load_environment(module, name="missing") - single = tmp_path / "single.py" - single.write_text("from hud import Environment\nenv = Environment('only')\n", encoding="utf-8") - assert load_environment(single).name == "only" +TEMPLATE = '@env.template(id="solve")\nasync def solve():\n yield "ok"\n yield 1.0\n' + +TREE = { + "env.py": declare("from-env-py"), + "envs/one.py": declare("from-tree", "foo"), + "pkg/__init__.py": ( + "from pathlib import Path\n" + "from hud.environment import Environment\n" + "def make_env(name='from-factory'):\n" + " env = Environment(name)\n" + " @env.shutdown\n" + " async def _stopped():\n" + " Path(f'stopped-{name}').write_text('stopped')\n" + " return env\n" + ), + "pkg/env.py": declare("from-pkg-source"), + "multi.py": declare("env-one", "first") + declare("env-two", "second"), + "single.py": declare("only"), + "alias.py": declare("shared") + "alias = env\n", + "twins.py": declare("shared", "one") + declare("shared", "two"), + "relpkg/__init__.py": "", + "relpkg/world.py": declare("relative"), + "relpkg/env.py": "from .world import env\n", + "reexport/__init__.py": "from .env import env\n", + "reexport/core.py": declare("reexported"), + "reexport/env.py": "from .core import env\n" + TEMPLATE, + "scan/scan_core.py": declare("scanned"), + "scan/scan_templates.py": "from scan_core import env\n" + TEMPLATE, + "scan/assembly.py": "from scan_core import env\nfrom scan_templates import solve\n", + "shadow/json.py": ( + "import json\nfrom hud import Environment\nenv = Environment(json.loads('\"local\"'))\n" + ), + "exports/bound_source.py": declare("bound") + + TEMPLATE + + "def make_task():\n return solve()\n", + "exports/as_task.py": "from bound_source import make_task\nrows = make_task()\n", + "exports/as_list.py": "from bound_source import make_task\nrows = [make_task()]\n", + "exports/as_tuple.py": "from bound_source import make_task\nrows = (make_task(),)\n", + "exports/as_taskset.py": ( + "from bound_source import make_task\nfrom hud.eval import Taskset\n" + "rows = Taskset('rows', [make_task()])\n" + ), + "modpkg/__init__.py": "", + "modpkg/factories.py": ( + "from hud.environment import Environment\n" + "env = Environment('declared')\n" + "def make_env(name='built'):\n return Environment(name)\n" + "class CallableEnvironment(Environment):\n" + " def __call__(self):\n raise AssertionError('called as a factory')\n" + "callable_env = CallableEnvironment('callable')\n" + "def not_env():\n return object()\n" + ), +} @pytest.fixture -def factory_module(request): - """An importable module exposing an env and a factory, cleaned up after.""" - import sys - from types import ModuleType - - from hud.environment import Environment - - name = f"_loader_target_{request.node.name}" - mod = ModuleType(name) - setattr(mod, "env", Environment("declared")) - setattr(mod, "make_env", lambda name="built": Environment(name)) - sys.modules[name] = mod - yield name - del sys.modules[name] - - -def test_module_factory_is_called_with_args(factory_module) -> None: - env = load_environment(factory_module, name="make_env", args={"name": "from-factory"}) - - assert env.name == "from-factory" - - -def test_module_env_attribute_is_returned_not_called(factory_module) -> None: - import sys - - from hud.environment import Environment - - class CallableEnvironment(Environment): - def __call__(self) -> None: - raise AssertionError("Environment instance was called as a factory") - - setattr(sys.modules[factory_module], "env", CallableEnvironment("callable")) - - assert load_environment(factory_module).name == "callable" - - -def test_module_factory_returning_non_environment_raises(factory_module) -> None: - import sys - - setattr(sys.modules[factory_module], "make_env", lambda: object()) - - with pytest.raises(ValueError, match="not an Environment"): - load_environment(factory_module, name="make_env") - - -def test_unresolvable_references_raise() -> None: - with pytest.raises(ModuleNotFoundError): - load_environment("no.such.module") - with pytest.raises(FileNotFoundError, match="no environment source"): - load_environment("missing/env.py") - with pytest.raises(ValueError, match="args= applies to factory targets"): - load_environment(__file__, args={"a": "b"}) - - -def test_package_dir_does_not_shadow_factory_target(tmp_path, monkeypatch) -> None: - # `mypkg:make_env` with a plain mypkg/ package in cwd is a module - # reference; source scanning is only for env-declaring source trees. - pkg = tmp_path / "shadowpkg" - pkg.mkdir() - (pkg / "__init__.py").write_text( - "from hud.environment import Environment\n\n" - "def make_env(name='shadowed'):\n return Environment(name)\n", - encoding="utf-8", - ) - monkeypatch.chdir(tmp_path) - monkeypatch.syspath_prepend(str(tmp_path)) - - env = load_environment("shadowpkg", name="make_env") - - assert env.name == "shadowed" - - -def test_named_attribute_resolves_a_package_that_also_has_env_py(tmp_path, monkeypatch) -> None: - # `pkg:make_env` addresses an attribute, so an env.py sitting inside the - # package must not capture it into a source scan. - pkg = tmp_path / "bothpkg" - pkg.mkdir() - (pkg / "__init__.py").write_text( - "from hud.environment import Environment\n\n" - "def make_env(name='from-factory'):\n return Environment(name)\n", - encoding="utf-8", - ) - (pkg / "env.py").write_text( - "from hud.environment import Environment\n\nenv = Environment('from-source')\n", - encoding="utf-8", - ) - monkeypatch.chdir(tmp_path) - monkeypatch.syspath_prepend(str(tmp_path)) - - assert load_environment("bothpkg", name="make_env").name == "from-factory" - # ...while a bare reference still scans the source tree. - assert load_environment("bothpkg").name == "from-source" - - -def test_every_reference_form_resolves(tmp_path, monkeypatch) -> None: - """The full matrix, in one place: the shapes that competed for the same - spelling are what made this resolution subtle.""" - (tmp_path / "env.py").write_text( - "from hud.environment import Environment\n\nenv = Environment('from-env-py')\n", - encoding="utf-8", - ) - envs = tmp_path / "envs" # a plain directory: an importable namespace package - envs.mkdir() - (envs / "one.py").write_text( - "from hud.environment import Environment\n\nfoo = Environment('from-tree')\n", - encoding="utf-8", - ) - pkg = tmp_path / "pkg" # a real package exposing a factory - pkg.mkdir() - (pkg / "__init__.py").write_text( - "from hud.environment import Environment\n\n" - "def make_env(name='from-factory'):\n return Environment(name)\n", - encoding="utf-8", - ) - (pkg / "env.py").write_text( - "from hud.environment import Environment\n\nenv = Environment('from-pkg-source')\n", - encoding="utf-8", - ) - monkeypatch.chdir(tmp_path) - monkeypatch.syspath_prepend(str(tmp_path)) - - # a source file, however it is spelled — including the common serve form - assert load_environment("env").name == "from-env-py" - assert load_environment("env", name="env").name == "from-env-py" - assert load_environment("env.py").name == "from-env-py" - assert load_environment(tmp_path / "env.py").name == "from-env-py" - - # a source tree, with a name selecting inside it - assert load_environment("envs", name="foo").name == "from-tree" - - # a package attribute: the factory wins over the env.py beside it - assert load_environment("pkg", name="make_env").name == "from-factory" - assert load_environment("pkg", name="make_env", args={"name": "x"}).name == "x" - # ...while a bare reference to the same package still scans its source - assert load_environment("pkg").name == "from-pkg-source" - - -def test_environment_reexports_are_not_ambiguous(tmp_path) -> None: - source = tmp_path / "env.py" - source.write_text('from hud import Environment\nenv = Environment("shared")\nalias = env\n') - assert load_environment(source, name="shared").name == "shared" - assert load_environment(source).name == "shared" - - -def test_distinct_environments_with_the_same_name_are_ambiguous(tmp_path) -> None: - source = tmp_path / "env.py" - source.write_text( - 'from hud import Environment\none = Environment("shared")\ntwo = Environment("shared")\n' - ) - with pytest.raises(ValueError, match="multiple Environments"): - load_environment(source, name="shared") +def tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: + """The source tree as the working directory and an import root, as in a project.""" + root = tmp_path / "project" + for name, content in TREE.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + monkeypatch.chdir(root) + monkeypatch.syspath_prepend(str(root)) + before = set(sys.modules) + yield root + for name in set(sys.modules) - before: + del sys.modules[name] + + +ROWS = [ + pytest.param("env", None, None, "from-env-py", id="bare-name-means-env-py"), + pytest.param("env", "env", None, "from-env-py", id="bare-name-with-attribute"), + pytest.param("env.py", None, None, "from-env-py", id="relative-file"), + pytest.param("{root}/env.py", None, None, "from-env-py", id="absolute-file"), + pytest.param("envs", "foo", None, "from-tree", id="directory-by-attribute"), + pytest.param("pkg", "make_env", None, "from-factory", id="package-factory"), + pytest.param("pkg", "make_env", {"name": "x"}, "x", id="package-factory-with-args"), + pytest.param("pkg", None, None, "from-pkg-source", id="bare-package-scans-its-source"), + pytest.param("multi.py", "first", None, "env-one", id="select-by-attribute"), + pytest.param("multi.py", "env-two", None, "env-two", id="select-by-env-name"), + pytest.param("single.py", None, None, "only", id="single-env-needs-no-name"), + pytest.param("alias.py", None, None, "shared", id="re-exported-env-is-one-env"), + pytest.param("alias.py", "shared", None, "shared", id="re-exported-env-by-name"), + pytest.param("relpkg/env.py", None, None, "relative", id="package-relative-import"), + pytest.param("reexport/env.py", None, None, "reexported", id="package-init-re-export"), + pytest.param("scan", "scanned", None, "scanned", id="directory-of-modules"), + pytest.param("exports/as_task.py", "bound", None, "bound", id="exported-task"), + pytest.param("exports/as_list.py", "bound", None, "bound", id="exported-task-list"), + pytest.param("exports/as_tuple.py", "bound", None, "bound", id="exported-task-tuple"), + pytest.param("exports/as_taskset.py", "bound", None, "bound", id="exported-taskset"), + pytest.param("shadow/json.py", None, None, "local", id="module-named-like-stdlib"), + pytest.param("shadow", None, None, "local", id="directory-with-stdlib-name"), + pytest.param("modpkg.factories", None, None, "declared", id="module-env-attribute"), + pytest.param( + "modpkg.factories", "make_env", {"name": "made"}, "made", id="module-factory-with-args" + ), + pytest.param( + "modpkg.factories", "callable_env", None, "callable", id="callable-env-is-not-called" + ), + pytest.param( + "multi.py", + None, + None, + (ValueError, "multiple Environments in multi.py; select one by name"), + id="several-envs-need-a-name", + ), + pytest.param( + "multi.py", + "missing", + None, + (ValueError, "no Environment named 'missing' found in multi.py"), + id="unknown-name", + ), + pytest.param( + "twins.py", + "shared", + None, + (ValueError, "multiple Environments in twins.py"), + id="distinct-envs-sharing-a-name", + ), + pytest.param( + "modpkg.factories", + "not_env", + None, + (ValueError, r"modpkg.factories:not_env resolved to , not an Env"), + id="factory-returning-something-else", + ), + pytest.param( + "no.such.module", None, None, (ModuleNotFoundError, "No module named 'no'"), id="no-module" + ), + pytest.param( + "missing/env.py", + None, + None, + (FileNotFoundError, "no environment source at missing/env.py"), + id="missing-file", + ), + pytest.param( + "env.py", + None, + {"a": "b"}, + (ValueError, "args= applies to factory targets, not source path env.py"), + id="args-on-a-source-file", + ), +] + + +@pytest.mark.parametrize(("reference", "name", "args", "expected"), ROWS) +def test_a_reference_resolves_to_one_environment( + tree: Path, + reference: str, + name: str | None, + args: dict[str, Any] | None, + expected: str | tuple[type[Exception], str], +) -> None: + target: str | Path = reference.format(root=tree) + if reference.startswith("{root}"): + target = Path(target) + if isinstance(expected, str): + assert load_environment(target, name=name, args=args).name == expected + else: + error, message = expected + with pytest.raises(error, match=message): + load_environment(target, name=name, args=args) -def test_source_supports_package_relative_imports(tmp_path) -> None: - package = tmp_path / "relative_env_package" - package.mkdir() - (package / "__init__.py").write_text("") - (package / "world.py").write_text( - 'from hud import Environment\nenv = Environment("relative")\n' - ) - (package / "env.py").write_text("from .world import env\n") - assert load_environment(package / "env.py").name == "relative" +def test_loading_a_source_again_returns_the_same_environment(tree: Path) -> None: + scanned = load_environment("scan", name="scanned") + package = load_environment("reexport/env.py") -def test_directory_imports_each_module_once(tmp_path) -> None: - (tmp_path / "scan_core.py").write_text( - 'from hud import Environment\nenv = Environment("scanned")\n' - ) - (tmp_path / "scan_templates.py").write_text( - "from scan_core import env\n" - '@env.template(id="solve")\nasync def solve():\n yield "ok"\n yield 1.0\n' - ) - (tmp_path / "assembly.py").write_text( - "from scan_core import env\nfrom scan_templates import solve\n" - ) - env = load_environment(tmp_path, name="scanned") - assert set(env.tasks) == {"solve"} - assert load_environment(tmp_path, name="scanned") is env - - -@pytest.mark.parametrize("export", ["task", "list", "tuple", "taskset"]) -def test_source_resolves_environments_from_exported_tasks(tmp_path, request, export) -> None: - name = f"bound_source_{export}" - (tmp_path / f"{name}.py").write_text( - 'from hud import Environment\nenv = Environment("bound")\n' - '@env.template(id="solve")\nasync def solve():\n yield "ok"\n yield 1.0\n' - "def make_task():\n return solve()\n" - ) - request.addfinalizer(lambda: sys.modules.pop(name, None)) - expression = { - "task": "make_task()", - "list": "[make_task()]", - "tuple": "(make_task(),)", - "taskset": 'Taskset("rows", [make_task()])', - }[export] - source = tmp_path / "tasks.py" - source.write_text( - f"from {name} import make_task\nfrom hud.eval import Taskset\nrows = {expression}\n" - ) - assert set(load_environment(source, name="bound").tasks) == {"solve"} + assert set(scanned.tasks) == {"solve"} + assert load_environment("scan", name="scanned") is scanned + assert load_environment(tree / "reexport", name="reexported") is package -@pytest.mark.parametrize("source", ["json.py", "."]) -def test_source_does_not_replace_an_imported_module(tmp_path, source) -> None: - (tmp_path / "json.py").write_text( - "import json\nfrom hud import Environment\nenv = Environment(json.loads('\"local\"'))\n" - ) - assert load_environment(tmp_path / source).name == "local" - assert importlib.import_module("json") is json - +@pytest.mark.parametrize("reference", ["shadow/json.py", "shadow"]) +def test_loading_a_module_named_like_the_stdlib_leaves_the_stdlib_imported( + tree: Path, reference: str +) -> None: + load_environment(reference) -def test_package_init_can_reexport_its_environment(tmp_path) -> None: - package = tmp_path / "reexported_env_package" - package.mkdir() - (package / "__init__.py").write_text("from .env import env\n") - (package / "core.py").write_text( - 'from hud import Environment\nenv = Environment("reexported")\n' - ) - (package / "env.py").write_text( - 'from .core import env\n@env.template(id="solve")\n' - 'async def solve():\n yield "ok"\n yield 1.0\n' - ) - env = load_environment(package / "env.py") - assert env.name == "reexported" - assert load_environment(package, name="reexported") is env + assert sys.modules["json"] is json @pytest.fixture -def package_sources(tmp_path: Path) -> Iterator[tuple[Path, Path]]: - packages = tuple(tmp_path / label / "source_root_package" for label in ("first", "second")) +def two_roots(tmp_path: Path) -> Iterator[tuple[Path, Path]]: + """The same package name under two source roots.""" + packages = tuple(tmp_path / root / "source_root_package" for root in ("first", "second")) for package in packages: package.mkdir(parents=True) (package / "__init__.py").write_text("from .env import env\n") - (package / "core.py").write_text( - f"from hud import Environment\nenv = Environment({package.parent.name!r})\n" - ) + (package / "core.py").write_text(declare(package.parent.name)) (package / "env.py").write_text("from .core import env\n") - try: - yield packages[0], packages[1] - finally: - for name in list(sys.modules): - if name == "source_root_package" or name.startswith("source_root_package."): - del sys.modules[name] + before = set(sys.modules) + yield packages[0], packages[1] + for name in set(sys.modules) - before: + del sys.modules[name] @pytest.mark.parametrize("source", ["env.py", "__init__.py", "."]) -def test_package_sources_reject_a_cached_namespace_from_another_root( - package_sources: tuple[Path, Path], source: str +def test_a_package_already_imported_from_another_root_is_refused( + two_roots: tuple[Path, Path], source: str ) -> None: - first, second = (package / source for package in package_sources) - assert load_environment(first).name == "first" + first, second = (package / source for package in two_roots) + assert load_environment(first).name == "first" with pytest.raises(ValueError, match="already imported from a different source root"): load_environment(second) - assert load_environment(first).name == "first" -def test_package_source_takes_precedence_over_other_import_roots( - package_sources: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +def test_a_package_source_wins_over_other_import_roots( + two_roots: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch ) -> None: - first, second = package_sources + first, second = two_roots monkeypatch.syspath_prepend(str(second.parent)) monkeypatch.syspath_prepend(str(first.parent)) assert load_environment(second / "env.py").name == "second" + + +SERVER = [sys.executable, "-m", "hud.environment.server"] +HUD_SERVE = [sys.executable, "-m", "hud.cli", "serve", "--port", "0"] + + +@pytest.mark.parametrize( + ("command", "served", "stop"), + [ + pytest.param([*SERVER, "env.py"], "from-env-py", signal.SIGTERM, id="server-sigterm"), + pytest.param( + [*SERVER, "multi.py", "--env", "env-two"], "env-two", signal.SIGTERM, id="server-env" + ), + pytest.param([*HUD_SERVE, "env:env"], "from-env-py", signal.SIGINT, id="hud-serve-ctrl-c"), + pytest.param( + [*HUD_SERVE, "pkg:make_env", "--arg", "name=demo"], + "demo", + signal.SIGINT, + id="hud-serve-factory", + ), + ], +) +async def test_the_serving_process_announces_its_port_and_runs_shutdown_hooks( + tree: Path, command: list[str], served: str, stop: signal.Signals +) -> None: + process = await asyncio.create_subprocess_exec( + *command, + cwd=tree, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env={**os.environ, "NO_COLOR": "1"}, + ) + try: + port = await announced_port(process) + async with wire(f"tcp://127.0.0.1:{port}") as control: + hello = await control.call("hello", {}) + process.send_signal(stop) + code = await asyncio.wait_for(process.wait(), timeout=60) + finally: + if process.returncode is None: + process.kill() + await process.wait() + + assert hello["result"]["env"]["name"] == served + assert code == 0 + assert [name for name in os.listdir(tree) if name.startswith("stopped-")] == [ + f"stopped-{served}" + ] + + +async def announced_port(process: asyncio.subprocess.Process) -> int: + assert process.stdout is not None + async with asyncio.timeout(60): + while line := await process.stdout.readline(): + text = line.decode().strip() + if text.startswith("HUD_SERVE_PORT="): + return int(text.removeprefix("HUD_SERVE_PORT=")) + assert process.stderr is not None + raise AssertionError(f"exited without a port:\n{(await process.stderr.read()).decode()}") + + +@pytest.mark.parametrize( + ("command", "error"), + [ + pytest.param([*SERVER, "multi.py"], "multiple Environments in multi.py", id="ambiguous"), + ], +) +async def test_the_serving_process_exits_with_the_load_error( + tree: Path, command: list[str], error: str +) -> None: + process = await asyncio.create_subprocess_exec( + *command, cwd=tree, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=60) + + assert process.returncode == 1 + assert b"HUD_SERVE_PORT=" not in stdout + assert error in stderr.decode() From d21b96ad57aa7fd8e800b5bdfe1b5f0f4681f551 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 009/248] fix(environment): name a missing .py source as missing load_environment raised FileNotFoundError for a missing source only when the reference contained a slash. A bare file reference such as `hud serve envv.py` fell through to a module import and failed with "No module named 'envv'". Any .py reference that is not on disk is now reported as a missing environment source. --- hud/environment/__init__.py | 2 +- tests/environment/test_loader.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hud/environment/__init__.py b/hud/environment/__init__.py index baa2f36e7..100832ade 100644 --- a/hud/environment/__init__.py +++ b/hud/environment/__init__.py @@ -91,7 +91,7 @@ def load_environment( raise ValueError(f"multiple Environments in {path}; select one by name") return next(iter(matched.values())) - if path.is_file() or "/" in str(target): + if path.suffix == ".py" or "/" in str(target): raise FileNotFoundError(f"no environment source at {target}") obj = getattr(importlib.import_module(str(target)), name or "env") env = obj if isinstance(obj, Environment) or not callable(obj) else obj(**args or {}) diff --git a/tests/environment/test_loader.py b/tests/environment/test_loader.py index e85a8fad3..6d788f2c5 100644 --- a/tests/environment/test_loader.py +++ b/tests/environment/test_loader.py @@ -180,6 +180,13 @@ def tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: (FileNotFoundError, "no environment source at missing/env.py"), id="missing-file", ), + pytest.param( + "nothing.py", + None, + None, + (FileNotFoundError, "no environment source at nothing.py"), + id="missing-file-in-cwd", + ), pytest.param( "env.py", None, @@ -328,6 +335,7 @@ async def announced_port(process: asyncio.subprocess.Process) -> int: ("command", "error"), [ pytest.param([*SERVER, "multi.py"], "multiple Environments in multi.py", id="ambiguous"), + pytest.param([*SERVER, "nothing.py"], "no environment source at nothing.py", id="missing"), ], ) async def test_the_serving_process_exits_with_the_load_error( From b2d581cf333a13315dd0662853144a288badff23 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 010/248] fix(environment): stop serving cleanly on SIGTERM from hud serve Only python -m hud.environment.server turned SIGTERM into a clean stop; hud serve called serve() directly, so a SIGTERM (docker stop on the harbor image, whose CMD is hud serve) killed the process without running the environment's shutdown hooks or stopping its daemons. serve() now owns the SIGTERM handling for every caller and returns normally when terminated. --- hud/environment/server.py | 47 ++++++++++++++++++++++---------- tests/environment/test_loader.py | 3 +- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/hud/environment/server.py b/hud/environment/server.py index 244d096a4..8adf1db65 100644 --- a/hud/environment/server.py +++ b/hud/environment/server.py @@ -574,7 +574,37 @@ async def _shutdown(server: asyncio.Server) -> None: async def serve(env: Environment, host: str = "127.0.0.1", port: int = 0) -> None: - """Start *env*'s daemons and serve its control channel until cancelled.""" + """Start *env*'s daemons and serve its control channel until cancelled or SIGTERM. + + SIGTERM (a container stop, the spawn provider's teardown) ends serving the + way cancellation does, so shutdown hooks run and backing daemons don't + orphan, and then returns normally. + """ + main_task = asyncio.current_task() + assert main_task is not None + loop = asyncio.get_running_loop() + terminated = False + + def terminate() -> None: + nonlocal terminated + terminated = True + main_task.cancel() + + # Signal handlers are not available on Windows loops. + with contextlib.suppress(NotImplementedError): + loop.add_signal_handler(signal.SIGTERM, terminate) + try: + await _serve(env, host, port) + except asyncio.CancelledError: + if not terminated: + raise + main_task.uncancel() + finally: + with contextlib.suppress(NotImplementedError): + loop.remove_signal_handler(signal.SIGTERM) + + +async def _serve(env: Environment, host: str, port: int) -> None: await env.start() server: asyncio.Server | None = None try: @@ -589,17 +619,6 @@ async def serve(env: Environment, host: str = "127.0.0.1", port: int = 0) -> Non await env.stop() -async def _serve_until_terminated(env: Environment, host: str, port: int) -> None: - main_task = asyncio.current_task() - assert main_task is not None - # SIGTERM (the spawn provider's teardown) cancels serving so env.stop() - # runs and backing daemons don't orphan. Not available on Windows loops. - with contextlib.suppress(NotImplementedError): - asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, main_task.cancel) - with contextlib.suppress(asyncio.CancelledError): - await serve(env, host, port) - - def main() -> None: from hud.environment import load_environment @@ -611,9 +630,7 @@ def main() -> None: ) parser.add_argument("--port", type=int, default=0, help="Port to bind (0 = ephemeral).") args = parser.parse_args() - asyncio.run( - _serve_until_terminated(load_environment(args.path, name=args.env), args.host, args.port) - ) + asyncio.run(serve(load_environment(args.path, name=args.env), args.host, args.port)) if __name__ == "__main__": diff --git a/tests/environment/test_loader.py b/tests/environment/test_loader.py index 6d788f2c5..efaafdaad 100644 --- a/tests/environment/test_loader.py +++ b/tests/environment/test_loader.py @@ -284,10 +284,11 @@ def test_a_package_source_wins_over_other_import_roots( [*SERVER, "multi.py", "--env", "env-two"], "env-two", signal.SIGTERM, id="server-env" ), pytest.param([*HUD_SERVE, "env:env"], "from-env-py", signal.SIGINT, id="hud-serve-ctrl-c"), + pytest.param([*HUD_SERVE, "env"], "from-env-py", signal.SIGTERM, id="hud-serve-sigterm"), pytest.param( [*HUD_SERVE, "pkg:make_env", "--arg", "name=demo"], "demo", - signal.SIGINT, + signal.SIGTERM, id="hud-serve-factory", ), ], From 15c7f04598473896715ec70247f68f1f91153823 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 011/248] test: table the direct-control rollouts and read their spans from disk Rewrite the direct-control tests as one table of rollouts: each row records what the sim was commanded, every tool result as the model reads it, and the reward. Add the yaw-only move at pitch +-pi/2, which ported the private orientation test. Telemetry is read from the span file instead of patching queue_span and swapping the TraceRecorder class, and the mid-move end comes from cancelling the session the way a failed agent does. The harness steps() takes an optional schema so robot telemetry, which uses its own span schema, is readable the same way. --- tests/environment/test_file_tracking.py | 3 +- tests/environment/test_robot_control.py | 1151 ++++++++++++++++------- tests/harness/__init__.py | 3 +- tests/harness/spans.py | 9 +- 4 files changed, 803 insertions(+), 363 deletions(-) diff --git a/tests/environment/test_file_tracking.py b/tests/environment/test_file_tracking.py index ec4d195ca..0129fd551 100644 --- a/tests/environment/test_file_tracking.py +++ b/tests/environment/test_file_tracking.py @@ -22,6 +22,7 @@ from .conftest import wire if TYPE_CHECKING: + from collections.abc import Mapping from pathlib import Path XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" @@ -73,7 +74,7 @@ } -def write(root: Path, files: dict[str, str | None]) -> None: +def write(root: Path, files: Mapping[str, str | None]) -> None: for name, content in files.items(): path = root / name if content is None: diff --git a/tests/environment/test_robot_control.py b/tests/environment/test_robot_control.py index 6d866bcb3..99094ffc4 100644 --- a/tests/environment/test_robot_control.py +++ b/tests/environment/test_robot_control.py @@ -1,9 +1,11 @@ -"""Direct control gates: an LLM-style agent drives a robot sim through MCP motion tools. - -Each test is a whole rollout: an in-process bridge serving the ``openpi/0`` wire, -an env publishing it plus :class:`DirectControl`'s ``mcp`` capability, and a -scripted agent calling the tools through the tunneled manifest binding, graded -by the sim. +"""Direct control: a tool-calling agent drives a robot sim through MCP motion tools. + +Each row is a whole rollout. A bridge in this process serves the ``openpi/0`` +wire and its control channel; an env publishes it through ``RobotEndpoint`` with +:class:`DirectControl` attached; a scripted agent calls the motion tool over the +tunneled ``mcp`` binding, and the sim grades the episode. A row asserts what the +sim was commanded, what each tool call answered, the reward, and the trace the +episode streamed to the span file. """ from __future__ import annotations @@ -14,7 +16,6 @@ import io import itertools import math -import time from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime from contextlib import asynccontextmanager from datetime import datetime @@ -23,17 +24,18 @@ import av import numpy as np import pytest +from fastmcp import FastMCP # noqa: TC002 - the override signature is resolved by FastMCP +from inline_snapshot import snapshot from hud.agents.base import Agent from hud.agents.openai.tools.strict_schema import ensure_strict_json_schema from hud.environment import Environment from hud.environment.robot import DirectControl, RobotBridge, RobotEndpoint from hud.eval import LocalRuntime, Task, rollout -from hud.telemetry.robot import TraceRecorder -from hud.telemetry.span import PAYLOAD_ATTRIBUTE, TASK_RUN_ID_ATTRIBUTE +from tests.harness import ROBOT_STEP_SCHEMA, steps if TYPE_CHECKING: - from collections.abc import AsyncIterator + from collections.abc import AsyncIterator, Callable from numpy.typing import NDArray @@ -42,13 +44,10 @@ from hud.types import MCPToolResult -def _contract( - action_type: str, - low: list[float], - high: list[float], - names: list[str] | None = None, +def contract( + action_type: str, low: list[float], high: list[float], names: list[str] | None = None ) -> dict[str, Any]: - """One camera, a state, and an action of *action_type*. Default action is ``[x, grip]``.""" + """One camera, a state as wide as the action, and an action of ``action_type``.""" labels = names or ["x", "grip"] return { "control_rate": 10, @@ -65,8 +64,8 @@ def _contract( } -class _Arm(RobotBridge): - """A 1-D arm plus gripper: succeeds (and terminates) holding ``x >= 0.5`` closed.""" +class Arm(RobotBridge): + """A 1-D arm and gripper that succeeds, ending the episode, at ``x >= 0.5`` closed.""" def __init__(self, contract: dict[str, Any]) -> None: super().__init__() @@ -93,29 +92,62 @@ def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: return data, np.array([self.success]) -class _ScriptedLLM(Agent): - """Stands in for a tool-calling LLM: plays fixed MCP calls, keeps their results.""" +class Pose(RobotBridge): + """Tracks an absolute action: ``lag`` 1 snaps to the command, less trails it.""" + + def __init__(self, contract: dict[str, Any], home: list[float], lag: float = 1.0) -> None: + super().__init__() + self.contract = contract + self.home = np.asarray(home, dtype=np.float64) + self.lag = lag + self.state = self.home.copy() + self.actions: list[NDArray[Any]] = [] + + def reset(self, **task_args: Any) -> str: + self.state = self.home.copy() + self.actions.clear() + return "move" + + def step(self, action: NDArray[Any]) -> None: + target = np.asarray(action[0], dtype=np.float64) + self.actions.append(target.copy()) + self.state = self.state + self.lag * (target - self.state) + + def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: + data = { + "observation/image": np.zeros((1, 8, 8, 3), dtype=np.uint8), + "observation/state": self.state[None].astype(np.float32), + } + return data, np.array([False]) + + +class GraspTool(DirectControl): + """An env serving its own motion tool on the same wire.""" + + def _bind_tools(self, server: FastMCP) -> None: + server.tool(self.move, name="move_eef", description="grasp targets", output_schema=None) + + +class ToolCaller(Agent): + """Stands in for a tool-calling LLM: makes fixed MCP calls and keeps the results.""" def __init__(self, *calls: tuple[str, dict[str, Any]]) -> None: super().__init__() self.calls = calls - self.tools: set[str] = set() self.schemas: dict[str, dict[str, Any]] = {} self.results: list[MCPToolResult] = [] async def __call__(self, run: Run) -> None: client = cast("MCPClient", await run.client.open("control")) - listed = await client.list_tools() - self.tools = {tool.name for tool in listed} - self.schemas = {tool.name: tool.inputSchema for tool in listed} + self.schemas = {tool.name: tool.inputSchema for tool in await client.list_tools()} for name, arguments in self.calls: self.results.append(await client.call_tool(name, arguments)) run.trace.content = "done" @asynccontextmanager -async def _served(sim: RobotBridge, control: DirectControl) -> AsyncIterator[Environment]: - """The docs' custom-bridge env with direct control attached; *sim* runs in this process.""" +async def arm_env(sim: RobotBridge, control: DirectControl) -> AsyncIterator[Environment]: + """The docs' custom-bridge env with direct control attached; ``sim`` runs in this process.""" await sim.start() server = await sim.serve_control() env = Environment("arm") @@ -123,19 +155,19 @@ async def _served(sim: RobotBridge, control: DirectControl) -> AsyncIterator[Env control.attach(endpoint) @env.initialize - async def _up() -> None: + async def connect_sim() -> None: await endpoint.start() - for cap in await endpoint.capabilities(): - env.add_capability(cap) + for capability in await endpoint.capabilities(): + env.add_capability(capability) @env.shutdown - async def _down() -> None: + async def disconnect_sim() -> None: await endpoint.stop() @env.template() async def reach() -> AsyncGenerator[Any, Any]: - ep = await endpoint.reset() - yield {"prompt": ep["prompt"]} + episode = await endpoint.reset() + yield {"prompt": episode["prompt"]} yield await endpoint.result() try: @@ -146,370 +178,773 @@ async def reach() -> AsyncGenerator[Any, Any]: await sim.stop() -def _text(result: MCPToolResult) -> str: - return "\n".join(block.text for block in result.content if block.type == "text") +async def drive(sim: RobotBridge, control: DirectControl, agent: Agent) -> Run: + async with arm_env(sim, control) as env: + return await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) -def _move(tool: str, **values: float) -> tuple[str, dict[str, Any]]: - """One required ``target`` plus any further dimensions in ``others``.""" - items = [{"name": name, "value": value} for name, value in values.items()] - target, *others = items +def move(tool: str, **values: float) -> tuple[str, dict[str, Any]]: + """One required ``target`` and any further dimensions in ``others``.""" + target, *others = [{"name": name, "value": value} for name, value in values.items()] return tool, {"target": target, "others": others, "note": "move toward the goal"} -async def test_move_to_interpolates_absolute_targets_until_the_sim_succeeds() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM( - _move("move_to", x=0.5), - _move("move_to", grip=1.0), - ) - - async with _served(sim, DirectControl(max_step={"grip": 2.0})) as env: - run = await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert agent.tools == {"move_to"} # the ee_abs contract picked the tool; no observe - schema = agent.schemas["move_to"] - assert set(schema["required"]) == {"target", "others", "note"} - strict = ensure_strict_json_schema(copy.deepcopy(schema)) - assert "target" in strict["required"] # survives strict mode, unlike minItems on a list - assert run.reward == 1.0 # graded by the sim, not the tools - # 0.1 of x's range per second at 10 Hz: 0.01 per tick, grip held at its reference. - # The sim matches the command, so the call returns when the target is reached. - reach_x, close = np.array(sim.actions[:50]), sim.actions[50:] - np.testing.assert_allclose(reach_x[:, 0], np.linspace(0.01, 0.5, 50), rtol=1e-6) - np.testing.assert_allclose(reach_x[:, 1], 0.0) - # max_step lets the gripper switch in one tick while x holds its commanded target; - # the sim succeeds on that tick, which ends the episode. - np.testing.assert_allclose(close, [[0.5, 1.0]]) - reached, closed = agent.results - assert [block.type for block in reached.content] == ["text", "text", "image"] - # The trace viewer parses this opening line to map tool calls to ticks. - assert _text(reached).startswith("Played 50 steps (5.0 s).") - assert "observation/state: x=0.5000, grip=0.0000" in _text(reached) - assert "pose: x=0.5000, grip=0.0000" in _text(reached) - assert "The episode has ended" in _text(closed) - - -async def test_move_by_splits_a_displacement_into_steps_within_the_per_step_box() -> None: - sim = _Arm(_contract("ee_del", [-0.1, -1.0], [0.1, 1.0])) - agent = _ScriptedLLM(_move("move_by", x=0.35)) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert agent.tools == {"move_by"} - # Four in-box steps, then two still ticks once the arm has stopped. - np.testing.assert_allclose(sim.actions, [[0.0875, 0.0]] * 4 + [[0.0, 0.0]] * 2) - assert "observation/state: x=0.3500" in _text(agent.results[0]) - - -@pytest.mark.parametrize( - ("call", "error"), - [ - (_move("move_to", z=0.1), "unknown dimension(s) ['z']"), - (_move("move_to", grip=1.0), "unknown dimension(s) ['grip']; valid: x"), - (_move("move_to", x=1.5), "x=1.5 is outside [0, 1]"), - ( - ("move_to", {"target": {"name": "x", "value": 0.1}, "others": [], "note": " "}), - "note must say", +def rpy(roll: float, pitch: float, yaw: float) -> list[float]: + """The xyzw quaternion of rotation Rz(yaw) Ry(pitch) Rx(roll).""" + + def product(a: list[float], b: list[float]) -> list[float]: + ax, ay, az, aw = a + bx, by, bz, bw = b + return [ + aw * bx + ax * bw + ay * bz - az * by, + aw * by - ax * bz + ay * bw + az * bx, + aw * bz + ax * by - ay * bx + az * bw, + aw * bw - ax * bx - ay * by - az * bz, + ] + + def axis(index: int, angle: float) -> list[float]: + quaternion = [0.0, 0.0, 0.0, math.cos(angle / 2)] + quaternion[index] = math.sin(angle / 2) + return quaternion + + return product(product(axis(2, yaw), axis(1, pitch)), axis(0, roll)) + + +XYZW = [f"tcp_quat.{part}" for part in ("qx", "qy", "qz", "qw")] +WXYZ = [f"tcp_quat.{part}" for part in ("qw", "qx", "qy", "qz")] +AXIS_ANGLE = [f"target_eef_axis_angle.{part}" for part in ("rx", "ry", "rz")] +EULER = ["ee_pos.x", "ee_pos.y", "ee_pos.z", "ee_euler.x", "ee_euler.y", "ee_euler.z", "gripper"] + + +def unit_quaternions(actions: NDArray[Any]) -> bool: + return bool(np.allclose(np.linalg.norm(actions, axis=1), 1.0, atol=1e-5)) + + +def even_rotation(actions: NDArray[Any]) -> list[float]: + """Distinct per-tick changes of the rotation angle: one value for a constant rate.""" + return sorted({round(float(gap), 2) for gap in np.diff(np.linalg.norm(actions, axis=1))}) + + +def tick_lengths(actions: NDArray[Any]) -> list[float]: + """Distinct distances commanded per tick: a constant pace has one value per phase.""" + return sorted({round(float(np.linalg.norm(gap)), 4) for gap in np.diff(actions, axis=0)}) + + +def quaternion_alignment(target: list[float]) -> Callable[[NDArray[Any]], float]: + """|cos| of the half-angle between the final command and ``target`` (1.0 is equal).""" + return lambda actions: round(abs(float(np.dot(actions[-1], target))), 6) + + +ARM = contract("ee_abs", [0.0, -1.0], [1.0, 1.0]) + +ROWS = [ + pytest.param( + Arm(ARM), + DirectControl(max_step={"grip": 2.0}), + [move("move_to", x=0.5), move("move_to", grip=1.0)], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 50 steps (5.0 s)."], + ["observation/state: x=0.5000", "grip=0.0000"], + ["pose: x=0.5000", "grip=0.0000"], + ["commanded: x=0.5000", "grip=0.0000"], + ["off: x=0.0000", "grip=0.0000"], + ["camera observation/image:"], + ], + "images": 1, + }, + { + "lines": [ + ["Played 1 steps (0.1 s)."], + ["observation/state: x=0.5000", "grip=1.0000"], + ["pose: x=0.5000", "grip=1.0000"], + ["commanded: x=0.5000", "grip=1.0000"], + ["off: x=0.0000", "grip=0.0000"], + ["The episode has ended; stop calling tools."], + ["camera observation/image:"], + ], + "images": 1, + }, + ], + "ticks": 51, + "first": [0.01, 0.0], + "last": [0.5, 1.0], + "measure": [0.01, 1.0], + "reward": 1.0, + } ), - ], -) -async def test_an_invalid_move_is_a_correctable_error_that_leaves_the_sim_still( - call: tuple[str, dict[str, Any]], error: str -) -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(call) - - async with _served(sim, DirectControl(dims=["x"], speed=0.05)) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - (result,) = agent.results - assert result.isError - assert error in _text(result) - assert sim.actions == [] - - -class _GraspTool(DirectControl): - """Stands in for an env that serves its own motion tool on this wire.""" - - def _bind_tools(self, server) -> None: - server.tool(self.move, name="move_eef", description="grasp targets", output_schema=None) - - -async def test_an_env_can_replace_the_contract_motion_tool() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_eef", x=0.5)) - - async with _served(sim, _GraspTool(max_step={"grip": 2.0})) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - assert agent.tools == {"move_eef"} - assert sim.actions # the replacement tool still plays through the shared wire - - -class _Pose(RobotBridge): - """Tracks an absolute action. ``lag`` of 1 snaps to the command; less lags behind.""" - - def __init__(self, contract: dict[str, Any], home: list[float], lag: float = 1.0) -> None: - super().__init__() - self.contract = contract - self.home = np.asarray(home, dtype=np.float64) - self.lag = lag - self.state = self.home.copy() - self.actions: list[NDArray[Any]] = [] - - def reset(self, **task_args: Any) -> str: - self.state = self.home.copy() - self.actions.clear() - return "move" - - def step(self, action: NDArray[Any]) -> None: - target = np.asarray(action[0], dtype=np.float64) - self.actions.append(target.copy()) - self.state = self.state + self.lag * (target - self.state) - self.success = False - - def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: - data = { - "observation/image": np.zeros((1, 8, 8, 3), dtype=np.uint8), - "observation/state": self.state[None].astype(np.float32), - } - return data, np.array([self.success]) - - -async def test_an_empty_target_list_is_rejected_and_does_not_move() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(("move_to", {"targets": [], "note": "move toward the goal"})) + id="move-to-interpolates-until-the-sim-succeeds", + ), + pytest.param( + Arm(contract("ee_del", [-0.1, -1.0], [0.1, 1.0])), + DirectControl(), + [move("move_by", x=0.35)], + tick_lengths, + snapshot( + { + "tools": ["move_by"], + "results": [ + { + "lines": [ + ["Played 6 steps (0.6 s)."], + ["observation/state: x=0.3500", "grip=0.0000"], + ["pose: x=0.3500", "grip=0.0000"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 6, + "first": [0.0875, 0.0], + "last": [0.0, 0.0], + "measure": [0.0, 0.0875], + "reward": 0.0, + } + ), + id="move-by-splits-a-displacement-into-in-box-steps", + ), + pytest.param( + Arm(ARM), + DirectControl(dims=["x"], speed=0.05), + [move("move_to", z=0.1)], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [{"error": "unknown dimension(s) ['z']; valid: x"}], + "ticks": 0, + "first": None, + "last": None, + "measure": None, + "reward": 0.0, + } + ), + id="unknown-dimension-is-a-correctable-error", + ), + pytest.param( + Arm(ARM), + DirectControl(dims=["x"], speed=0.05), + [move("move_to", grip=1.0)], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [{"error": "unknown dimension(s) ['grip']; valid: x"}], + "ticks": 0, + "first": None, + "last": None, + "measure": None, + "reward": 0.0, + } + ), + id="dimension-outside-dims-is-a-correctable-error", + ), + pytest.param( + Arm(ARM), + DirectControl(dims=["x"], speed=0.05), + [move("move_to", x=1.5)], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [{"error": "x=1.5 is outside [0, 1]"}], + "ticks": 0, + "first": None, + "last": None, + "measure": None, + "reward": 0.0, + } + ), + id="target-out-of-range-is-a-correctable-error", + ), + pytest.param( + Arm(ARM), + DirectControl(dims=["x"], speed=0.05), + [("move_to", {"target": {"name": "x", "value": 0.1}, "others": [], "note": " "})], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [{"error": "note must say what you see and why you chose this motion"}], + "ticks": 0, + "first": None, + "last": None, + "measure": None, + "reward": 0.0, + } + ), + id="blank-note-is-a-correctable-error", + ), + pytest.param( + Arm(ARM), + DirectControl(), + [("move_to", {"targets": [], "note": "move toward the goal"})], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [{"error": "3 validation errors for call[move]"}], + "ticks": 0, + "first": None, + "last": None, + "measure": None, + "reward": 0.0, + } + ), + id="missing-target-is-rejected", + ), + pytest.param( + Arm(ARM), + GraspTool(max_step={"grip": 2.0}), + [move("move_eef", x=0.5)], + tick_lengths, + snapshot( + { + "tools": ["move_eef"], + "results": [ + { + "lines": [ + ["Played 50 steps (5.0 s)."], + ["observation/state: x=0.5000", "grip=0.0000"], + ["pose: x=0.5000", "grip=0.0000"], + ["commanded: x=0.5000", "grip=0.0000"], + ["off: x=0.0000", "grip=0.0000"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 50, + "first": [0.01, 0.0], + "last": [0.5, 0.0], + "measure": [0.01], + "reward": 0.0, + } + ), + id="env-replaces-the-contract-motion-tool", + ), + pytest.param( + Arm(ARM), + DirectControl(speed=0.05, max_step={"grip": 2.0}), + [move("move_to", x=0.6)], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 120 steps (12.0 s)."], + ["observation/state: x=0.6000", "grip=0.0000"], + ["pose: x=0.6000", "grip=0.0000"], + ["commanded: x=0.6000", "grip=0.0000"], + ["off: x=0.0000", "grip=0.0000"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 120, + "first": [0.005, 0.0], + "last": [0.6, 0.0], + "measure": [0.005], + "reward": 0.0, + } + ), + id="a-twelve-second-move-plays-until-arrival", + ), + pytest.param( + Arm(ARM), + DirectControl(dims=["x"], speed=0.05, timeout=0.3), + [move("move_to", x=1.0)], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 3 steps (0.3 s)."], + ["Stopped at the 0.3 s safety timeout before the motion finished."], + ["observation/state: x=0.0150", "grip=0.0000"], + ["pose: x=0.0150"], + ["commanded: x=1.0000"], + ["off: x=-0.9850"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 3, + "first": [0.005, 0.0], + "last": [0.015, 0.0], + "measure": [0.005], + "reward": 0.0, + } + ), + id="safety-timeout-returns-the-pose", + ), + pytest.param( + Pose(ARM, [0.0, 0.0], lag=0.25), + DirectControl(max_step={"grip": 2.0}), + [move("move_to", x=0.5)], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 54 steps (5.4 s)."], + ["observation/state: x=0.4905", "grip=0.0000"], + ["pose: x=0.4905", "grip=0.0000"], + ["commanded: x=0.5000", "grip=0.0000"], + ["off: x=-0.0095", "grip=0.0000"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 54, + "first": [0.01, 0.0], + "last": [0.5, 0.0], + "measure": [0.0, 0.01], + "reward": 0.0, + } + ), + id="a-lagging-arm-is-held-until-it-arrives", + ), + pytest.param( + Pose(contract("ee_abs", [-1.0] * 4, [1.0] * 4, XYZW), rpy(0, 0, 0)), + DirectControl(), + [move("move_to", **{"tcp.yaw": math.pi / 2})], + unit_quaternions, + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 25 steps (2.5 s)."], + [ + "observation/state: tcp_quat.qx=0.0000", + "tcp_quat.qy=0.0000", + "tcp_quat.qz=0.7071", + "tcp_quat.qw=0.7071", + ], + ["pose: tcp.roll=0.0000", "tcp.pitch=-0.0000", "tcp.yaw=1.5708"], + ["commanded: tcp.roll=0.0000", "tcp.pitch=-0.0000", "tcp.yaw=1.5708"], + ["off: orientation=0.0000 rad"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 25, + "first": [0.0, 0.0, 0.0314, 0.9995], + "last": [0.0, 0.0, 0.7071, 0.7071], + "measure": True, + "reward": 0.0, + } + ), + id="yaw-slerps-an-xyzw-quaternion", + ), + pytest.param( + Pose(contract("ee_abs", [-1.0] * 4, [1.0] * 4, WXYZ), [1.0, 0.0, 0.0, 0.0]), + DirectControl(), + [move("move_to", **{"tcp.roll": math.pi / 2})], + unit_quaternions, + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 25 steps (2.5 s)."], + [ + "observation/state: tcp_quat.qw=0.7071", + "tcp_quat.qx=0.7071", + "tcp_quat.qy=0.0000", + "tcp_quat.qz=0.0000", + ], + ["pose: tcp.roll=1.5708", "tcp.pitch=-0.0000", "tcp.yaw=0.0000"], + ["commanded: tcp.roll=1.5708", "tcp.pitch=-0.0000", "tcp.yaw=0.0000"], + ["off: orientation=0.0000 rad"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 25, + "first": [0.9995, 0.0314, 0.0, 0.0], + "last": [0.7071, 0.7071, 0.0, 0.0], + "measure": True, + "reward": 0.0, + } + ), + id="roll-writes-a-wxyz-quaternion", + ), + pytest.param( + Pose(contract("ee_abs", [-1.0] * 4, [1.0] * 4, XYZW), rpy(0.4, math.pi / 2, 0)), + DirectControl(), + [move("move_to", **{"tcp.yaw": -0.25})], + quaternion_alignment(rpy(0.4, math.pi / 2, -0.25)), + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 4 steps (0.4 s)."], + [ + "observation/state: tcp_quat.qx=0.2258", + "tcp_quat.qy=0.6701", + "tcp_quat.qz=-0.2258", + "tcp_quat.qw=0.6701", + ], + ["pose: tcp.roll=0.6500", "tcp.pitch=1.5708", "tcp.yaw=0.0000"], + ["commanded: tcp.roll=0.6500", "tcp.pitch=1.5708", "tcp.yaw=0.0000"], + ["off: orientation=0.0000 rad"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 4, + "first": [0.1621, 0.6883, -0.1621, 0.6883], + "last": [0.2258, 0.6701, -0.2258, 0.6701], + "measure": 1.0, + "reward": 0.0, + } + ), + id="yaw-at-pitch-up-keeps-the-wrist-roll", + ), + pytest.param( + Pose(contract("ee_abs", [-1.0] * 4, [1.0] * 4, XYZW), rpy(0.4, -math.pi / 2, 0)), + DirectControl(), + [move("move_to", **{"tcp.yaw": -0.25})], + quaternion_alignment(rpy(0.4, -math.pi / 2, -0.25)), + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 4 steps (0.4 s)."], + [ + "observation/state: tcp_quat.qx=0.0530", + "tcp_quat.qy=-0.7051", + "tcp_quat.qz=0.0530", + "tcp_quat.qw=0.7051", + ], + ["pose: tcp.roll=0.1500", "tcp.pitch=-1.5708", "tcp.yaw=0.0000"], + ["commanded: tcp.roll=0.1500", "tcp.pitch=-1.5708", "tcp.yaw=0.0000"], + ["off: orientation=0.0000 rad"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 4, + "first": [0.1188, -0.6971, 0.1188, 0.6971], + "last": [0.053, -0.7051, 0.053, 0.7051], + "measure": 1.0, + "reward": 0.0, + } + ), + id="yaw-at-pitch-down-keeps-the-wrist-roll", + ), + pytest.param( + Pose( + contract("ee_abs", [0.0, -1, -1, -1, -1], [1, 1, 1, 1, 1], ["x", *XYZW]), + [0.0, 0.0, 0.2, 0.0, 0.9], + ), + DirectControl(), + [move("move_to", x=0.2)], + lambda actions: sorted({tuple(row) for row in np.round(actions[:, 1:], 6).tolist()}), + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 20 steps (2.0 s)."], + [ + "observation/state: x=0.2000", + "tcp_quat.qx=0.0000", + "tcp_quat.qy=0.2000", + "tcp_quat.qz=0.0000", + "tcp_quat.qw=0.9000", + ], + [ + "pose: x=0.2000", + "tcp.roll=0.0000", + "tcp.pitch=0.4373", + "tcp.yaw=0.0000", + ], + [ + "commanded: x=0.2000", + "tcp.roll=0.0000", + "tcp.pitch=0.4373", + "tcp.yaw=0.0000", + ], + ["off: x=0.0000", "orientation=0.0000 rad"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 20, + "first": [0.01, 0.0, 0.2, 0.0, 0.9], + "last": [0.2, 0.0, 0.2, 0.0, 0.9], + "measure": [(0.0, 0.2, 0.0, 0.9)], + "reward": 0.0, + } + ), + id="an-unnamed-quaternion-is-held-unchanged", + ), + pytest.param( + Pose(contract("ee_abs", [-np.pi] * 3, [np.pi] * 3, AXIS_ANGLE), [0.0, 0.0, 0.0]), + DirectControl(), + [move("move_to", **{"target_eef.roll": math.pi / 2, "target_eef.yaw": math.pi / 2})], + even_rotation, + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 34 steps (3.4 s)."], + [ + "observation/state: target_eef_axis_angle.rx=1.2092", + "target_eef_axis_angle.ry=1.2092", + "target_eef_axis_angle.rz=1.2092", + ], + [ + "pose: target_eef.roll=1.5708", + "target_eef.pitch=0.0000", + "target_eef.yaw=1.5708", + ], + [ + "commanded: target_eef.roll=1.5708", + "target_eef.pitch=-0.0000", + "target_eef.yaw=1.5708", + ], + ["off: orientation=0.0000 rad"], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 34, + "first": [0.0356, 0.0356, 0.0356], + "last": [1.2092, 1.2092, 1.2092], + "measure": [0.06], + "reward": 0.0, + } + ), + id="euler-commands-slerp-onto-axis-angle", + ), + pytest.param( + Pose( + contract( + "ee_abs", + [-1.0, -1.0, 0.0, -np.pi, -np.pi, -np.pi, 0.0], + [1.0, 1.0, 2.0, np.pi, np.pi, np.pi, 0.04], + EULER, + ), + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.04], + ), + DirectControl(), + [move("move_to", **{"ee_pos.z": 1.2, "ee.yaw": 0.4})], + tick_lengths, + snapshot( + { + "tools": ["move_to"], + "results": [ + { + "lines": [ + ["Played 10 steps (1.0 s)."], + [ + "observation/state: ee_pos.x=0.0000", + "ee_pos.y=0.0000", + "ee_pos.z=1.2000", + "ee_euler.x=0.0000", + "ee_euler.y=0.0000", + "ee_euler.z=0.4000", + "gripper=0.0400", + ], + [ + "pose: ee_pos.x=0.0000", + "ee_pos.y=0.0000", + "ee_pos.z=1.2000", + "ee.roll=0.0000", + "ee.pitch=-0.0000", + "ee.yaw=0.4000", + "gripper=0.0400", + ], + [ + "commanded: ee_pos.x=0.0000", + "ee_pos.y=0.0000", + "ee_pos.z=1.2000", + "ee.roll=0.0000", + "ee.pitch=-0.0000", + "ee.yaw=0.4000", + "gripper=0.0400", + ], + [ + "off: ee_pos.x=0.0000", + "ee_pos.y=0.0000", + "ee_pos.z=0.0000", + "orientation=0.0000 rad", + "gripper=0.0000", + ], + ["camera observation/image:"], + ], + "images": 1, + } + ], + "ticks": 10, + "first": [0.0, 0.0, 1.02, 0.0, 0.0, 0.04, 0.04], + "last": [0.0, 0.0, 1.2, 0.0, 0.0, 0.4, 0.04], + "measure": [0.0447], + "reward": 0.0, + } + ), + id="an-euler-contract-is-addressed-as-roll-pitch-yaw", + ), +] - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - (result,) = agent.results - assert result.isError - assert sim.actions == [] +def answered(result: MCPToolResult) -> dict[str, Any]: + """A tool result as the model reads it: an error's first line, or the reply's lines.""" + texts = [block.text for block in result.content if block.type == "text"] + if result.isError: + return {"error": texts[0].splitlines()[0]} + return { + "lines": [line.split(", ") for text in texts for line in text.splitlines()], + "images": sum(block.type == "image" for block in result.content), + } -async def test_a_move_longer_than_ten_seconds_plays_until_the_arm_arrives() -> None: - """VLABench was rejected at 10.1 s. The cap is a 60 s safety timeout, not that reject.""" - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to", x=0.6)) +def rounded(action: NDArray[Any]) -> list[float]: + return (np.round(action, 4) + 0.0).tolist() - async with _served(sim, DirectControl(speed=0.05, max_step={"grip": 2.0})) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - (result,) = agent.results - assert not result.isError - # 0.6 / 0.05 s = 12 s at 10 Hz, which used to exceed the 10 s cap. - assert len(sim.actions) >= 120 - assert sim.state[0] == pytest.approx(0.6) - assert "safety timeout" not in _text(result) +@pytest.mark.parametrize(("sim", "control", "calls", "measure", "expected"), ROWS) +async def test_a_motion_tool_call_plays_out_on_the_sim( + sim: Arm | Pose, + control: DirectControl, + calls: list[tuple[str, dict[str, Any]]], + measure: Callable[[NDArray[Any]], Any], + expected: Any, +) -> None: + agent = ToolCaller(*calls) + run = await drive(sim, control, agent) -async def test_the_safety_timeout_returns_the_pose_instead_of_rejecting_the_call() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to", x=1.0)) + actions = np.array(sim.actions, dtype=np.float64) + assert { + "tools": sorted(agent.schemas), + "results": [answered(result) for result in agent.results], + "ticks": len(actions), + "first": rounded(actions[0]) if len(actions) else None, + "last": rounded(actions[-1]) if len(actions) else None, + "measure": measure(actions) if len(actions) else None, + "reward": run.reward, + } == expected - async with _served(sim, DirectControl(dims=["x"], speed=0.05, timeout=0.3)) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - (result,) = agent.results - assert not result.isError - assert len(sim.actions) == 3 # 0.3 s at 10 Hz - assert sim.state[0] < 0.05 # the prefix of the move, not a jump to the goal - assert "safety timeout" in _text(result) - assert "pose:" in _text(result) - assert "commanded:" in _text(result) +async def test_the_motion_tool_schema_keeps_its_target_required_under_strict_mode() -> None: + agent = ToolCaller() + await drive(Arm(ARM), DirectControl(), agent) -async def test_a_lagging_arm_keeps_stepping_until_it_reaches_the_target() -> None: - sim = _Pose(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0]), [0.0, 0.0], lag=0.25) - agent = _ScriptedLLM(_move("move_to", x=0.5)) + schema = agent.schemas["move_to"] + assert sorted(schema["required"]) == ["note", "others", "target"] + assert "target" in ensure_strict_json_schema(copy.deepcopy(schema))["required"] - async with _served(sim, DirectControl(max_step={"grip": 2.0})) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - assert len(sim.actions) > 50 # interpolation, then a hold while the arm catches up - assert sim.state[0] == pytest.approx(0.5, abs=0.01) - assert "pose:" in _text(agent.results[0]) - assert "off:" in _text(agent.results[0]) +async def test_a_contract_without_a_motion_action_type_is_refused_at_start() -> None: + sim = Arm(contract("joint_vel", [0.0, -1.0], [1.0, 1.0])) + with pytest.raises(ValueError, match="direct control needs an action type in"): + async with arm_env(sim, DirectControl()) as env: + await env.start() -def _quat_names(order: str) -> list[str]: - parts = ("qx", "qy", "qz", "qw") if order == "xyzw" else ("qw", "qx", "qy", "qz") - return [f"tcp_quat.{part}" for part in parts] +def frames_in(segments: list[dict[str, Any]]) -> int: + mp4 = b"".join(base64.b64decode(segment["segment"]["data"]) for segment in segments) + with av.open(io.BytesIO(mp4), mode="r") as container: + return sum(1 for _ in container.decode(video=0)) -async def test_a_yaw_slerps_an_xyzw_quaternion_instead_of_lerping_components() -> None: - names = _quat_names("xyzw") - sim = _Pose(_contract("ee_abs", [-1.0] * 4, [1.0] * 4, names), [0.0, 0.0, 0.0, 1.0]) - agent = _ScriptedLLM(_move("move_to", **{"tcp.yaw": math.pi / 2})) - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) +@pytest.mark.parametrize(("control_rate", "tick_seconds"), [(10, 0.1), (0.4, 1.0)]) +async def test_every_played_tick_streams_to_the_trace_as_state_and_video( + control_rate: float, tick_seconds: float +) -> None: + sim = Arm(ARM | {"control_rate": control_rate}) - actions = np.array(sim.actions) - norms = np.linalg.norm(actions, axis=1) - np.testing.assert_allclose(norms, 1.0, atol=1e-5) - np.testing.assert_allclose( - actions[-1], [0.0, 0.0, math.sin(math.pi / 4), math.cos(math.pi / 4)], atol=1e-5 + run = await drive( + sim, DirectControl(), ToolCaller(move("move_to", x=0.3), move("move_to", x=0.5)) ) - # Component lerp of the endpoints is not unit length; slerp stays on the arc. - assert len(actions) > 2 + payloads = steps(run.trace_id, schema=ROBOT_STEP_SCHEMA) + observations = [p for p in payloads if p.get("source") == "observation"] + segments = [p for p in payloads if p.get("source") == "video_segment"] + starts = [datetime.fromisoformat(obs["started_at"]) for obs in observations] + # The opening scene, then one observation and one frame per tick of both moves. + assert [obs["tick"] for obs in observations] == list(range(1 + len(sim.actions))) + assert {segment["camera"] for segment in segments} == {"observation/image"} + assert frames_in(segments) == 1 + len(sim.actions) + assert observations[-1]["state"] + assert [(b - a).total_seconds() for a, b in itertools.pairwise(starts)] == pytest.approx( + [tick_seconds] * len(sim.actions) + ) -async def test_a_roll_writes_a_wxyz_quaternion() -> None: - names = _quat_names("wxyz") - sim = _Pose(_contract("ee_abs", [-1.0] * 4, [1.0] * 4, names), [1.0, 0.0, 0.0, 0.0]) - agent = _ScriptedLLM(_move("move_to", **{"tcp.roll": math.pi / 2})) - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) +class WatchedArm(Arm): + """An arm that announces its third tick.""" - np.testing.assert_allclose( - sim.actions[-1], - [np.cos(np.pi / 4), np.sin(np.pi / 4), 0.0, 0.0], - atol=1e-5, - ) + def __init__(self, contract: dict[str, Any]) -> None: + super().__init__(contract) + self.moving = asyncio.Event() + def step(self, action: NDArray[Any]) -> None: + super().step(action) + if len(self.actions) >= 3: + self.moving.set() -async def test_an_unnamed_quaternion_is_copied_unchanged() -> None: - names = ["x", "tcp_quat.qx", "tcp_quat.qy", "tcp_quat.qz", "tcp_quat.qw"] - home = [0.0, 0.0, 0.2, 0.0, 0.9] # not a unit quaternion - sim = _Pose(_contract("ee_abs", [0.0, -1, -1, -1, -1], [1, 1, 1, 1, 1], names), home) - agent = _ScriptedLLM(_move("move_to", x=0.2)) - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) +class AbandonsMidMove(Agent): + """Starts a long move, then fails once the sim has stepped a few ticks.""" - actions = np.array(sim.actions) - assert len(actions) > 1 - held = np.broadcast_to(actions[0, 1:], actions[:, 1:].shape) - np.testing.assert_array_equal(actions[:, 1:], held) - assert abs(np.linalg.norm(actions[0, 1:]) - 1.0) > 1e-3 + def __init__(self, sim: WatchedArm) -> None: + super().__init__() + self.sim = sim + async def __call__(self, run: Run) -> None: + client = cast("MCPClient", await run.client.open("control")) + moving = asyncio.create_task(client.call_tool(*move("move_to", x=0.5))) + await asyncio.wait_for(self.sim.moving.wait(), timeout=30) + moving.cancel() + raise RuntimeError("the agent gave up") -async def test_euler_commands_slerp_onto_axis_angle() -> None: - names = [ - "target_eef_axis_angle.rx", - "target_eef_axis_angle.ry", - "target_eef_axis_angle.rz", - ] - sim = _Pose(_contract("ee_abs", [-np.pi] * 3, [np.pi] * 3, names), [0.0, 0.0, 0.0]) - agent = _ScriptedLLM( - _move("move_to", **{"target_eef.roll": math.pi / 2, "target_eef.yaw": math.pi / 2}) - ) - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - # Intrinsic XYZ of roll=yaw=pi/2 is the quaternion (0.5, 0.5, 0.5, 0.5), angle 2pi/3. - component = (2.0 * np.pi / 3.0) / np.sqrt(3.0) - np.testing.assert_allclose(sim.actions[-1], [component, component, component], atol=1e-4) - angles = np.linalg.norm(sim.actions, axis=1) - gaps = np.diff(angles) - assert gaps.min() > 0 - np.testing.assert_allclose(gaps, gaps[0], rtol=0.05) - - -async def test_an_euler_contract_is_addressed_as_roll_pitch_yaw() -> None: - names = [ - "ee_pos.x", - "ee_pos.y", - "ee_pos.z", - "ee_euler.x", - "ee_euler.y", - "ee_euler.z", - "gripper", - ] - low = [-1.0, -1.0, 0.0, -np.pi, -np.pi, -np.pi, 0.0] - high = [1.0, 1.0, 2.0, np.pi, np.pi, np.pi, 0.04] - sim = _Pose(_contract("ee_abs", low, high, names), [0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.04]) - agent = _ScriptedLLM(_move("move_to", **{"ee_pos.z": 1.2, "ee.yaw": 0.4})) - - async with _served(sim, DirectControl()) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - final = sim.actions[-1] - assert final[2] == pytest.approx(1.2, abs=1e-4) - assert final[5] == pytest.approx(0.4, abs=1e-4) - np.testing.assert_allclose(final[3:5], 0.0, atol=1e-4) - assert final[6] == pytest.approx(0.04) - assert "ee.yaw" not in sim.contract["features"]["action"]["names"] - - -async def test_a_contract_without_a_motion_type_is_refused_at_start() -> None: - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - sim.contract["features"]["action"]["type"] = "joint_vel" - - with pytest.raises(ValueError, match="direct control needs an action type"): - async with _served(sim, DirectControl()) as env: - await env.start() +async def test_cancelling_the_session_mid_move_ends_the_recording() -> None: + sim = WatchedArm(ARM) + run = await drive(sim, DirectControl(speed=0.001), AbandonsMidMove(sim)) -@pytest.mark.parametrize(("control_rate", "tick_seconds"), [(10, 0.1), (0.4, 1.0)]) -async def test_every_played_tick_streams_to_the_trace_as_state_and_one_video_per_camera( - monkeypatch: pytest.MonkeyPatch, control_rate: float, tick_seconds: float -) -> None: - spans: list[dict[str, Any]] = [] - monkeypatch.setattr("hud.types.queue_span", spans.append) - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - sim.contract["control_rate"] = control_rate - agent = _ScriptedLLM(_move("move_to", x=0.3), _move("move_to", x=0.5)) - - async with _served(sim, DirectControl()) as env: - run = await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - - payloads = [ - span["attributes"][PAYLOAD_ATTRIBUTE] - for span in spans - if span["attributes"][TASK_RUN_ID_ATTRIBUTE] == run.trace_id - ] + payloads = steps(run.trace_id, schema=ROBOT_STEP_SCHEMA) + observations = [p["tick"] for p in payloads if p.get("source") == "observation"] segments = [p for p in payloads if p.get("source") == "video_segment"] - assert {segment["camera"] for segment in segments} == {"observation/image"} - mp4 = b"".join(base64.b64decode(segment["segment"]["data"]) for segment in segments) - with av.open(io.BytesIO(mp4), mode="r") as container: - frames = sum(1 for _ in container.decode(video=0)) - assert frames == 1 + len(sim.actions) # the opening scene, then every tick of both moves - observations = [p for p in payloads if p.get("source") == "observation"] - assert [obs["tick"] for obs in observations] == list(range(frames)) - assert observations[-1]["state"] - # The viewer's clock is the stamps' span: the video's length, not the calls' latency. - starts = [datetime.fromisoformat(obs["started_at"]) for obs in observations] - assert [(b - a).total_seconds() for a, b in itertools.pairwise(starts)] == pytest.approx( - [tick_seconds] * (frames - 1) - ) - - -async def test_ending_the_episode_mid_move_closes_the_recording_after_the_tick_in_flight( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr("hud.types.queue_span", lambda _span: None) - events: list[str] = [] - ends: list[asyncio.Task[None]] = [] - control = DirectControl() - - class _Recorder(TraceRecorder): - def record_observation(self, data: dict[str, Any], *, tick: int) -> None: - if tick == 2: # mid-move, the episode is ended (as a cancel would) - loop.call_soon_threadsafe( - lambda: ends.append(loop.create_task(control.end_episode())) - ) - time.sleep(0.2) - super().record_observation(data, tick=tick) - events.append("record") - - def close(self) -> None: - events.append("close") - super().close() - - monkeypatch.setattr("hud.environment.robot.control.TraceRecorder", _Recorder) - loop = asyncio.get_running_loop() - sim = _Arm(_contract("ee_abs", [0.0, -1.0], [1.0, 1.0])) - agent = _ScriptedLLM(_move("move_to", x=0.5)) - - async with _served(sim, control) as env: - await rollout(Task(env="arm", id="reach"), agent, runtime=LocalRuntime(env)) - await asyncio.gather(*ends) - - assert events == ["record"] * 3 + ["close"] # ticks 0-2, then nothing after the close + assert run.trace.status == "error" + assert observations == list(range(len(observations))) + assert 3 <= len(observations) < 1 + len(sim.actions) + assert frames_in(segments) == len(observations) diff --git a/tests/harness/__init__.py b/tests/harness/__init__.py index dbb9a67d4..4858ef700 100644 --- a/tests/harness/__init__.py +++ b/tests/harness/__init__.py @@ -13,9 +13,10 @@ from .rfb import FakeScreen, KeyEvent, PointerEvent, fake_screen from .scenario import RecordingProvider, ScriptedAgent, served, task_row from .services import FakeServices, Reply, Request -from .spans import spans, steps +from .spans import ROBOT_STEP_SCHEMA, spans, steps __all__ = [ + "ROBOT_STEP_SCHEMA", "FakeDocker", "FakeScreen", "FakeServices", diff --git a/tests/harness/spans.py b/tests/harness/spans.py index 47633fd52..bcd28af02 100644 --- a/tests/harness/spans.py +++ b/tests/harness/spans.py @@ -10,6 +10,7 @@ from hud.settings import settings STEP_SCHEMA = "hud.step.v1" +ROBOT_STEP_SCHEMA = "hud.robot.step.v1" def spans(trace_id: str | None, span_dir: Path | None = None) -> list[dict[str, Any]]: @@ -29,10 +30,12 @@ def spans(trace_id: str | None, span_dir: Path | None = None) -> list[dict[str, return sorted(records, key=lambda span: span.get("start_time", "")) -def steps(trace_id: str | None, span_dir: Path | None = None) -> list[dict[str, Any]]: - """The step payloads of a run, in order.""" +def steps( + trace_id: str | None, span_dir: Path | None = None, *, schema: str = STEP_SCHEMA +) -> list[dict[str, Any]]: + """The step payloads of a run, in order; ``schema=ROBOT_STEP_SCHEMA`` for robot telemetry.""" return [ span["attributes"]["hud.payload"] for span in spans(trace_id, span_dir) - if span.get("attributes", {}).get("hud.schema") == STEP_SCHEMA + if span.get("attributes", {}).get("hud.schema") == schema ] From 56970d6ca2520bada2b9902b6f1ebf3c38a5ea3f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 012/248] test: claim, step and free robot slots through the public control plane The slot tests mocked RobotEndpoint._call, seeded _claims and the bridge registry, and drove the private tick loop with fake sockets. Serve a stub bridge instead and claim slots over its control channel, through an env template driving RobotEndpoint, and from RobotClient connections. The rows cover a full batch, mismatched batch args, cancel/bye/grade/drop each freeing a slot exactly once (with a failing release retried, and drained on stop), a silent slot timed out beside an active peer, an undialed claim holding the batch, and no stepping after a terminal observation. --- tests/environment/test_robot_slots.py | 347 ++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 tests/environment/test_robot_slots.py diff --git a/tests/environment/test_robot_slots.py b/tests/environment/test_robot_slots.py new file mode 100644 index 000000000..4bc3ea7e2 --- /dev/null +++ b/tests/environment/test_robot_slots.py @@ -0,0 +1,347 @@ +"""The robot control plane: how sessions claim, step and free a sim's slots. + +A bridge in this process serves its JSON-RPC control channel and its ``openpi/0`` +WebSocket. Scenarios claim slots the way an env does (``reset`` / ``result`` on +the control channel, or a template driving ``RobotEndpoint``), step them the way +an agent does (``RobotClient``), and assert what the sim saw: its steps, its +``result`` calls, and the errors each peer got. +""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest + +from hud.capabilities import Capability +from hud.capabilities.robot import RobotClient +from hud.environment import Environment +from hud.environment.robot import RobotBridge, RobotEndpoint +from hud.eval import LocalRuntime, Task + +from .conftest import Wire, wire + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from numpy.typing import NDArray + + +class Sim(RobotBridge): + """``num_envs`` lockstep slots of a one-dimensional action. + + Records every batched step and every ``result`` call; the first ``failures`` + calls to ``result`` raise, and every slot terminates after ``terminal_at`` steps. + """ + + def __init__( + self, + num_envs: int = 1, + *, + failures: int = 0, + terminal_at: int | None = None, + step_timeout: float = 30.0, + ) -> None: + super().__init__() + self.num_envs = num_envs + self.failures = failures + self.terminal_at = terminal_at + self.step_timeout = step_timeout + self.contract = { + "control_rate": 10, + "features": {"action": {"role": "action", "names": ["a"]}}, + } + self.steps: list[list[float]] = [] + self.results = 0 + + def reset(self, **task_args: Any) -> str: + return f"prompt {sorted(task_args.items())}" + + def step(self, action: NDArray[Any]) -> None: + self.steps.append(np.asarray(action).reshape(-1).tolist()) + + def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: + done = self.terminal_at is not None and len(self.steps) >= self.terminal_at + tick = np.full((self.num_envs, 1), len(self.steps), dtype=np.float32) + return {"tick": tick}, np.full(self.num_envs, done) + + def result(self) -> dict[str, Any]: + self.results += 1 + if self.results <= self.failures: + raise RuntimeError("the sim is busy") + return {"score": 0.42, "success": True, "total_reward": 3.0, "detail": "custom"} + + +class AsyncResetSim(Sim): + """A bridge written before hooks had to be synchronous: its reset is a coroutine.""" + + def __init__(self) -> None: + super().__init__() + + async def reset(**task_args: Any) -> str: + await asyncio.sleep(0) + return "async prompt" + + setattr(self, "reset", reset) + + +@asynccontextmanager +async def serving(sim: Sim) -> AsyncIterator[tuple[int, Capability]]: + """Serve ``sim``: its control port and the ``robot`` capability an agent dials.""" + await sim.start() + control = await sim.serve_control() + try: + yield ( + control.sockets[0].getsockname()[1], + Capability.robot(url=sim.url, contract=sim.contract), + ) + finally: + control.close() + await sim.stop() + + +async def claim(control: Wire, **task_args: Any) -> dict[str, Any]: + return await control.call("reset", task_args) + + +@pytest.mark.parametrize( + ("sim", "requests", "expected"), + [ + pytest.param( + Sim(), + [("reset", {"goal": "a"}), ("reset", {"goal": "a"})], + ["prompt [('goal', 'a')]", "all 1 slots are claimed"], + id="a-full-batch-refuses-a-claim", + ), + pytest.param( + Sim(num_envs=2), + [("reset", {"task": "A"}), ("reset", {})], + [ + "prompt [('task', 'A')]", + "slots share one batch reset ({'task': 'A'}); a concurrent claim cannot use " + "different task kwargs ({}) — group tasks with identical args, or use one sim " + "per distinct task/seed", + ], + id="a-concurrent-claim-needs-identical-args", + ), + pytest.param( + Sim(), + [("reset", {}), ("result", {}), ("reset", {})], + [ + "prompt []", + {"score": 0.42, "success": True, "total_reward": 3.0, "detail": "custom"}, + "prompt []", + ], + id="result-frees-the-slot-with-the-bridges-grade", + ), + pytest.param( + AsyncResetSim(), [("reset", {})], ["async prompt"], id="an-async-reset-is-awaited" + ), + pytest.param( + Sim(), + [("result", {"token": "slot-9-nope"})], + ["unknown episode token: 'slot-9-nope'"], + id="an-unknown-token-is-refused", + ), + pytest.param(Sim(), [("teleport", {})], ["unknown method 'teleport'"], id="unknown-method"), + ], +) +async def test_the_control_channel_claims_and_frees_slots( + sim: Sim, requests: list[tuple[str, dict[str, Any]]], expected: list[Any] +) -> None: + seen: list[Any] = [] + + async with serving(sim) as (port, _), wire(f"tcp://127.0.0.1:{port}") as control: + for method, params in requests: + reply = await control.call(method, params) + if "error" in reply: + seen.append(reply["error"]["message"]) + elif "prompt" in reply["result"]: + seen.append(reply["result"]["prompt"]) + else: + seen.append(reply["result"]) + + assert seen == expected + + +async def test_an_endpoint_reset_waits_for_a_peer_to_free_the_slot() -> None: + sim = Sim() + + async with serving(sim) as (port, _): + endpoint = RobotEndpoint.remote("127.0.0.1", port) + await endpoint.start() + try: + first = await endpoint.reset(goal="a") + waiting = asyncio.create_task(endpoint.reset(goal="a")) + with pytest.raises(TimeoutError): + await asyncio.wait_for(asyncio.shield(waiting), timeout=0.2) + await endpoint.result(token=first["token"]) + second = await asyncio.wait_for(waiting, timeout=10) + finally: + await endpoint.stop() + + assert second["token"] != first["token"] + assert sim.results == 1 + + +def slot_env(endpoint: RobotEndpoint) -> Environment: + env = Environment("slots") + endpoint.attach(env) + + @env.initialize + async def connect_sim() -> None: + await endpoint.start() + + @env.shutdown + async def disconnect_sim() -> None: + await endpoint.stop() + + @env.template() + async def episode(task: str = "pick"): + claimed = await endpoint.reset(task=task) + yield claimed["prompt"] + yield await endpoint.result() + + return env + + +@pytest.mark.parametrize( + ("failures", "ending", "results_while_served", "results_after_stop"), + [ + pytest.param(0, "tasks.cancel", 2, 2, id="cancel-frees-the-slot"), + pytest.param(0, "bye", 2, 2, id="bye-frees-the-slot"), + pytest.param(0, "tasks.grade", 2, 2, id="grade-frees-the-slot-once"), + pytest.param(0, "adopt", 2, 2, id="a-dropped-session-frees-it-when-graded"), + pytest.param(1, "tasks.cancel", 3, 3, id="cancel-retries-a-failed-release"), + pytest.param(3, "tasks.cancel", 3, 4, id="stopping-frees-what-cancel-could-not"), + ], +) +async def test_ending_a_session_frees_its_slot_exactly_once( + failures: int, ending: str, results_while_served: int, results_after_stop: int +) -> None: + sim = Sim(failures=failures) + + async with serving(sim) as (port, _): + env = slot_env(RobotEndpoint.remote("127.0.0.1", port)) + async with LocalRuntime(env)(Task(env="slots", id="episode")) as runtime: + async with wire(runtime.url) as first: + await first.call("tasks.start", {"id": "episode"}) + if ending == "adopt": + await first.hang_up() + else: + await first.call(ending) + async with wire(runtime.url) as second: + if ending == "adopt": + await second.call("tasks.grade", {"answer": "x"}) + if results_while_served == results_after_stop: + started = await asyncio.wait_for( + second.call("tasks.start", {"id": "episode"}), timeout=10 + ) + graded = await second.call("tasks.grade", {"answer": "x"}) + assert (started["result"], graded["result"]["score"]) == ( + {"prompt": "prompt [('task', 'pick')]"}, + 0.42, + ) + while_served = sim.results + async with wire(f"tcp://127.0.0.1:{port}") as control: + reclaimed = await claim(control, task="pick") + + assert (while_served, sim.results) == (results_while_served, results_after_stop) + assert reclaimed["result"]["prompt"] == "prompt [('task', 'pick')]" + + +async def test_concurrent_sessions_with_different_args_cannot_share_a_batch() -> None: + sim = Sim(num_envs=2) + + async with serving(sim) as (port, _): + env = slot_env(RobotEndpoint.remote("127.0.0.1", port)) + async with ( + LocalRuntime(env)(Task(env="slots", id="episode")) as runtime, + wire(runtime.url) as first, + wire(runtime.url) as second, + ): + started = await first.call("tasks.start", {"id": "episode", "args": {"task": "A"}}) + refused = await second.call("tasks.start", {"id": "episode", "args": {"task": "B"}}) + + assert started["result"] == {"prompt": "prompt [('task', 'A')]"} + assert refused["error"]["message"].startswith( + "reset failed: slots share one batch reset ({'task': 'A'})" + ) + + +async def test_a_silent_slot_times_out_while_its_peer_keeps_stepping() -> None: + sim = Sim(num_envs=2, step_timeout=0.05) + + async with serving(sim) as (port, robot), wire(f"tcp://127.0.0.1:{port}") as control: + tokens = [(await claim(control))["result"]["token"] for _ in range(2)] + active, silent = [await RobotClient.connect(robot, token=token) for token in tokens] + try: + await active.get_observation() + await silent.get_observation() + await active.send_action([1.0]) + stepped = await active.get_observation() + with pytest.raises(RuntimeError, match="slot timed out waiting for an action"): + await silent.get_observation() + await active.send_action([2.0]) + stepped_again = await active.get_observation() + finally: + await active.close() + await silent.close() + + assert (stepped["data"]["tick"].tolist(), stepped_again["data"]["tick"].tolist()) == ( + [1.0], + [2.0], + ) + assert sim.steps == [[1.0, 0.0], [2.0, 0.0]] + + +async def test_a_claim_that_never_dials_holds_the_batch_until_it_joins() -> None: + sim = Sim(num_envs=2, step_timeout=0.5) + + async with serving(sim) as (port, robot), wire(f"tcp://127.0.0.1:{port}") as control: + tokens = [(await claim(control))["result"]["token"] for _ in range(2)] + early = await RobotClient.connect(robot, token=tokens[0]) + try: + await early.get_observation() + await early.send_action([1.0]) + # Longer than the step timeout: a claim still dialing is never timed out. + with pytest.raises(TimeoutError): + await asyncio.wait_for(early.get_observation(), timeout=1.0) + steps_before_join = len(sim.steps) + late = await RobotClient.connect(robot, token=tokens[1]) + try: + await late.get_observation() + await late.send_action([2.0]) + joined = await asyncio.wait_for(late.get_observation(), timeout=10) + finally: + await late.close() + finally: + await early.close() + + assert steps_before_join == 0 + assert joined["data"]["tick"].tolist() == [1.0] + assert sim.steps == [[1.0, 2.0]] + + +async def test_a_terminated_slot_is_not_stepped_without_new_actions() -> None: + sim = Sim(terminal_at=3) + + async with serving(sim) as (port, robot), wire(f"tcp://127.0.0.1:{port}") as control: + await claim(control) + agent = await RobotClient.connect(robot) + try: + await agent.get_observation() + terminated = [] + for value in (1.0, 2.0, 3.0): + await agent.send_action([value]) + terminated.append((await agent.get_observation())["terminated"]) + with pytest.raises(TimeoutError): + await asyncio.wait_for(agent.get_observation(), timeout=0.3) + finally: + await agent.close() + + assert terminated == [False, False, True] + assert sim.steps == [[1.0], [2.0], [3.0]] From 4918a5b7f868a6574d1ab5b3217ab0738dbed83b Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 013/248] test: spawn robot sims and gym envs through their public entry points The regression tests packed and applied spawn state through private helpers, hand-built GymBridge internals to check action reshaping, and patched the recorder's report function. Spawn a declared bridge and read back its contract, slots and metadata from the child, and check that a sim exiting before its port fails start. Serve env.gym over a Box(2, 2) env and a **kwargs vector factory and send flat actions from a RobotClient; each env reports the shape it was stepped with. The job-id spelling rows now observe the reports on the fake platform. --- tests/environment/robot_sims.py | 95 ++++ tests/environment/test_robot_gym.py | 191 +++++++ tests/environment/test_robot_regressions.py | 593 -------------------- 3 files changed, 286 insertions(+), 593 deletions(-) create mode 100644 tests/environment/robot_sims.py create mode 100644 tests/environment/test_robot_gym.py delete mode 100644 tests/environment/test_robot_regressions.py diff --git a/tests/environment/robot_sims.py b/tests/environment/robot_sims.py new file mode 100644 index 000000000..27a37fb97 --- /dev/null +++ b/tests/environment/robot_sims.py @@ -0,0 +1,95 @@ +"""Sims the robot scenarios spawn in a child process, so they live at module level. + +A spawned sim re-imports its bridge class or env factory from this file by path. +Each env reports, in its observation, the shape of the last action ``step`` got. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import numpy as np +from gymnasium.spaces import Box + +from hud.environment.robot import RobotBridge + +if TYPE_CHECKING: + from numpy.typing import NDArray + + +class Reach(RobotBridge): + """A bridge with a constructor parameter; its contract is set after construction.""" + + def __init__(self, *, use_delta: bool = False) -> None: + super().__init__() + self.use_delta = use_delta + + def reset(self, **task_args: Any) -> str: + return f"reach with use_delta={self.use_delta}" + + def step(self, action: NDArray[Any]) -> None: + del action + + def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: + return {"x": np.zeros((self.num_envs, 1), dtype=np.float32)}, np.zeros(self.num_envs, bool) + + +class PlainEnv: + """A single gym env with a ``Box(2, 2)`` action space.""" + + action_space = Box(-1.0, 1.0, shape=(2, 2), dtype=np.float32) + + def __init__(self) -> None: + self.received: tuple[int, ...] = () + + def reset(self, seed: int | None = None, options: dict[str, Any] | None = None) -> Any: + return self.observation(), {} + + def step(self, action: NDArray[Any]) -> tuple[Any, ...]: + self.received = np.shape(action) + return self.observation(), 0.5, False, False, {} + + def observation(self) -> dict[str, NDArray[Any]]: + return { + "received": np.array(self.received or (0,), dtype=np.float32), + "camera": np.zeros((4, 4, 3), dtype=np.uint8), + } + + def close(self) -> None: + pass + + +class VectorEnv: + """``num_envs`` envs stepped together, exposing the per-env ``single_action_space``.""" + + single_action_space = Box(-1.0, 1.0, shape=(2,), dtype=np.float32) + + def __init__(self, num_envs: int) -> None: + self.num_envs = num_envs + self.action_space = Box(-1.0, 1.0, shape=(num_envs, 2), dtype=np.float32) + self.received: tuple[int, ...] = () + + def reset(self, seed: int | None = None, options: dict[str, Any] | None = None) -> Any: + return self.observation(), {} + + def step(self, action: NDArray[Any]) -> tuple[Any, ...]: + self.received = np.shape(action) + rewards = np.full(self.num_envs, 0.25) + done = np.zeros(self.num_envs, dtype=bool) + return self.observation(), rewards, done, done, {} + + def observation(self) -> dict[str, NDArray[Any]]: + received = np.array(self.received or (0, 0), dtype=np.float32) + return {"received": np.tile(received, (self.num_envs, 1))} + + def close(self) -> None: + pass + + +def plain_env() -> PlainEnv: + return PlainEnv() + + +def vector_env(**build: Any) -> VectorEnv: + """A factory taking its build args as ``**kwargs``, like many registry wrappers.""" + return VectorEnv(int(build["num_envs"])) diff --git a/tests/environment/test_robot_gym.py b/tests/environment/test_robot_gym.py new file mode 100644 index 000000000..d3952ecb5 --- /dev/null +++ b/tests/environment/test_robot_gym.py @@ -0,0 +1,191 @@ +"""Spawned sims and gym shapes: what a sim process reports and what its env receives. + +``RobotEndpoint(bridge)`` and ``env.gym(factory)`` both run the sim in a child +process. These scenarios spawn the sims in ``robot_sims`` and drive them through +the public surface: the endpoint's control calls, the served env's ``robot`` +binding, and ``RobotClient``. A gym env reports in its observation the shape of +the action it was stepped with. The recorder rows check the platform reports a +shared job id produces. +""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest +import websockets +from openpi_client import msgpack_numpy + +from hud.capabilities.robot import RobotClient +from hud.environment import Environment +from hud.environment.robot import RobotEndpoint +from hud.telemetry.robot import JobRecorder +from tests.harness import served + +from .robot_sims import Reach, plain_env, vector_env + +if TYPE_CHECKING: + from collections.abc import Callable + + from tests.harness import FakeServices, HudEnv + +CONTRACT = {"control_rate": 10, "features": {"action": {"role": "action", "names": ["a"]}}} + + +async def test_a_spawned_bridge_carries_its_declaration_into_the_child() -> None: + declared = Reach(use_delta=True) + declared.contract = CONTRACT + declared.num_envs = 2 + declared.metadata = {"backend": "test"} + endpoint = RobotEndpoint(declared) + + await endpoint.start() + try: + contract = await endpoint.contract() + (robot,) = await endpoint.capabilities() + first = await endpoint.reset() + second = await endpoint.reset() + async with websockets.connect(robot.url) as socket: + greeting = msgpack_numpy.unpackb(await socket.recv()) + finally: + await endpoint.stop() + + assert contract == CONTRACT + assert (robot.name, robot.protocol, robot.params["contract"]) == ("robot", "openpi/0", CONTRACT) + assert first["prompt"] == "reach with use_delta=True" + assert [first["token"][:7], second["token"][:7]] == ["slot-0-", "slot-1-"] + assert greeting == {"backend": "test", "claim_required": True} + with pytest.raises(OSError): + await websockets.connect(robot.url, open_timeout=5) + + +async def test_a_sim_that_exits_before_announcing_its_port_fails_start() -> None: + endpoint = RobotEndpoint.spawn([sys.executable, "-c", "import sys; sys.exit(3)"]) + + with pytest.raises(RuntimeError, match="exited with code 3 before announcing its port"): + await endpoint.start() + + +def gym_env(factory: Callable[..., Any], **defaults: Any) -> Environment: + env = Environment("gym") + sim = env.gym(factory, contract=None, **defaults) + + @env.template() + async def episode(): + claimed = await sim.reset() + yield {"prompt": claimed["prompt"], "bindings": {"robot": {"token": claimed["token"]}}} + yield await sim.result(token=claimed["token"]) + + return env + + +@pytest.mark.parametrize( + ("factory", "defaults", "action", "received", "reward"), + [ + pytest.param(plain_env, {}, [0.1, 0.2, 0.3, 0.4], [2.0, 2.0], 0.5, id="box-2x2-action"), + pytest.param( + vector_env, {"num_envs": 2}, [0.1, 0.2], [2.0, 2.0], 0.25, id="kwargs-vector-env" + ), + ], +) +async def test_a_flat_action_reaches_the_gym_env_in_its_own_shape( + factory: Callable[..., Any], + defaults: dict[str, Any], + action: list[float], + received: list[float], + reward: float, +) -> None: + async with served(gym_env(factory, **defaults)) as client: + started = await client.start_task("episode") + agent = await RobotClient.connect( + client.binding("robot"), token=started["bindings"]["robot"]["token"] + ) + try: + first = await agent.get_observation() + await agent.send_action(action) + stepped = await agent.get_observation() + finally: + await agent.close() + graded = await client.grade({"answer": "done"}) + + assert sorted(first["data"]) == sorted(stepped["data"]) + assert stepped["data"]["received"].tolist() == received + assert float(stepped["reward"]) == reward + assert stepped["terminated"] is False + assert graded == {"score": reward, "success": False, "total_reward": reward} + + +async def test_a_plain_env_observation_is_sliced_to_one_slot_with_its_frames() -> None: + async with served(gym_env(plain_env)) as client: + started = await client.start_task("episode") + agent = await RobotClient.connect( + client.binding("robot"), token=started["bindings"]["robot"]["token"] + ) + try: + observation = await agent.get_observation() + finally: + await agent.close() + await client.cancel() + + shapes = {name: np.shape(value) for name, value in observation["data"].items()} + assert shapes == {"received": (1,), "camera": (4, 4, 3)} + assert float(observation["reward"]) == 0.0 + + +COMPACT_JOB = "03dd2a73d3df4d10a54ae3d87c2d530d" +CANONICAL_JOB = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" + + +@pytest.fixture +def platform(services: FakeServices, hud_env: HudEnv) -> FakeServices: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + for path in ("/v2/trace/job/{id}/enter", "/v2/trace/{id}/enter", "/v2/trace/{id}/exit"): + services.route("api", "POST", path, json={}) + services.route("telemetry", "POST", "/trace/{id}/telemetry-upload", json={}) + return services + + +@pytest.mark.parametrize("job_id", [COMPACT_JOB, CANONICAL_JOB]) +def test_a_shared_job_id_is_reported_canonically(platform: FakeServices, job_id: str) -> None: + recorder = JobRecorder("suite", 1, record_indices=[], job_id=job_id) + + assert recorder.job_url == f"{platform.url('web')}/jobs/{CANONICAL_JOB}" + assert [request.path for request in platform.requests("api")] == [ + f"/v2/trace/job/{CANONICAL_JOB}/enter" + ] + assert platform.bodies("api", "POST", "/v2/trace/job/{id}/enter") == [ + {"name": "suite", "group": 1} + ] + + +@pytest.mark.parametrize( + ("job_id", "trace_id"), + [ + (COMPACT_JOB, "e9266339c6fa58b08102ce7f41c4f372"), + (CANONICAL_JOB, "7b3269f689b65c5887542a9bc1ffb5c8"), + ], +) +def test_a_seeded_recording_keeps_the_trace_id_of_its_job_id_spelling( + platform: FakeServices, job_id: str, trace_id: str +) -> None: + JobRecorder("suite", 1, record_indices=[0], seed=7, job_id=job_id).record(done=np.array([True])) + + entered = platform.requests("api", "POST", "/v2/trace/{id}/enter") + assert [(request.params["id"], request.json) for request in entered] == [ + (trace_id, {"job_id": CANONICAL_JOB}) + ] + exited = platform.bodies("api", "POST", "/v2/trace/{id}/exit") + assert exited == [ + { + "status": "completed", + "reward": 0.0, + "metadata": {"env_index": 0, "episode_index": 0, "seed": 7}, + } + ] + + +def test_a_job_id_must_be_a_uuid() -> None: + with pytest.raises(ValueError, match="job_id must be a UUID"): + JobRecorder("suite", 1, record_indices=[], job_id="shared-suite") diff --git a/tests/environment/test_robot_regressions.py b/tests/environment/test_robot_regressions.py deleted file mode 100644 index 151d9e890..000000000 --- a/tests/environment/test_robot_regressions.py +++ /dev/null @@ -1,593 +0,0 @@ -"""Unit regressions for robot bridge, endpoint claims, gym shapes, and spawn state.""" - -from __future__ import annotations - -import asyncio -import contextlib -import math -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, cast -from unittest.mock import AsyncMock - -import numpy as np -import pytest - -from hud.environment.env import current_session_id -from hud.environment.robot import DirectControl -from hud.environment.robot.bridge import _HUD_STATE, RobotBridge, _apply_declaration_state -from hud.environment.robot.endpoint import RobotEndpoint, _bridge_init_kwargs -from hud.environment.robot.gym import GymBridge, action_dim_of -from hud.environment.robot.orientation import angular_distance, euler_to_xyzw, xyzw_to_euler -from hud.telemetry.robot.recorder import JobRecorder - -if TYPE_CHECKING: - from numpy.typing import NDArray - -# --- spawn state (declaration → child) ------------------------------------------------- - - -class _CustomBridge(RobotBridge): - """Subclass with a real ctor param — contract is assigned after init (docs style).""" - - def __init__(self, *, use_delta: bool = False) -> None: - super().__init__() - self.use_delta = use_delta - - def reset(self, **kwargs: Any) -> str: - return "p" - - def step(self, action: Any) -> None: - return None - - def get_observation(self) -> tuple[dict[str, Any], Any] | None: - return None - - -def test_bridge_init_kwargs_packs_declaration_contract_and_ctor_params() -> None: - bridge = _CustomBridge(use_delta=True) - bridge.contract = { - "control_rate": 10, - "features": {"action": {"role": "action", "names": ["a"]}}, - } - bridge.num_envs = 4 - bridge.metadata = {"backend": "test"} - - kwargs = _bridge_init_kwargs(bridge) - assert kwargs["use_delta"] is True - assert kwargs[_HUD_STATE]["contract"] == bridge.contract - assert kwargs[_HUD_STATE]["num_envs"] == 4 - assert kwargs[_HUD_STATE]["metadata"] == {"backend": "test"} - # Bind address stays with the child — never forwarded. - assert "host" not in kwargs and "port" not in kwargs - - -def test_bridge_init_kwargs_skips_empty_default_state() -> None: - bridge = _CustomBridge() - kwargs = _bridge_init_kwargs(bridge) - assert kwargs == {"use_delta": False} - assert _HUD_STATE not in kwargs - - -def test_apply_declaration_state_sets_contract_after_ctor() -> None: - """Child reconstructs with ctor kwargs only; state is applied afterward.""" - bridge = _CustomBridge(use_delta=True) - assert bridge.contract == {} - _apply_declaration_state( - bridge, - { - "contract": {"features": {"action": {"role": "action", "names": ["a"]}}}, - "num_envs": 2, - "metadata": {"k": 1}, - }, - ) - assert bridge.contract["features"]["action"]["names"] == ["a"] - assert bridge.num_envs == 2 - assert len(bridge._registry.slots) == 2 - assert bridge.metadata == {"k": 1} - - -# --- gym shapes ------------------------------------------------------------------------ - - -@pytest.mark.parametrize( - ("single_shape",), - [ - pytest.param((2, 2), id="keeps_single_action_space_shape"), - pytest.param(None, id="strips_batched_action_space_without_single"), - ], -) -def test_action_dim_of_batched(single_shape: tuple[int, ...] | None) -> None: - env = SimpleNamespace( - single_action_space=None if single_shape is None else SimpleNamespace(shape=single_shape), - action_space=SimpleNamespace(shape=(4, 2, 2)), - ) - assert action_dim_of(env, batched=True) == 4 - - -def test_var_keyword_factory_defaults_are_build_params() -> None: - def make_env(**kwargs: Any) -> Any: - return kwargs - - bridge = GymBridge(make_env, contract=None, num_envs=8) - assert "num_envs" in bridge._build_params - - -def test_step_reshapes_float_box_action() -> None: - class _FakeEnv: - action_space = SimpleNamespace(shape=(2, 2), dtype=np.float32) - - def __init__(self) -> None: - self.last_action: Any = None - - def step(self, action: Any) -> tuple[Any, ...]: - self.last_action = action - return np.zeros(3, dtype=np.float32), 0.0, False, False, {} - - bridge = GymBridge(lambda: None, contract=None) - bridge.env = _FakeEnv() - bridge.batched = False - bridge.num_envs = 1 - bridge._done = np.zeros(1, dtype=bool) - bridge._success = np.zeros(1, dtype=bool) - bridge._acc_reward = np.zeros(1) - bridge._step_reward = np.zeros(1) - bridge.step(np.arange(4, dtype=np.float32).reshape(1, 4)) - assert bridge.env.last_action.shape == (2, 2) - - -def test_step_reshapes_batched_float_box_action() -> None: - class _FakeEnv: - action_space = SimpleNamespace(shape=(2, 2, 2), dtype=np.float32) - - def __init__(self) -> None: - self.last_action: Any = None - - def step(self, action: Any) -> tuple[Any, ...]: - self.last_action = action - return ( - np.zeros((2, 3), dtype=np.float32), - np.zeros(2), - np.zeros(2, dtype=bool), - np.zeros(2, dtype=bool), - {}, - ) - - bridge = GymBridge(lambda: None, contract=None) - bridge.env = _FakeEnv() - bridge.batched = True - bridge.num_envs = 2 - bridge._done = np.zeros(2, dtype=bool) - bridge._success = np.zeros(2, dtype=bool) - bridge._acc_reward = np.zeros(2) - bridge._step_reward = np.zeros(2) - bridge.step(np.arange(8, dtype=np.float32).reshape(2, 4)) - assert bridge.env.last_action.shape == (2, 2, 2) - - -def test_plain_env_observation_always_gets_batch_axis() -> None: - bridge = GymBridge(lambda: None, contract=None) - bridge.env = object() - bridge.batched = False - bridge.num_envs = 1 - bridge._done = np.zeros(1, dtype=bool) - bridge._step_reward = np.array([0.5], dtype=np.float32) - bridge._obs = { - "state": np.array([1.0], dtype=np.float32), - "camera": np.zeros((1, 4, 3), dtype=np.uint8), - } - observation = bridge.get_observation() - assert observation is not None - data, terminated = observation - assert data is not None - assert data["state"].shape == (1, 1) - assert data["camera"].shape == (1, 1, 4, 3) - assert data["reward"].shape == (1,) - assert data["reward"][0] == pytest.approx(0.5) - assert terminated.shape == (1,) - - -def test_job_recorder_canonicalizes_shared_job_id(monkeypatch: pytest.MonkeyPatch) -> None: - from hud.settings import settings - - compact_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - canonical_id = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" - reports: list[tuple[str, dict[str, Any]]] = [] - monkeypatch.setattr(settings, "hud_web_url", "https://hud.test") - monkeypatch.setattr( - "hud.telemetry.robot.recorder._report_sync", - lambda path, payload: reports.append((path, payload)), - ) - - recorder = JobRecorder("test", 1, record_indices=[], job_id=compact_id) - - assert recorder.job_id == canonical_id - assert recorder.job_url == f"https://hud.test/jobs/{canonical_id}" - assert reports == [(f"/trace/job/{canonical_id}/enter", {"name": "test", "group": 1})] - - -def test_seeded_recording_preserves_trace_id_for_each_job_id_spelling( - monkeypatch: pytest.MonkeyPatch, -) -> None: - compact_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - canonical_id = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" - legacy_trace_ids = { - compact_id: "e9266339c6fa58b08102ce7f41c4f372", - canonical_id: "7b3269f689b65c5887542a9bc1ffb5c8", - } - reports: list[tuple[str, dict[str, Any]]] = [] - - def capture_report(path: str, payload: dict[str, Any]) -> None: - reports.append((path, payload)) - - monkeypatch.setattr("hud.telemetry.robot.recorder._report_sync", capture_report) - - for job_id, expected_id in legacy_trace_ids.items(): - reports.clear() - recorder = JobRecorder("test", 1, record_indices=[0], seed=7, job_id=job_id) - recorder.record(done=np.array([True])) - - assert ( - f"/trace/{expected_id}/enter", - {"job_id": canonical_id, "group_id": None, "model": None}, - ) in reports - - -def test_job_recorder_rejects_non_uuid_job_id() -> None: - with pytest.raises(ValueError, match="job_id must be a UUID"): - JobRecorder("test", 1, record_indices=[], job_id="shared-suite") - - -# --- endpoint session claims ----------------------------------------------------------- - - -@pytest.fixture -def endpoint() -> RobotEndpoint: - ep = RobotEndpoint.remote("127.0.0.1", 9) - setattr(ep, "_call", AsyncMock()) - return ep - - -@pytest.mark.asyncio -async def test_release_claim_frees_current_session_slot(endpoint: RobotEndpoint) -> None: - call = cast("AsyncMock", endpoint._call) - call.return_value = {"prompt": "p", "token": "slot-0-abcd"} - token = current_session_id.set("sess-a") - try: - await endpoint.reset() - assert endpoint._claims["sess-a"] == "slot-0-abcd" - call.return_value = {"score": 0.0} - await endpoint.release_claim() - assert "sess-a" not in endpoint._claims - call.assert_called_with("result", {"token": "slot-0-abcd"}) - finally: - current_session_id.reset(token) - - -@pytest.mark.asyncio -async def test_release_claim_ends_the_direct_control_episode( - endpoint: RobotEndpoint, monkeypatch: pytest.MonkeyPatch -) -> None: - control = DirectControl().attach(endpoint) - end_episode = AsyncMock() - monkeypatch.setattr(control, "end_episode", end_episode) - cast("AsyncMock", endpoint._call).return_value = {"prompt": "p", "token": "slot-0-abcd"} - token = current_session_id.set("sess-a") - try: - await endpoint.reset() - end_episode.reset_mock() - await endpoint.release_claim() - end_episode.assert_awaited_once() - finally: - current_session_id.reset(token) - - -@pytest.mark.asyncio -async def test_release_claim_on_shutdown_frees_each_session(endpoint: RobotEndpoint) -> None: - call = cast("AsyncMock", endpoint._call) - endpoint._claims["sess-a"] = "tok-a" - endpoint._claims["sess-b"] = "tok-b" - call.return_value = {"score": 0.0} - - for sid in ("sess-a", "sess-b"): - token = current_session_id.set(sid) - try: - await endpoint.release_claim() - finally: - current_session_id.reset(token) - - assert endpoint._claims == {} - tokens = [c.args[1]["token"] for c in call.call_args_list] - assert sorted(tokens) == ["tok-a", "tok-b"] - - -@pytest.mark.asyncio -async def test_result_marks_claim_freed_so_disconnect_does_not_re_result( - endpoint: RobotEndpoint, -) -> None: - call = cast("AsyncMock", endpoint._call) - token = current_session_id.set("sess-a") - try: - endpoint._claims["sess-a"] = "tok-a" - call.return_value = {"score": 1.0, "success": True, "total_reward": 1.0} - await endpoint.result(token="tok-a") - assert endpoint._claims["sess-a"] == "" - call.reset_mock() - await endpoint.release_claim() - call.assert_not_called() - finally: - current_session_id.reset(token) - - -@pytest.mark.asyncio -async def test_failed_release_rpc_retries_then_succeeds(endpoint: RobotEndpoint) -> None: - call = cast("AsyncMock", endpoint._call) - token = current_session_id.set("sess-a") - try: - endpoint._claims["sess-a"] = "tok-a" - call.side_effect = [ - ConnectionError("sim down"), - {"score": 0.0}, - ] - await endpoint.release_claim() - assert "sess-a" not in endpoint._claims - assert call.call_count == 2 - finally: - current_session_id.reset(token) - - -@pytest.mark.asyncio -async def test_stop_drains_claims_that_cancel_failed_to_free(endpoint: RobotEndpoint) -> None: - call = cast("AsyncMock", endpoint._call) - endpoint._claims["sess-a"] = "tok-a" - call.side_effect = [ - ConnectionError("sim down"), - ConnectionError("sim down"), - ConnectionError("sim down"), - ] - token = current_session_id.set("sess-a") - try: - await endpoint.release_claim() # retries exhausted — claim kept - assert endpoint._claims["sess-a"] == "tok-a" - finally: - current_session_id.reset(token) - - call.side_effect = None - call.return_value = {"score": 0.0} - await endpoint.stop() # last chance while the control link is up - assert endpoint._claims == {} - - -# --- bridge barrier / claim ------------------------------------------------------------ - - -class _StubBridge(RobotBridge): - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self.steps: list[NDArray[Any]] = [] - self.contract = {"features": {"action": {"role": "action", "names": ["a"]}}} - - def reset(self, **kwargs: Any) -> str: - return f"prompt:{kwargs!r}" - - def step(self, action: NDArray[Any]) -> None: - self.steps.append(np.asarray(action)) - - def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]] | None: - return {"x": np.zeros((self.num_envs, 1), dtype=np.float32)}, np.zeros( - self.num_envs, dtype=bool - ) - - -class _FakeWS: - async def send(self, _data: Any) -> None: - return None - - -@pytest.mark.asyncio -async def test_claim_awaits_legacy_async_reset() -> None: - bridge = _StubBridge() - - async def async_reset(**kwargs: Any) -> str: - await asyncio.sleep(0) - return "async-prompt" - - setattr(bridge, "reset", async_reset) - ep = await bridge._claim_episode() - assert ep["prompt"] == "async-prompt" - - -@pytest.mark.asyncio -async def test_claim_rejects_empty_kwargs_after_nonempty_batch() -> None: - bridge = _StubBridge() - bridge.num_envs = 2 - first = await bridge._claim_episode(task="A") - assert first["token"] - with pytest.raises(ValueError, match="identical args"): - await bridge._claim_episode() - - -@pytest.mark.asyncio -async def test_release_uses_overridden_result() -> None: - class _CustomGrade(_StubBridge): - def result(self) -> dict[str, Any]: - return {"score": 0.42, "success": True, "total_reward": 3.0, "detail": "custom"} - - bridge = _CustomGrade() - ep = await bridge._claim_episode() - grade = await bridge._release_episode(ep["token"]) - assert grade == {"score": 0.42, "success": True, "total_reward": 3.0, "detail": "custom"} - - -@pytest.mark.asyncio -async def test_tick_loop_does_not_hold_spin_when_all_claimed_idle() -> None: - """Terminated slots may keep WS open until close(); barrier must not step.""" - bridge = _StubBridge() - bridge.num_envs = 1 - bridge._registry.configure(1) - slot = bridge._registry.slots[0] - bridge._registry.claim(slot) - slot.ws = _FakeWS() # still connected - slot.idle = True # terminated - slot.action = None - - task = asyncio.create_task(bridge._tick_loop()) - bridge._action_event.set() - await asyncio.sleep(0.05) - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - assert bridge.steps == [] - - -@pytest.mark.asyncio -async def test_tick_loop_stops_after_terminal_obs_while_ws_still_open() -> None: - """Terminate wakes the barrier; idle+open WS must not cascade hold-steps. - - The e2e gate ``sim.tick == EPISODE_TICKS`` fails if each terminal fan-out - re-arms the loop into a hold-spin before the agent closes the socket. - """ - - class _Terminating(_StubBridge): - def __init__(self) -> None: - super().__init__() - self.tick = 0 - - def step(self, action: NDArray[Any]) -> None: - self.tick += 1 - super().step(action) - - def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]] | None: - data = {"x": np.zeros((self.num_envs, 1), dtype=np.float32)} - return data, np.array([self.tick >= 3]) - - bridge = _Terminating() - bridge.num_envs = 1 - bridge._registry.configure(1) - slot = bridge._registry.slots[0] - bridge._registry.claim(slot) - slot.ws = _FakeWS() - task = asyncio.create_task(bridge._tick_loop()) - try: - for expected in (1, 2, 3): - slot.idle = False - slot.action = np.array([1.0], dtype=np.float32) - bridge._action_event.set() - for _ in range(50): - if bridge.tick >= expected: - break - await asyncio.sleep(0.01) - assert bridge.tick == expected - assert slot.idle # terminal obs dropped the slot out of the barrier - for _ in range(20): # spam the wake that terminal fan-out itself sets - bridge._action_event.set() - await asyncio.sleep(0.005) - assert bridge.tick == 3 - assert len(bridge.steps) == 3 - finally: - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - - -@pytest.mark.asyncio -async def test_claim_raises_when_batch_full() -> None: - bridge = _StubBridge() - bridge.num_envs = 1 - bridge._registry.configure(1) - first = await bridge._claim_episode(goal="a") - with pytest.raises(RuntimeError, match="slots are claimed"): - await bridge._claim_episode(goal="a") - await bridge._release_episode(first["token"]) - second = await bridge._claim_episode(goal="a") - assert second["token"] - - -@pytest.mark.asyncio -async def test_endpoint_reset_retries_until_peer_result_frees_slot() -> None: - """Shared width+1: reset retries outside the RPC lock so result is not deadlocked.""" - bridge = _StubBridge() - bridge.num_envs = 1 - server = await bridge.serve_control("127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - ep = RobotEndpoint.remote("127.0.0.1", port) - await ep.start() - try: - first = await ep.reset(goal="a") - waiting = asyncio.create_task(ep.reset(goal="a")) - await asyncio.sleep(0.05) - assert not waiting.done() - await asyncio.wait_for(ep.result(token=first["token"]), timeout=1.0) - second = await asyncio.wait_for(waiting, timeout=1.0) - assert second["token"] != first["token"] - finally: - await ep.stop() - server.close() - await server.wait_closed() - - -@pytest.mark.asyncio -async def test_tick_loop_times_out_silent_live_slot() -> None: - """A connected agent that never sends must not stall the barrier forever.""" - bridge = _StubBridge() - bridge.step_timeout = 0.05 - bridge.num_envs = 2 - bridge._registry.configure(2) - a, b = bridge._registry.slots - bridge._registry.claim(a) - bridge._registry.claim(b) - a.ws, b.ws = _FakeWS(), _FakeWS() - a.idle = b.idle = False - a.action = np.array([1.0], dtype=np.float32) # ready - b.action = None # silent - - task = asyncio.create_task(bridge._tick_loop()) - bridge._action_event.set() - for _ in range(50): - if bridge.steps: - break - await asyncio.sleep(0.02) - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - assert len(bridge.steps) == 1 - assert b.idle # timed out - - -@pytest.mark.asyncio -async def test_tick_loop_does_not_hold_step_undialed_claimed_slot() -> None: - """A peer that has claimed but not WS-connected must not be hold-advanced.""" - bridge = _StubBridge() - bridge.step_timeout = 0.05 - bridge.num_envs = 2 - bridge._registry.configure(2) - a, b = bridge._registry.slots - bridge._registry.claim(a) - bridge._registry.claim(b) - a.ws = _FakeWS() - a.idle = b.idle = False - a.action = np.array([1.0], dtype=np.float32) - # b: claimed, still dialing (ws is None) - - task = asyncio.create_task(bridge._tick_loop()) - bridge._action_event.set() - await asyncio.sleep(0.15) - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - assert bridge.steps == [] - assert not b.idle # still waiting to dial — not timed out into hold - - -def test_both_pitch_singularities_keep_roll_through_a_partial_yaw_merge() -> None: - """A wrist at ±pi/2 reports its twist as roll, so naming only yaw rebuilds that start.""" - roll = 0.4 - yaw = -0.25 - for pitch in (math.pi / 2, -math.pi / 2): - got_roll, got_pitch, got_yaw = xyzw_to_euler(euler_to_xyzw(roll, pitch, 0.0)) - assert got_roll == pytest.approx(roll) - assert got_pitch == pytest.approx(pitch) - assert got_yaw == pytest.approx(0.0, abs=1e-8) - merged = euler_to_xyzw(got_roll, got_pitch, yaw) - assert angular_distance(merged, euler_to_xyzw(roll, pitch, yaw)) == pytest.approx( - 0.0, abs=1e-8 - ) From 4b70807286530ceecd4d09070dc07846a519ce53 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 014/248] refactor: remove the unused hint catalog and private grader export The hint catalog in hud/utils/hints.py was only rendered by HUDConsole.render_exception, which nothing calls; the CLI prints errors through CliError. Exceptions no longer carry hint lists. hud.graders no longer exports the private _combine_subscores, which combine() uses internally. --- hud/graders/__init__.py | 3 +- hud/graders/combine.py | 2 +- hud/utils/exceptions.py | 86 +------------- hud/utils/hints.py | 201 -------------------------------- hud/utils/hud_console.py | 47 -------- tests/utils/test_exceptions.py | 102 ---------------- tests/utils/test_hints.py | 167 -------------------------- tests/utils/test_hud_console.py | 16 --- tests/utils/test_requests.py | 16 --- 9 files changed, 6 insertions(+), 634 deletions(-) delete mode 100644 hud/utils/hints.py delete mode 100644 tests/utils/test_exceptions.py delete mode 100644 tests/utils/test_hints.py diff --git a/hud/graders/__init__.py b/hud/graders/__init__.py index 8df731d98..a6bc2c8ca 100644 --- a/hud/graders/__init__.py +++ b/hud/graders/__init__.py @@ -25,7 +25,7 @@ from .base import Grader from .bash import BashGrader -from .combine import _combine_subscores, combine, combine_all, combine_any +from .combine import combine, combine_all, combine_any from .judge import LLMJudgeGrader from .results import EvaluationResult, SubScore from .text import ( @@ -44,7 +44,6 @@ "Grader", "LLMJudgeGrader", "SubScore", - "_combine_subscores", "combine", "combine_all", "combine_any", diff --git a/hud/graders/combine.py b/hud/graders/combine.py index e48886796..5b697b471 100644 --- a/hud/graders/combine.py +++ b/hud/graders/combine.py @@ -159,4 +159,4 @@ def combine_all(weight: float, subscores: list[SubScore], *, name: str = "all") return _boolean_subscore(name, weight, subscores, min(s.value for s in subscores)) -__all__ = ["_combine_subscores", "combine", "combine_all", "combine_any"] +__all__ = ["combine", "combine_all", "combine_any"] diff --git a/hud/utils/exceptions.py b/hud/utils/exceptions.py index 2bbdb72ba..22bfdbf8e 100644 --- a/hud/utils/exceptions.py +++ b/hud/utils/exceptions.py @@ -1,54 +1,25 @@ -"""HUD SDK exceptions. - -A small typed hierarchy rooted at :class:`HudException`. Subclasses carry -default :class:`~hud.utils.hints.Hint` lists that the console renderer -displays alongside the error. -""" +"""HUD SDK exceptions: a small typed hierarchy rooted at :class:`HudException`.""" from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from typing import Self import httpx -from hud.utils.hints import ( - CLIENT_NOT_INITIALIZED, - CREDITS_EXHAUSTED, - ENV_VAR_MISSING, - HUD_API_KEY_MISSING, - INVALID_CONFIG, - MCP_SERVER_ERROR, - PRO_PLAN_REQUIRED, - RATE_LIMIT_HIT, - TOOL_NOT_FOUND, - Hint, -) - logger = logging.getLogger(__name__) class HudException(Exception): """Base exception class for all HUD SDK errors.""" - # Subclasses can override this class attribute - default_hints: ClassVar[list[Hint]] = [] - - def __init__( - self, - message: str = "", - response_json: dict[str, Any] | None = None, - *, - hints: list[Hint] | None = None, - ) -> None: + def __init__(self, message: str = "", response_json: dict[str, Any] | None = None) -> None: super().__init__(message) self.message = message self.response_json = response_json - # If hints not provided, use defaults defined by subclass - self.hints: list[Hint] = hints if hints is not None else list(self.default_hints) def __str__(self) -> str: if self.response_json: @@ -67,46 +38,11 @@ def __init__( response_text: str | None = None, response_json: dict[str, Any] | None = None, response_headers: dict[str, str] | None = None, - *, - hints: list[Hint] | None = None, ) -> None: self.status_code = status_code self.response_text = response_text self.response_headers = response_headers - if hints is None: - hints = self._hints_for_status(status_code, message, response_text, response_json) - super().__init__(message, response_json, hints=hints) - - @staticmethod - def _hints_for_status( - status_code: int | None, - message: str, - response_text: str | None, - response_json: dict[str, Any] | None, - ) -> list[Hint] | None: - if status_code == 401: - return [HUD_API_KEY_MISSING] - if status_code == 402: - return [CREDITS_EXHAUSTED] - if status_code == 429: - return [RATE_LIMIT_HIT] - if status_code == 403: - # Default 403 to auth unless the message clearly indicates Pro plan - combined = message.lower() - if response_text: - combined += "\n" + response_text.lower() - if response_json: - detail = response_json.get("detail") - if isinstance(detail, str): - combined += "\n" + detail.lower() - mentions_pro = ( - "pro plan" in combined - or "requires pro" in combined - or "pro mode" in combined - or combined.strip().startswith("pro ") - ) - return [PRO_PLAN_REQUIRED] if mentions_pro else [HUD_API_KEY_MISSING] - return None + super().__init__(message, response_json) def __str__(self) -> str: parts = [self.message] @@ -182,14 +118,10 @@ def from_httpx_error(cls, error: httpx.HTTPStatusError, context: str = "") -> Se class HudAuthenticationError(HudException): """Missing or invalid HUD API key.""" - default_hints: ClassVar[list[Hint]] = [HUD_API_KEY_MISSING] - class HudRateLimitError(HudException): """Too many requests to the API.""" - default_hints: ClassVar[list[Hint]] = [RATE_LIMIT_HIT] - class HudTimeoutError(HudException): """Request timed out.""" @@ -202,32 +134,22 @@ class HudNetworkError(HudException): class HudClientError(HudException): """MCP client not initialized.""" - default_hints: ClassVar[list[Hint]] = [CLIENT_NOT_INITIALIZED] - class HudConfigError(HudException): """Invalid or missing configuration.""" - default_hints: ClassVar[list[Hint]] = [INVALID_CONFIG] - class HudEnvVarError(HudException): """Missing required environment variables.""" - default_hints: ClassVar[list[Hint]] = [ENV_VAR_MISSING] - class HudToolNotFoundError(HudException): """Requested tool not found.""" - default_hints: ClassVar[list[Hint]] = [TOOL_NOT_FOUND] - class HudMCPError(HudException): """MCP protocol or server error.""" - default_hints: ClassVar[list[Hint]] = [MCP_SERVER_ERROR] - class HudDeprecationWarning(FutureWarning): """The HUD API deprecated a route this client calls.""" diff --git a/hud/utils/hints.py b/hud/utils/hints.py deleted file mode 100644 index 2c66e7986..000000000 --- a/hud/utils/hints.py +++ /dev/null @@ -1,201 +0,0 @@ -from __future__ import annotations - -import logging -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from collections.abc import Iterable -logger = logging.getLogger(__name__) - - -@dataclass -class Hint: - """Structured hint for user guidance. - - Attributes: - title: Short title describing the hint. - message: Main explanatory message. - tips: Optional list of short actionable tips. - docs_url: Optional URL for documentation. - command_examples: Optional list of command examples to show. - code: Optional machine-readable code (e.g., "AUTH_API_KEY_MISSING"). - context: Optional context tags (e.g., ["auth", "docker", "mcp"]). - """ - - title: str - message: str - tips: list[str] | None = None - docs_url: str | None = None - command_examples: list[str] | None = None - code: str | None = None - context: list[str] | None = None - - -# Common, reusable hints -HUD_API_KEY_MISSING = Hint( - title="HUD API key required", - message="Missing or invalid HUD_API_KEY.", - tips=[ - "Set HUD_API_KEY in your environment or run: hud set HUD_API_KEY=your-key-here", - "Get a key at https://hud.ai", - "Check for whitespace or truncation", - ], - docs_url=None, - command_examples=None, - code="HUD_AUTH_MISSING", - context=["auth", "hud"], -) - -RATE_LIMIT_HIT = Hint( - title="Rate limit reached", - message="Too many requests.", - tips=[ - "Lower --max-concurrent", - "Add retry delay", - "Check API quotas", - ], - docs_url=None, - command_examples=None, - code="RATE_LIMIT", - context=["network"], -) - -# Billing / plan upgrade -PRO_PLAN_REQUIRED = Hint( - title="Pro plan required", - message="This feature requires Pro.", - tips=[ - "Upgrade your plan to continue", - ], - docs_url="https://hud.ai/project/billing", - command_examples=None, - code="PRO_PLAN_REQUIRED", - context=["billing", "plan"], -) - -CREDITS_EXHAUSTED = Hint( - title="Credits exhausted", - message="Your credits are exhausted.", - tips=[ - "Top up credits or upgrade your plan", - ], - docs_url="https://hud.ai/project/billing", - command_examples=None, - code="CREDITS_EXHAUSTED", - context=["billing", "credits"], -) - -TOOL_NOT_FOUND = Hint( - title="Tool not found", - message="Requested tool doesn't exist.", - tips=[ - "Check tool name spelling", - "Run: hud analyze --live ", - "Verify server implements tool", - ], - docs_url=None, - command_examples=None, - code="TOOL_NOT_FOUND", - context=["mcp", "tools"], -) - -CLIENT_NOT_INITIALIZED = Hint( - title="Client not initialized", - message="MCP client must be initialized before use.", - tips=[ - "Call client.initialize() first", - "Or use async with client:", - "Check connection succeeded", - ], - docs_url=None, - command_examples=None, - code="CLIENT_NOT_INIT", - context=["mcp", "client"], -) - -INVALID_CONFIG = Hint( - title="Invalid configuration", - message="Configuration is missing or malformed.", - tips=[ - "Check JSON syntax", - "Verify required fields", - "See examples in docs", - ], - docs_url=None, - command_examples=None, - code="INVALID_CONFIG", - context=["config"], -) - -ENV_VAR_MISSING = Hint( - title="Environment variable required", - message="Required environment variables are missing.", - tips=[ - "Set required environment variables", - "Use -e flag: hud deploy . -e VAR_NAME=value", - "Check Dockerfile for ENV requirements", - ], - docs_url=None, - command_examples=["hud deploy . -e BROWSER_PROVIDER=anchorbrowser"], - code="ENV_VAR_MISSING", - context=["env", "config"], -) - -MCP_SERVER_ERROR = Hint( - title="MCP server error", - message="The MCP server encountered an error.", - tips=[ - "Check server logs for details", - "Verify server configuration", - "Ensure all dependencies are installed", - ], - docs_url=None, - command_examples=["hud serve --verbose"], - code="MCP_SERVER_ERROR", - context=["mcp", "server"], -) - - -def render_hints(hints: Iterable[Hint] | None, *, design: Any | None = None) -> None: - """Render a collection of hints using the HUD design system if available. - - If design is not provided, this is a no-op to keep library use headless. - """ - if not hints: - return - - hud_console = design - try: - if hud_console is None: - from hud.utils.hud_console import hud_console as default_design # lazy import - - hud_console = default_design - except Exception: - # If design is unavailable (non-CLI contexts), silently skip rendering - return - - for hint in hints: - try: - # Compact rendering - skip title if same as message - if hint.title and hint.title != hint.message: - hud_console.warning(f"{hint.title}: {hint.message}") - else: - hud_console.warning(hint.message) - - # Tips as bullet points - if hint.tips: - for tip in hint.tips: - hud_console.info(f" • {tip}") - - # Only show command examples if provided - if hint.command_examples: - for cmd in hint.command_examples: - hud_console.command_example(cmd) - - # Only show docs URL if provided - if hint.docs_url: - hud_console.link(hint.docs_url) - except Exception: - logger.warning("Failed to render hint: %s", hint) - continue diff --git a/hud/utils/hud_console.py b/hud/utils/hud_console.py index a6e47ba4b..374ba881f 100644 --- a/hud/utils/hud_console.py +++ b/hud/utils/hud_console.py @@ -249,53 +249,6 @@ def command_example( else: console.print(f" [{SECONDARY}]{command}[/{SECONDARY}]") - # Exception rendering utilities - def render_support_hint(self, stderr: bool = True) -> None: - """Render a standard support message for users encountering issues.""" - support = ( - "If this looks like an issue with the sdk, please make a github issue at " - "https://github.com/hud-evals/hud-python/issues" - ) - self.info(support, stderr=stderr) - - def render_exception(self, error: BaseException, *, stderr: bool = True) -> None: - """Render exceptions consistently using the HUD design system. - - - Shows exception type and message - - Displays structured hints if present on the exception (e.g., HudException.hints) - - Prints a link to open an issue for SDK problems - """ - from hud.utils.exceptions import HudRequestError # lazy import: avoid import cycle - - # Header with exception type - ex_type = type(error).__name__ - message = getattr(error, "message", "") or str(error) or ex_type - self.error(f"{ex_type}: {message}", stderr=stderr) - - # Specialized details for request errors - if isinstance(error, HudRequestError): - details: dict[str, str | int | float] = {} - if error.status_code is not None: - details["Status"] = str(error.status_code) - if error.response_text: - # Limit very long responses - text = error.response_text - details["Response"] = text[:500] + ("..." if len(text) > 500 else "") - if error.response_json and "Response" not in details: - details["Response JSON"] = str(error.response_json) - if details: - self.key_value_table(details, show_header=False, stderr=stderr) - - # Structured hints, if available - hints = getattr(error, "hints", None) - if hints: - from hud.utils.hints import render_hints # lazy import: avoid import cycle - - render_hints(hints, design=self) - - # Standard support hint - self.render_support_hint(stderr=stderr) - @property def stdout(self) -> Console: return self._stdout_console diff --git a/tests/utils/test_exceptions.py b/tests/utils/test_exceptions.py deleted file mode 100644 index 96dae8d95..000000000 --- a/tests/utils/test_exceptions.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Tests for the HUD SDK exception hierarchy.""" - -from __future__ import annotations - -import httpx - -from hud.utils.exceptions import ( - HudAuthenticationError, - HudException, - HudRequestError, -) -from hud.utils.hints import ( - HUD_API_KEY_MISSING, - PRO_PLAN_REQUIRED, - RATE_LIMIT_HIT, -) - - -class TestHudException: - def test_message_and_str(self): - error = HudException("Something broke") - assert error.message == "Something broke" - assert str(error) == "Something broke" - assert error.hints == [] - - def test_response_json_in_str(self): - error = HudException("Bad payload", response_json={"detail": "nope"}) - assert str(error) == "Bad payload | Response: {'detail': 'nope'}" - - def test_subclass_default_hints(self): - error = HudAuthenticationError("API key missing") - assert error.hints == [HUD_API_KEY_MISSING] - assert error.hints[0].title == "HUD API key required" - # Hint copy evolved; keep the assertion robust to minor copy changes - tips = error.hints[0].tips - assert tips and "Set HUD_API_KEY" in tips[0] - - -class TestHudRequestError: - """Test HudRequestError specific behavior.""" - - def test_401_adds_auth_hint(self): - error = HudRequestError("Unauthorized", status_code=401) - assert HUD_API_KEY_MISSING in error.hints - - def test_403_adds_auth_hint(self): - error = HudRequestError("Forbidden", status_code=403) - assert HUD_API_KEY_MISSING in error.hints - - def test_403_pro_plan_message_sets_pro_hint(self): - """403 with Pro wording should map to PRO_PLAN_REQUIRED, not auth.""" - error = HudRequestError("Feature requires Pro plan", status_code=403) - assert PRO_PLAN_REQUIRED in error.hints - assert HUD_API_KEY_MISSING not in error.hints - - def test_403_pro_plan_detail_sets_pro_hint(self): - """403 with detail indicating Pro should map to PRO_PLAN_REQUIRED.""" - error = HudRequestError( - "Forbidden", - status_code=403, - response_json={"detail": "Requires Pro plan"}, - ) - assert PRO_PLAN_REQUIRED in error.hints - assert HUD_API_KEY_MISSING not in error.hints - - def test_429_adds_rate_limit_hint(self): - error = HudRequestError("Too Many Requests", status_code=429) - assert RATE_LIMIT_HIT in error.hints - - def test_other_status_no_default_hints(self): - error = HudRequestError("Server Error", status_code=500) - assert error.hints == [] - - def test_explicit_hints_override_defaults(self): - from hud.utils.hints import Hint - - custom_hint = Hint(title="Custom Error", message="This is a custom hint") - error = HudRequestError("Unauthorized", status_code=401, hints=[custom_hint]) - assert error.hints == [custom_hint] - assert HUD_API_KEY_MISSING not in error.hints - - def test_from_httpx_error(self): - request = httpx.Request("GET", "https://api.test.com") - response = httpx.Response(404, json={"detail": "Not found"}, request=request) - httpx_error = httpx.HTTPStatusError("Not found", request=request, response=response) - - error = HudRequestError.from_httpx_error(httpx_error, context="Testing") - - assert error.status_code == 404 - assert "Testing" in str(error) - assert "Not found" in str(error) - assert error.response_json == {"detail": "Not found"} - - def test_string_representation(self): - error = HudRequestError( - "Request failed", status_code=404, response_json={"error": "Not found"} - ) - - error_str = str(error) - assert "Request failed" in error_str - assert "Status: 404" in error_str - assert "Response JSON: {'error': 'Not found'}" in error_str diff --git a/tests/utils/test_hints.py b/tests/utils/test_hints.py deleted file mode 100644 index 94a07e264..000000000 --- a/tests/utils/test_hints.py +++ /dev/null @@ -1,167 +0,0 @@ -from __future__ import annotations - -from unittest.mock import MagicMock, patch - -from hud.utils.hints import ( - CLIENT_NOT_INITIALIZED, - ENV_VAR_MISSING, - HUD_API_KEY_MISSING, - INVALID_CONFIG, - MCP_SERVER_ERROR, - RATE_LIMIT_HIT, - TOOL_NOT_FOUND, - Hint, - render_hints, -) - - -def test_hint_objects_basic(): - assert HUD_API_KEY_MISSING.title and isinstance(HUD_API_KEY_MISSING.tips, list) - assert RATE_LIMIT_HIT.code == "RATE_LIMIT" - assert TOOL_NOT_FOUND.title.startswith("Tool") - assert CLIENT_NOT_INITIALIZED.message - assert ENV_VAR_MISSING.command_examples is not None - - -def test_all_hint_constants(): - """Test that all predefined hint constants have required fields.""" - hints = [ - HUD_API_KEY_MISSING, - RATE_LIMIT_HIT, - TOOL_NOT_FOUND, - CLIENT_NOT_INITIALIZED, - INVALID_CONFIG, - ENV_VAR_MISSING, - MCP_SERVER_ERROR, - ] - - for hint in hints: - assert hint.title - assert hint.message - assert hint.code - - -def test_hint_creation(): - """Test creating a custom Hint.""" - hint = Hint( - title="Test Hint", - message="This is a test", - tips=["Tip 1", "Tip 2"], - docs_url="https://example.com", - command_examples=["command 1"], - code="TEST_CODE", - context=["test", "custom"], - ) - - assert hint.title == "Test Hint" - assert hint.message == "This is a test" - assert hint.tips and len(hint.tips) == 2 - assert hint.docs_url == "https://example.com" - assert hint.command_examples and len(hint.command_examples) == 1 - assert hint.code == "TEST_CODE" - assert hint.context and "test" in hint.context - - -def test_hint_minimal(): - """Test creating a minimal Hint with only required fields.""" - hint = Hint(title="Minimal", message="Just basics") - - assert hint.title == "Minimal" - assert hint.message == "Just basics" - assert hint.tips is None - assert hint.docs_url is None - assert hint.command_examples is None - assert hint.code is None - assert hint.context is None - - -def test_render_hints_none(): - """Test that render_hints handles None gracefully.""" - # Should not raise - render_hints(None) - - -def test_render_hints_empty_list(): - """Test that render_hints handles empty list gracefully.""" - # Should not raise - render_hints([]) - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_with_tips(mock_console): - """Test rendering hints with tips.""" - render_hints([HUD_API_KEY_MISSING]) - - # Should call warning for title/message - mock_console.warning.assert_called() - # Should call info for tips - assert mock_console.info.call_count >= 1 - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_with_command_examples(mock_console): - """Test rendering hints with command examples.""" - render_hints([ENV_VAR_MISSING]) - - # Should call command_example - mock_console.command_example.assert_called() - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_with_docs_url(mock_console): - """Test rendering hints with documentation URL.""" - hint = Hint( - title="Test", - message="Test message", - docs_url="https://docs.example.com", - ) - - render_hints([hint]) - - # Should call link for docs URL - mock_console.link.assert_called_with("https://docs.example.com") - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_same_title_and_message(mock_console): - """Test rendering hints when title equals message.""" - hint = Hint(title="Same", message="Same") - - render_hints([hint]) - - # Should only call warning once with just the message - mock_console.warning.assert_called_once_with("Same") - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_different_title_and_message(mock_console): - """Test rendering hints when title differs from message.""" - hint = Hint(title="Title", message="Different message") - - render_hints([hint]) - - # Should call warning with both title and message - mock_console.warning.assert_called_once() - call_args = mock_console.warning.call_args[0][0] - assert "Title" in call_args - assert "Different message" in call_args - - -def test_render_hints_with_custom_design(): - """Test rendering hints with custom design object.""" - custom_design = MagicMock() - - hint = Hint(title="Test", message="Message") - # Should not raise when custom design is provided - render_hints([hint], design=custom_design) - - -@patch("hud.utils.hud_console.hud_console") -def test_render_hints_handles_exception(mock_console): - """Test that render_hints handles exceptions gracefully.""" - mock_console.warning.side_effect = Exception("Test error") - - hint = Hint(title="Test", message="Message") - - # Should not raise, just log warning - render_hints([hint]) diff --git a/tests/utils/test_hud_console.py b/tests/utils/test_hud_console.py index b9ea2dcc4..7508a5d31 100644 --- a/tests/utils/test_hud_console.py +++ b/tests/utils/test_hud_console.py @@ -47,7 +47,6 @@ def test_output_methods_do_not_raise() -> None: c.status_item("label", "value") c.command_example("hud eval tasks.json") c.key_value_table({"key": "value"}) - c.render_support_hint() def test_debug_respects_logger_level() -> None: @@ -66,21 +65,6 @@ def test_format_helpers_return_strings() -> None: assert isinstance(c.format_tool_result("error text", is_error=True), str) -def test_render_exception_does_not_raise() -> None: - c = HUDConsole() - try: - raise ValueError("boom") - except ValueError as exc: - c.render_exception(exc) - - -def test_render_exception_request_error_details() -> None: - from hud.utils.exceptions import HudRequestError - - c = HUDConsole() - c.render_exception(HudRequestError("nope", status_code=403, response_text="forbidden")) - - def _capture_select_choices(monkeypatch: pytest.MonkeyPatch, returns: str) -> dict[str, Any]: """Patch the ``questionary`` boundary so ``HUDConsole.select`` runs without a TTY, recording the choice list it builds.""" diff --git a/tests/utils/test_requests.py b/tests/utils/test_requests.py index 08b95fb4e..49c32c009 100644 --- a/tests/utils/test_requests.py +++ b/tests/utils/test_requests.py @@ -54,22 +54,6 @@ def handle(request): assert calls == 3 -@pytest.mark.parametrize("asynchronous", [False, True]) -@pytest.mark.parametrize("status", [402, 429]) -async def test_requests_preserve_status_hints_without_retry(asynchronous, status): - from hud.utils.hints import CREDITS_EXHAUSTED, RATE_LIMIT_HIT - - transport = httpx.MockTransport(lambda request: httpx.Response(status, json={"detail": "stop"})) - with pytest.raises(HudRequestError) as error: - if asynchronous: - async with httpx.AsyncClient(transport=transport) as client: - await make_request("GET", "https://test/data", api_key="key", client=client) - else: - with httpx.Client(transport=transport) as client: - make_request_sync("GET", "https://test/data", api_key="key", client=client) - assert error.value.hints == [CREDITS_EXHAUSTED if status == 402 else RATE_LIMIT_HIT] - - def test_requests_warn_once_per_deprecation_notice(): successor = f"https://api.test/v2/{uuid.uuid4()}" headers = { From 7255a614beaa14d7c61f34c267342b63ff413d38 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 015/248] test: pin each package's public names and the CLI identity One snapshot of every package's __all__ replaces the hand-kept export lists in test_init.py and test_init_module.py; each name must resolve and none may be private. hud version --json and hud --help run in a subprocess, the way users see them. --- tests/test_init.py | 46 ------ tests/test_init_module.py | 53 ------- tests/test_public_surface.py | 266 +++++++++++++++++++++++++++++++++++ tests/test_version.py | 10 -- 4 files changed, 266 insertions(+), 109 deletions(-) delete mode 100644 tests/test_init.py delete mode 100644 tests/test_init_module.py create mode 100644 tests/test_public_surface.py delete mode 100644 tests/test_version.py diff --git a/tests/test_init.py b/tests/test_init.py deleted file mode 100644 index 67ac56186..000000000 --- a/tests/test_init.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Tests for hud.__init__ module.""" - -from __future__ import annotations - - -class TestHudInit: - """Tests for the hud package initialization.""" - - def test_version_import_success(self): - """Test that version is imported successfully.""" - import hud - - # Version should be available - assert hasattr(hud, "__version__") - assert isinstance(hud.__version__, str) - - def test_all_exports_available(self): - """Test that all exported functions are available.""" - import hud - - expected_exports = [ - "Chat", - "DockerRuntime", - "Environment", - "Grade", - "Job", - "HUDRuntime", - "HostedRuntime", - "Run", - "Runtime", - "RuntimeConfig", - "RuntimeGPU", - "RuntimeLimits", - "RuntimeResources", - "RuntimeTPU", - "LocalRuntime", - "SubprocessRuntime", - "SyncPlan", - "Task", - "Taskset", - "connect", - "instrument", - ] - - for export in expected_exports: - assert hasattr(hud, export), f"Missing export: {export}" diff --git a/tests/test_init_module.py b/tests/test_init_module.py deleted file mode 100644 index 83437c458..000000000 --- a/tests/test_init_module.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Tests for hud/__init__.py module initialization.""" - -from __future__ import annotations - - -class TestInitModule: - """Test the hud module initialization.""" - - def test_version_exposed(self): - """Test that the package exposes its version.""" - import hud - - assert hasattr(hud, "__version__") - assert isinstance(hud.__version__, str) - - def test_all_exports(self): - """Test that __all__ contains expected exports.""" - import hud - - expected = [ - "Chat", - "ComposeProject", - "Connection", - "DockerRuntime", - "Environment", - "Grade", - "Job", - "HUDRuntime", - "HostedRuntime", - "Run", - "Runtime", - "RuntimeConfig", - "RuntimeGPU", - "RuntimeLimits", - "RuntimeResources", - "RuntimeTPU", - "LocalRuntime", - "SubprocessRuntime", - "SyncPlan", - "Task", - "Taskset", - "Trace", - "TrainingClient", - "__version__", - "connect", - "instrument", - ] - - assert set(hud.__all__) == set(expected) - - # Verify all exported items are actually available - for item in hud.__all__: - assert hasattr(hud, item) diff --git a/tests/test_public_surface.py b/tests/test_public_surface.py new file mode 100644 index 000000000..398bbeaf9 --- /dev/null +++ b/tests/test_public_surface.py @@ -0,0 +1,266 @@ +"""The names each package exports, and the CLI's identity.""" + +from __future__ import annotations + +import importlib +from importlib.metadata import version +from typing import TYPE_CHECKING + +import pytest +from inline_snapshot import snapshot + +from hud import __version__ + +if TYPE_CHECKING: + from tests.harness import Hud + +PACKAGES = [ + "hud", + "hud.environment", + "hud.environment.robot", + "hud.eval", + "hud.eval.runtime", + "hud.agents", + "hud.agents.robot", + "hud.agents.tools", + "hud.capabilities", + "hud.clients", + "hud.graders", + "hud.integrations.harbor", + "hud.telemetry", + "hud.train", + "hud.utils", +] + + +def test_each_package_exports_the_documented_names() -> None: + exports = {name: sorted(importlib.import_module(name).__all__) for name in PACKAGES} + + assert exports == snapshot( + { + "hud": [ + "Chat", + "ComposeProject", + "Connection", + "DockerRuntime", + "Environment", + "Grade", + "HUDRuntime", + "HostedRuntime", + "Job", + "LocalRuntime", + "Run", + "Runtime", + "RuntimeConfig", + "RuntimeGPU", + "RuntimeLimits", + "RuntimeResources", + "RuntimeTPU", + "SubprocessRuntime", + "SyncPlan", + "Task", + "Taskset", + "Trace", + "TrainingClient", + "__version__", + "connect", + "instrument", + ], + "hud.environment": [ + "Answer", + "Capability", + "DEFAULT_SYSTEM_MOUNTS", + "DataFileArg", + "DataFileRef", + "DataFilesArg", + "Environment", + "GradingArg", + "Mount", + "MountKind", + "Peer", + "PromptArg", + "Workspace", + "WorkspaceRoute", + "load_environment", + ], + "hud.environment.robot": [ + "DirectControl", + "GymBridge", + "RobotBridge", + "RobotEndpoint", + "TracedEnv", + "serve_bridge", + "wrap", + ], + "hud.eval": [ + "Chat", + "ComposeProject", + "DaytonaRuntime", + "DockerRuntime", + "Grade", + "HUDRuntime", + "HostedRuntime", + "Job", + "LocalRuntime", + "ModalRuntime", + "Provider", + "Run", + "Runtime", + "RuntimeConfig", + "RuntimeGPU", + "RuntimeLimits", + "RuntimeResources", + "RuntimeTPU", + "Shared", + "SubprocessRuntime", + "SyncPlan", + "Task", + "Taskset", + "Trace", + "rollout", + ], + "hud.eval.runtime": [ + "ComposeProject", + "DaytonaRuntime", + "DockerBindMount", + "DockerRuntime", + "HUDRuntime", + "HostedRuntime", + "LocalRuntime", + "ModalRuntime", + "Provider", + "Runtime", + "RuntimeConfig", + "RuntimeGPU", + "RuntimeLimits", + "RuntimeResources", + "RuntimeTPU", + "Shared", + "SubprocessRuntime", + ], + "hud.agents": [ + "ClaudeAgent", + "ClaudeCLIAgent", + "ClaudeCLIConfig", + "CodexCLIAgent", + "CodexCLIConfig", + "GeminiAgent", + "MCPAgent", + "OpenAIAgent", + "OpenAIChatAgent", + "create_agent", + ], + "hud.agents.robot": [ + "Adapter", + "BatchedAgent", + "BatchedModel", + "DatasetWriter", + "LeRobotAdapter", + "LeRobotModel", + "Model", + "OpenPIAdapter", + "ROBOT_PROTOCOL", + "RemoteModel", + "RobotAgent", + ], + "hud.agents.tools": [ + "AgentTool", + "AgentToolSpec", + "ClientT", + "HostedTool", + "MCPTool", + "RFBTool", + "SSHTool", + "result_text", + "tool_err", + "tool_ok", + ], + "hud.capabilities": [ + "CDPClient", + "Capability", + "CapabilityClient", + "Connection", + "MCPClient", + "RFBClient", + "RobotClient", + "SSHClient", + ], + "hud.clients": ["HudClient", "HudProtocolError", "Manifest", "ServerInfo", "connect"], + "hud.graders": [ + "BashGrader", + "EvaluationResult", + "Grader", + "LLMJudgeGrader", + "SubScore", + "combine", + "combine_all", + "combine_any", + "contains", + "contains_all", + "contains_any", + "exact_match", + "f1_score", + "normalize", + "numeric_match", + ], + "hud.integrations.harbor": [ + "AdaptFailure", + "AdaptFinding", + "AdaptResult", + "adapt", + "export", + ], + "hud.telemetry": ["flush", "instrument", "queue_span"], + "hud.train": [ + "BackwardRequest", + "BaseTrainingClient", + "BuiltinLoss", + "CheckpointResponse", + "DatumTensors", + "ForwardBackwardRequest", + "ForwardBackwardResult", + "ForwardRequest", + "ForwardResult", + "LossFn", + "OptimStepRequest", + "OptimStepResult", + "TrainInput", + "TrainingClient", + "TrainingDatum", + "TrajectoryPayload", + "TrajectorySample", + ], + "hud.utils": [ + "HUDConsole", + "PlatformClient", + "hud_console", + "make_request", + "make_request_sync", + ], + } + ) + + +@pytest.mark.parametrize("package", PACKAGES) +def test_every_exported_name_resolves_and_none_is_private(package: str) -> None: + module = importlib.import_module(package) + + assert [name for name in module.__all__ if not hasattr(module, name)] == [] + assert [name for name in module.__all__ if name.startswith("_") and name != "__version__"] == [] + + +def test_the_cli_reports_the_installed_version(hud: Hud) -> None: + result = hud("version", "--json") + + assert result.exit_code == 0, result + assert result.json == {"name": "hud", "version": __version__} + assert __version__ == version("hud") + + +def test_help_lists_the_commands_without_writing_to_stderr(hud: Hud) -> None: + result = hud("--help") + + assert result.exit_code == 0, result + assert result.stderr == "" + for command in ("init", "serve", "deploy", "eval", "task", "sync", "jobs", "trace", "models"): + assert f" {command} " in result.stdout diff --git a/tests/test_version.py b/tests/test_version.py deleted file mode 100644 index c569e4d2a..000000000 --- a/tests/test_version.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import annotations - -from importlib.metadata import version - - -def test_import(): - """Test that the package can be imported.""" - import hud - - assert hud.__version__ == version("hud") From 4e03010c6c6f1679b4a5d06abd0a9b3de8d329c0 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 016/248] test: drive telemetry uploads through real rollouts Rollouts with telemetry on run against the fake telemetry service. The scenario pins the upload path, bearer, body shape, per-run trace grouping, step order and status, the 100-span/4 MiB batch limits, the 503 retry, the no-key and uploads-off cases, flush's deadline, the exit flush, and that a dead service leaves the run and Taskset.run intact. --- tests/telemetry/test_upload.py | 291 +++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 tests/telemetry/test_upload.py diff --git a/tests/telemetry/test_upload.py b/tests/telemetry/test_upload.py new file mode 100644 index 000000000..c9205b4d8 --- /dev/null +++ b/tests/telemetry/test_upload.py @@ -0,0 +1,291 @@ +"""Rollout spans reach the HUD telemetry service, and its failures never reach the run.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import textwrap +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest +from inline_snapshot import snapshot + +from hud import Environment, instrument +from hud.agents.openai_compatible import OpenAIChatAgent +from hud.agents.types import OpenAIChatConfig +from hud.eval import LocalRuntime, Task, Taskset, rollout +from hud.telemetry import flush +from tests.harness import Reply, ScriptedAgent, call, say + +if TYPE_CHECKING: + from hud.eval import Run + from tests.harness import FakeServices, HudEnv, Models + +UPLOAD = "/trace/{id}/telemetry-upload" +ROOT = Path(__file__).parents[2] + + +def _env() -> Environment: + env = Environment("upload") + + @env.template() + async def answer(broken: bool = False): + reply = yield "Reply with ok." + if broken: + raise RuntimeError("grader broke") + yield 1.0 if reply == "ok" else 0.0 + + return env + + +def _platform(services: FakeServices, *upload_replies: Reply) -> None: + for path in ("/v2/trace/job/{id}/enter", "/v2/trace/{id}/enter", "/v2/trace/{id}/exit"): + services.route("api", "POST", path, json={}) + services.route("telemetry", "POST", UPLOAD, *(upload_replies or (Reply(json={}),))) + + +def _uploaded(services: FakeServices) -> dict[str, list[dict[str, Any]]]: + """Every uploaded span, grouped by the trace id in the upload path.""" + by_trace: dict[str, list[dict[str, Any]]] = {} + for request in services.requests("telemetry", "POST", UPLOAD): + by_trace.setdefault(request.params["id"], []).extend(request.json["telemetry"]) + return by_trace + + +def _step_rows(spans: list[dict[str, Any]]) -> list[tuple[str, str, int]]: + steps = [span for span in spans if span["attributes"].get("hud.schema") == "hud.step.v1"] + return [ + (span["name"], span["status_code"], span["attributes"]["hud.payload"]["step_id"]) + for span in sorted(steps, key=lambda span: span["attributes"]["hud.payload"]["step_id"]) + ] + + +@pytest.mark.parametrize( + ("broken", "reward", "rows"), + [ + pytest.param( + False, + 1.0, + snapshot( + [ + ("step.task", "OK", 1), + ("step.user", "OK", 2), + ("step.agent", "OK", 3), + ("step.tool", "OK", 4), + ("step.agent", "OK", 5), + ("step.task", "OK", 6), + ] + ), + id="graded", + ), + pytest.param( + True, + 0.0, + snapshot( + [ + ("step.task", "OK", 1), + ("step.user", "OK", 2), + ("step.agent", "OK", 3), + ("step.tool", "OK", 4), + ("step.agent", "OK", 5), + ("step.system", "ERROR", 6), + ] + ), + id="grading-error", + ), + ], +) +async def test_every_rollout_step_is_uploaded_to_its_own_trace( + services: FakeServices, + models: Models, + hud_env: HudEnv, + broken: bool, + reward: float, + rows: list[tuple[str, str, int]], +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + _platform(services) + models.script([call("missing_tool", x=1), say("ok")]) + agent = OpenAIChatAgent(OpenAIChatConfig(model="scripted", max_steps=4)) + task = Task(env="upload", id="answer", args={"broken": broken}) + + job = await Taskset("t", [task]).run(agent, runtime=LocalRuntime(_env()), group=2) + + assert [run.reward for run in job.runs] == [reward, reward] + uploaded = _uploaded(services) + assert sorted(uploaded) == sorted(str(run.trace_id).replace("-", "") for run in job.runs) + for request in services.requests("telemetry", "POST", UPLOAD): + assert request.bearer == "k" + assert set(request.json) == {"telemetry"} + for trace_id, spans in uploaded.items(): + assert {span["attributes"]["hud.task_run_id"].replace("-", "") for span in spans} == { + trace_id + } + assert _step_rows(spans) == rows + + +def _instrumented_agent(calls: int, payload_bytes: int) -> ScriptedAgent: + @instrument(name="probe") + def probe(index: int) -> str: + return "x" * payload_bytes + + def answer(prompt: str) -> str: + del prompt + for index in range(calls): + probe(index) + return "ok" + + return ScriptedAgent(answer) + + +@pytest.mark.parametrize( + ("calls", "payload_bytes", "uploads"), + [ + pytest.param(250, 10, 3, id="100-spans"), + pytest.param(3, 3 * 1024 * 1024, 2, id="4-mib"), + ], +) +async def test_a_batch_closes_at_100_spans_or_4_mib( + services: FakeServices, hud_env: HudEnv, calls: int, payload_bytes: int, uploads: int +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + _platform(services) + + job = await Taskset("t", [Task(env="upload", id="answer")]).run( + _instrumented_agent(calls, payload_bytes), runtime=LocalRuntime(_env()) + ) + + batches = [ + upload.json["telemetry"] for upload in services.requests("telemetry", "POST", UPLOAD) + ] + assert job.reward == 1.0 + assert sum(span["name"] == "probe" for batch in batches for span in batch) == calls + assert len(batches) >= uploads + for batch in batches: + assert len(batch) <= 100 + assert len(json.dumps(batch[:-1])) < 4 * 1024 * 1024 + + +@pytest.mark.parametrize( + "configuration", + [ + pytest.param({"HUD_TELEMETRY_ENABLED": "1"}, id="no-api-key"), + pytest.param({"HUD_API_KEY": "k", "HUD_TELEMETRY_ENABLED": "0"}, id="uploads-disabled"), + ], +) +async def test_nothing_is_uploaded_without_a_key_or_with_uploads_off( + services: FakeServices, hud_env: HudEnv, configuration: dict[str, str] +) -> None: + hud_env.set(**configuration) + _platform(services) + + job = await Taskset("t", [Task(env="upload", id="answer")]).run( + ScriptedAgent("ok"), runtime=LocalRuntime(_env()) + ) + + assert job.reward == 1.0 + assert services.requests("telemetry") == [] + + +async def test_spans_outside_a_run_are_not_uploaded( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + _platform(services) + + @instrument + def outside() -> int: + return 1 + + assert outside() == 1 + assert flush(timeout=10) + assert services.requests("telemetry") == [] + + +async def test_an_upload_answered_503_is_retried_until_delivered( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + _platform(services, Reply(status=503), Reply(status=503), Reply(json={})) + + job = await Taskset("t", [Task(env="upload", id="answer")]).run( + ScriptedAgent("ok"), runtime=LocalRuntime(_env()) + ) + + first, *retries = services.requests("telemetry", "POST", UPLOAD) + assert job.reward == 1.0 + assert [retry.json for retry in retries] == [first.json, first.json] + + +async def test_a_dead_telemetry_service_leaves_the_run_and_the_job_unaffected( + services: FakeServices, hud_env: HudEnv +) -> None: + _platform(services) + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1", HUD_TELEMETRY_URL="http://127.0.0.1:9") + + job = await Taskset("t", [Task(env="upload", id="answer")]).run( + ScriptedAgent("ok"), runtime=LocalRuntime(_env()) + ) + + assert job.reward == 1.0 + assert [run.trace.status for run in job.runs] == ["completed"] + assert flush(timeout=10) + + +async def test_flush_returns_false_at_its_deadline_while_an_upload_hangs( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + _platform(services, Reply(json={}, delay=2.0)) + + run: Run = await rollout( + Task(env="upload", id="answer"), ScriptedAgent("ok"), runtime=LocalRuntime(_env()) + ) + + assert run.reward == 1.0 + assert flush(timeout=0.2) is False + assert flush(timeout=30) is True + assert services.requests("telemetry", "POST", UPLOAD) != [] + + +def test_spans_still_queued_at_interpreter_exit_are_uploaded( + services: FakeServices, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + _platform(services) + script = tmp_path / "rollout_then_exit.py" + script.write_text( + textwrap.dedent( + """ + import asyncio + from hud import Environment + from hud.eval import LocalRuntime, Task, rollout + from tests.harness import ScriptedAgent + + env = Environment("upload") + + @env.template() + async def answer(): + reply = yield "Reply with ok." + yield 1.0 if reply == "ok" else 0.0 + + task = Task(env="upload", id="answer") + run = asyncio.run(rollout(task, ScriptedAgent("ok"), runtime=LocalRuntime(env))) + print(run.trace_id) + """ + ) + ) + + completed = subprocess.run( + [sys.executable, str(script)], + capture_output=True, + text=True, + timeout=120, + check=True, + cwd=ROOT, + ) + + trace_id = completed.stdout.strip().replace("-", "") + assert [name for name in _uploaded(services)] == [trace_id] From 7c5217d578d06701060c387561c3eab27060771a Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 017/248] test: read local span files and instrument spans from real runs Runs with each span-directory configuration assert where their spans land, that an unwritable directory does not fail the run, the step span schema and timing, and what @hud.instrument records under a real trace (self dropped, defaults bound, lists truncated, exceptions, record_args/record_result off, unserializable values). The exporter and instrument tests that patched hud seams or never set a trace context are removed. --- tests/telemetry/test_exporter.py | 182 ----------- tests/telemetry/test_instrument.py | 449 ---------------------------- tests/telemetry/test_local_spans.py | 193 ++++++++++++ 3 files changed, 193 insertions(+), 631 deletions(-) delete mode 100644 tests/telemetry/test_exporter.py delete mode 100644 tests/telemetry/test_instrument.py create mode 100644 tests/telemetry/test_local_spans.py diff --git a/tests/telemetry/test_exporter.py b/tests/telemetry/test_exporter.py deleted file mode 100644 index 5f5566890..000000000 --- a/tests/telemetry/test_exporter.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Tests for telemetry exporter with mock backend.""" - -from __future__ import annotations - -import json -import subprocess -import sys -from typing import Any -from unittest.mock import patch - -import pytest - -from hud.settings import settings -from hud.telemetry.exporter import _do_upload, flush, queue_span - - -@pytest.fixture(autouse=True) -def drain_exporter(): - """Drain the background worker before and after each test.""" - assert flush(timeout=1.0) - yield - assert flush(timeout=1.0) - - -class _RecordingUpload: - """Captures (task_run_id, spans, api_key) for each upload.""" - - def __init__(self) -> None: - self.calls: list[tuple[str, list[dict[str, Any]], str]] = [] - - def __call__( - self, - task_run_id: str, - spans: list[dict[str, Any]], - telemetry_url: str, - api_key: str, - ) -> None: - self.calls.append((task_run_id, spans, api_key)) - - -def _configure( - monkeypatch: pytest.MonkeyPatch, - *, - api_key: str | None = "test-key", - enabled: bool = True, -) -> None: - """Pin the settings fields the exporter reads, on the real object. - - Field patches (not a whole-object MagicMock) keep every other setting - real — including the root conftest's isolation pins, which a mock object - would replace with truthy auto-attributes. - """ - monkeypatch.setattr(settings, "api_key", api_key) - monkeypatch.setattr(settings, "telemetry_enabled", enabled) - monkeypatch.setattr(settings, "hud_telemetry_url", "https://api.hud.ai") - - -class TestDoUpload: - def test_upload_posts_to_trace_endpoint(self): - with patch("hud.telemetry.exporter.make_request_sync") as mock_request: - _do_upload( - task_run_id="test-task-123", - spans=[{"name": "test.span"}], - telemetry_url="https://api.hud.ai", - api_key="test-key", - ) - - mock_request.assert_called_once() - kwargs = mock_request.call_args.kwargs - assert kwargs["method"] == "POST" - assert "test-task-123" in kwargs["url"] - assert kwargs["api_key"] == "test-key" - assert kwargs["json"] == {"telemetry": [{"name": "test.span"}]} - - def test_upload_swallows_request_errors(self): - with patch("hud.telemetry.exporter.make_request_sync", side_effect=Exception("boom")): - _do_upload("test-task-123", [{"name": "test.span"}], "https://api.hud.ai", "test-key") - - -class TestQueueSpan: - @pytest.mark.parametrize( - ("api_key", "enabled", "attributes"), - [ - (None, True, {"hud.task_run_id": "123"}), - ("test-key", False, {"hud.task_run_id": "123"}), - ("test-key", True, {}), - ], - ) - def test_span_is_dropped(self, monkeypatch, api_key, enabled, attributes): - upload = _RecordingUpload() - _configure(monkeypatch, api_key=api_key, enabled=enabled) - with patch("hud.telemetry.exporter._do_upload", side_effect=upload): - queue_span({"name": "test", "attributes": attributes}) - assert flush(timeout=1.0) - - assert upload.calls == [] - - def test_spans_upload_in_one_batch_per_trace(self, monkeypatch): - upload = _RecordingUpload() - _configure(monkeypatch) - with patch("hud.telemetry.exporter._do_upload", side_effect=upload): - queue_span({"name": "span-1", "attributes": {"hud.task_run_id": "task-1"}}) - queue_span({"name": "span-2", "attributes": {"hud.task_run_id": "task-1"}}) - queue_span({"name": "span-3", "attributes": {"hud.task_run_id": "task-2"}}) - assert flush(timeout=1.0) - - by_task = {task_run_id: spans for task_run_id, spans, _ in upload.calls} - assert [span["name"] for span in by_task["task-1"]] == ["span-1", "span-2"] - assert [span["name"] for span in by_task["task-2"]] == ["span-3"] - - def test_upload_uses_settings_api_key(self, monkeypatch): - upload = _RecordingUpload() - _configure(monkeypatch) - with patch("hud.telemetry.exporter._do_upload", side_effect=upload): - queue_span({"name": "test", "attributes": {"hud.task_run_id": "task-1"}}) - assert flush(timeout=1.0) - - assert [api_key for _, _, api_key in upload.calls] == ["test-key"] - - -class TestLocalExport: - def test_span_written_to_local_dir(self, monkeypatch, tmp_path): - _configure(monkeypatch, api_key=None, enabled=False) - monkeypatch.setattr(settings, "telemetry_local_dir", str(tmp_path)) - - queue_span({"name": "s", "trace_id": "t-1", "attributes": {"hud.task_run_id": "task-1"}}) - - lines = (tmp_path / "t-1.jsonl").read_text(encoding="utf-8").splitlines() - assert json.loads(lines[0])["name"] == "s" - - def test_disabled_uploads_write_to_home_spans(self, monkeypatch, tmp_path): - _configure(monkeypatch, api_key=None, enabled=False) - monkeypatch.setattr(settings, "telemetry_local_dir", None) - monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) - - queue_span({"name": "s", "trace_id": "t-1", "attributes": {"hud.task_run_id": "task-1"}}) - - lines = (tmp_path / ".hud" / "spans" / "t-1.jsonl").read_text(encoding="utf-8").splitlines() - assert json.loads(lines[0])["name"] == "s" - - def test_non_str_local_dir_fails_loudly(self, monkeypatch): - _configure(monkeypatch, api_key=None, enabled=False) - monkeypatch.setattr(settings, "telemetry_local_dir", object()) - - with pytest.raises(TypeError, match="telemetry_local_dir"): - queue_span({"name": "s", "attributes": {"hud.task_run_id": "task-1"}}) - - -class TestFlush: - def test_flush_is_noop_when_idle(self): - assert flush(timeout=1.0) - - def test_flush_drains_queued_spans(self, monkeypatch): - upload = _RecordingUpload() - _configure(monkeypatch) - with patch("hud.telemetry.exporter._do_upload", side_effect=upload): - queue_span({"name": "final-span", "attributes": {"hud.task_run_id": "task-1"}}) - assert flush(timeout=1.0) - - assert [span["name"] for _, spans, _ in upload.calls for span in spans] == ["final-span"] - - def test_interpreter_exit_uploads_spans_queued_without_a_flush(self, tmp_path): - uploads = tmp_path / "uploads.jsonl" - script = f""" -import json -from hud.settings import settings -from hud.telemetry import exporter - -def record(**kwargs): - with open({str(uploads)!r}, "a") as out: - out.write(json.dumps(kwargs["json"]) + "\\n") - -settings.api_key = "test-key" -settings.telemetry_enabled = True -settings.telemetry_local_dir = None -exporter.make_request_sync = record -exporter.queue_span({{"name": "last-span", "attributes": {{"hud.task_run_id": "task-1"}}}}) -""" - subprocess.run([sys.executable, "-c", script], check=True, timeout=60) - - uploaded = [json.loads(line) for line in uploads.read_text().splitlines()] - assert [span["name"] for body in uploaded for span in body["telemetry"]] == ["last-span"] diff --git a/tests/telemetry/test_instrument.py b/tests/telemetry/test_instrument.py deleted file mode 100644 index 72737b601..000000000 --- a/tests/telemetry/test_instrument.py +++ /dev/null @@ -1,449 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any -from unittest.mock import patch - -import pytest - -from hud.telemetry.context import set_trace_context -from hud.telemetry.instrument import _serialize_value, instrument -from hud.types import MCPToolResult - - -def test_serialize_value_simple_types(): - """Test _serialize_value with simple types.""" - assert _serialize_value("string") == "string" - assert _serialize_value(42) == 42 - assert _serialize_value(3.14) == 3.14 - assert _serialize_value(True) is True - assert _serialize_value(None) is None - - -def test_serialize_value_list(): - """Test _serialize_value with lists.""" - result = _serialize_value([1, 2, 3]) - assert result == [1, 2, 3] - - -def test_serialize_value_list_truncation(): - """Test _serialize_value truncates long lists.""" - long_list = list(range(20)) - result = _serialize_value(long_list, max_items=5) - assert result == [0, 1, 2, 3, 4] - - -def test_serialize_value_tuple(): - """Test _serialize_value with tuples.""" - result = _serialize_value((1, 2, 3)) - assert result == [1, 2, 3] # Converted to list by JSON - - -def test_serialize_value_tuple_truncation(): - """Test _serialize_value truncates long tuples.""" - long_tuple = tuple(range(20)) - result = _serialize_value(long_tuple, max_items=5) - assert result == [0, 1, 2, 3, 4] - - -def test_serialize_value_dict(): - """Test _serialize_value with dicts.""" - result = _serialize_value({"key": "value"}) - assert result == {"key": "value"} - - -def test_serialize_value_dict_truncation(): - """Test _serialize_value truncates large dicts.""" - large_dict = {f"key{i}": i for i in range(20)} - result = _serialize_value(large_dict, max_items=5) - assert isinstance(result, dict) - assert len(result) == 5 - - -def test_serialize_value_complex_object(): - """Test _serialize_value with custom objects.""" - - @dataclass - class CustomObj: - name: str - value: int - - obj = CustomObj(name="test", value=42) - result = _serialize_value(obj) - assert isinstance(result, dict) - assert result["name"] == "test" - assert result["value"] == 42 - - -def test_serialize_value_fallback(): - """Test _serialize_value fallback for non-serializable objects.""" - - class WeirdObj: - def __init__(self): - raise Exception("Can't access") - - obj = WeirdObj.__new__(WeirdObj) - result = _serialize_value(obj) - # The result is a string representation of the object - assert isinstance(result, str) - assert "WeirdObj" in result - - -def test_serialize_value_pydantic_model_dumps_generically(): - """Domain models serialize through the generic pydantic path (no special-casing).""" - result = _serialize_value(MCPToolResult(content=[], isError=False)) - assert isinstance(result, dict) - assert result["isError"] is False - assert result["content"] == [] - - -@pytest.mark.asyncio -async def test_instrument_async_basic(): - """Test instrument decorator on async function.""" - - @instrument - async def test_func(x: int, y: int) -> int: - return x + y - - result = await test_func(2, 3) - assert result == 5 - - -@pytest.mark.asyncio -async def test_instrument_async_with_params(): - """Test instrument with custom parameters.""" - - @instrument(name="custom_name") - async def test_func(x: int) -> int: - return x * 2 - - result = await test_func(5) - assert result == 10 - - -@pytest.mark.asyncio -async def test_instrument_async_with_exception(): - """Test instrument handles exceptions.""" - - @instrument - async def test_func(): - raise ValueError("Test error") - - with pytest.raises(ValueError, match="Test error"): - await test_func() - - -@pytest.mark.asyncio -async def test_instrument_async_no_record_args(): - """Test instrument with record_args=False.""" - - @instrument(record_args=False) - async def test_func(x: int) -> int: - return x - - result = await test_func(42) - assert result == 42 - - -@pytest.mark.asyncio -async def test_instrument_async_no_record_result(): - """Test instrument with record_result=False.""" - - @instrument(record_result=False) - async def test_func() -> str: - return "test" - - result = await test_func() - assert result == "test" - - -def test_instrument_sync_basic(): - """Test instrument decorator on sync function.""" - - @instrument - def test_func(x: int, y: int) -> int: - return x + y - - result = test_func(2, 3) - assert result == 5 - - -def test_instrument_sync_with_params(): - """Test instrument on sync function with parameters.""" - - @instrument(name="sync_custom") - def test_func(x: int) -> int: - return x * 2 - - result = test_func(5) - assert result == 10 - - -def test_instrument_sync_with_exception(): - """Test instrument handles exceptions in sync functions.""" - - @instrument - def test_func(): - raise ValueError("Sync error") - - with pytest.raises(ValueError, match="Sync error"): - test_func() - - -def test_instrument_sync_no_record_args(): - """Test instrument sync with record_args=False.""" - - @instrument(record_args=False) - def test_func(x: int) -> int: - return x - - result = test_func(42) - assert result == 42 - - -def test_instrument_sync_no_record_result(): - """Test instrument sync with record_result=False.""" - - @instrument(record_result=False) - def test_func() -> str: - return "test" - - result = test_func() - assert result == "test" - - -def test_instrument_already_instrumented(): - """Test that instrumenting already instrumented function is skipped.""" - - @instrument - def test_func(): - return "original" - - # Try to instrument again - test_func2 = instrument(test_func) - - # Should be the same function - assert test_func2 is test_func - - -def test_instrument_marks_as_instrumented(): - """Test that instrument marks functions correctly.""" - - @instrument - def test_func(): - return True - - assert hasattr(test_func, "_hud_instrumented") - assert test_func._hud_instrumented is True - assert hasattr(test_func, "_hud_original") - - -@pytest.mark.asyncio -async def test_instrument_async_complex_result(): - """Test instrument with complex result object.""" - - @instrument - async def test_func() -> dict[str, Any]: - return {"nested": {"data": [1, 2, 3]}, "count": 3} - - result = await test_func() - assert result["count"] == 3 - - -def test_instrument_sync_complex_result(): - """Test instrument sync with complex result.""" - - @dataclass - class Result: - value: int - name: str - - @instrument - def test_func() -> Result: - return Result(value=42, name="test") - - result = test_func() - assert result.value == 42 - - -@pytest.mark.asyncio -async def test_instrument_async_with_self_param(): - """Test instrument properly handles 'self' parameter.""" - - class TestClass: - @instrument - async def method(self, x: int) -> int: - return x * 2 - - obj = TestClass() - result = await obj.method(5) - assert result == 10 - - -def test_instrument_sync_with_cls_param(): - """Test instrument properly handles 'cls' parameter.""" - - class TestClass: - @classmethod - @instrument - def method(cls, x: int) -> int: - return x * 3 - - result = TestClass.method(4) - assert result == 12 - - -@pytest.mark.asyncio -async def test_instrument_async_serialization_error(): - """Test instrument handles serialization errors gracefully.""" - - class UnserializableArg: - def __getattribute__(self, name): - raise Exception("Can't serialize") - - @instrument - async def test_func(arg): - return "success" - - # Should not raise, just skip serialization - result = await test_func(UnserializableArg()) - assert result == "success" - - -def test_instrument_function_without_signature(): - """Test instrument on functions without inspectable signature.""" - # Built-in functions don't have signatures - instrumented_len = instrument(len) - result = instrumented_len([1, 2, 3]) - assert result == 3 - - -@pytest.mark.asyncio -async def test_instrument_async_result_serialization_error(): - """Test instrument handles result serialization errors.""" - - class UnserializableResult: - def __iter__(self): - raise Exception("Can't iterate") - - @instrument - async def test_func(): - return UnserializableResult() - - # Should not raise, just skip result recording - result = await test_func() - assert isinstance(result, UnserializableResult) - - -def test_instrument_without_parentheses(): - """Test using @instrument without parentheses.""" - - @instrument - def test_func(x: int) -> int: - return x + 1 - - assert test_func(5) == 6 - - -def test_instrument_with_parentheses(): - """Test using @instrument() with parentheses.""" - - @instrument() - def test_func(x: int) -> int: - return x + 1 - - assert test_func(5) == 6 - - -@pytest.mark.asyncio -async def test_instrument_emits_debug_span_under_trace_context(): - """Spans carry the namespaced run id, no schema tag, and request/result events.""" - captured: list[dict[str, Any]] = [] - - @instrument(name="diag") - async def fn(x: int) -> str: - return "ok" - - with ( - patch("hud.telemetry.instrument.queue_span", side_effect=captured.append), - set_trace_context("run-123"), - ): - assert await fn(2) == "ok" - - (span,) = captured - assert span["name"] == "diag" - assert span["attributes"]["hud.task_run_id"] == "run-123" - assert "hud.schema" not in span["attributes"] - assert span["status_code"] == "OK" - assert [event["name"] for event in span["events"]] == ["hud.request", "hud.result"] - assert span["events"][0]["attributes"]["hud.payload"] == {"x": 2} - - -@pytest.mark.asyncio -async def test_instrument_records_exception_event(): - captured: list[dict[str, Any]] = [] - - @instrument(name="boom") - async def fn() -> None: - raise ValueError("bad") - - with ( - patch("hud.telemetry.instrument.queue_span", side_effect=captured.append), - set_trace_context("run-123"), - pytest.raises(ValueError, match="bad"), - ): - await fn() - - (span,) = captured - assert span["status_code"] == "ERROR" - assert span["status_message"] == "ValueError: bad" - assert span["events"][-1]["name"] == "exception" - - -@pytest.mark.asyncio -async def test_instrument_emits_nothing_without_trace_context(): - captured: list[dict[str, Any]] = [] - - @instrument - async def fn() -> int: - return 1 - - with patch("hud.telemetry.instrument.queue_span", side_effect=captured.append): - assert await fn() == 1 - - assert captured == [] - - -@pytest.mark.asyncio -async def test_instrument_async_with_defaults(): - """Test instrument with function that has default arguments.""" - - @instrument - async def test_func(x: int, y: int = 10) -> int: - return x + y - - assert await test_func(5) == 15 - assert await test_func(5, 20) == 25 - - -def test_instrument_sync_with_kwargs(): - """Test instrument with keyword arguments.""" - - @instrument - def test_func(x: int, **kwargs: Any) -> dict[str, Any]: - return {"x": x, **kwargs} - - result = test_func(1, a=2, b=3) - assert result == {"x": 1, "a": 2, "b": 3} - - -@pytest.mark.asyncio -async def test_instrument_async_with_varargs(): - """Test instrument with *args.""" - - @instrument - async def test_func(*args) -> int: - return sum(args) - - result = await test_func(1, 2, 3, 4) - assert result == 10 diff --git a/tests/telemetry/test_local_spans.py b/tests/telemetry/test_local_spans.py new file mode 100644 index 000000000..84675546b --- /dev/null +++ b/tests/telemetry/test_local_spans.py @@ -0,0 +1,193 @@ +"""Spans a run writes to local files, including ``@hud.instrument`` spans.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest +from dirty_equals import IsStr +from inline_snapshot import snapshot + +from hud import Environment, instrument +from hud.agents.base import Agent +from hud.eval import LocalRuntime, Task, Taskset, rollout +from tests.harness import ScriptedAgent, spans, steps + +if TYPE_CHECKING: + from pathlib import Path + + from hud.eval import Run + from tests.harness import FakeServices, HudEnv + +ISO = IsStr(regex=r"\d{4}-\d\d-\d\dT.*") + + +def _env() -> Environment: + env = Environment("spans") + + @env.template() + async def answer(): + reply = yield "Reply with ok." + yield 1.0 if reply == "ok" else 0.0 + + return env + + +TASK = Task(env="spans", id="answer") + + +@pytest.mark.parametrize( + ("uploads", "local_dir", "written_to", "uploaded"), + [ + pytest.param("0", None, "home", False, id="uploads-off"), + pytest.param("0", "spans", "spans", False, id="uploads-off-with-dir"), + pytest.param("1", "spans", "spans", True, id="uploads-on-with-dir"), + pytest.param("1", None, None, True, id="uploads-on"), + ], +) +async def test_spans_are_written_where_the_configuration_says( + services: FakeServices, + hud_env: HudEnv, + tmp_path: Path, + uploads: str, + local_dir: str | None, + written_to: str | None, + uploaded: bool, +) -> None: + services.route("telemetry", "POST", "/trace/{id}/telemetry-upload", json={}) + hud_env.set( + HUD_API_KEY="k", + HUD_TELEMETRY_ENABLED=uploads, + HUD_TELEMETRY_LOCAL_DIR=str(tmp_path / local_dir) if local_dir else None, + ) + directories = {"home": hud_env.home / ".hud" / "spans", "spans": tmp_path / "spans"} + + job = await Taskset("t", [TASK]).run(ScriptedAgent("ok"), runtime=LocalRuntime(_env())) + + (run,) = job.runs + written = {name: spans(run.trace_id, path) != [] for name, path in directories.items()} + assert written == {name: name == written_to for name in directories} + assert (services.requests("telemetry") != []) is uploaded + + +async def test_an_unwritable_span_directory_does_not_fail_the_run( + hud_env: HudEnv, tmp_path: Path +) -> None: + blocker = tmp_path / "not-a-directory" + blocker.write_text("") + hud_env.set(HUD_TELEMETRY_LOCAL_DIR=str(blocker / "spans")) + + run = await rollout(TASK, ScriptedAgent("ok"), runtime=LocalRuntime(_env())) + + assert run.reward == 1.0 + assert run.trace.status == "completed" + + +async def test_step_spans_carry_the_step_schema_payload_and_timing() -> None: + run = await rollout(TASK, ScriptedAgent("ok"), runtime=LocalRuntime(_env())) + + exported = [span for span in spans(run.trace_id) if span["name"].startswith("step.")] + assert [ + (span["name"], span["attributes"]["hud.schema"], span["status_code"]) for span in exported + ] == snapshot( + [ + ("step.task", "hud.step.v1", "OK"), + ("step.user", "hud.step.v1", "OK"), + ("step.task", "hud.step.v1", "OK"), + ] + ) + assert {span["attributes"]["hud.task_run_id"] for span in exported} == {run.trace_id} + assert [step["step_id"] for step in steps(run.trace_id)] == [1, 2, 3] + assert all(span["end_time"] == ISO for span in exported) + assert steps(run.trace_id)[1]["messages"][0]["content"]["text"] == "Reply with ok." + + +class Probe: + """An agent-side helper whose methods are instrumented.""" + + @instrument(name="probe.lookup") + async def lookup(self, items: list[int], limit: int = 3) -> dict[str, Any]: + return {"found": items[:limit]} + + @instrument(name="probe.quiet", record_args=False, record_result=False) + def quiet(self, secret: str) -> str: + return secret + + @instrument(name="probe.fail") + def fail(self, reason: str) -> None: + raise ValueError(reason) + + @instrument(name="probe.opaque") + def opaque(self, value: object) -> object: + return value + + +class Unprintable: + def __repr__(self) -> str: + raise RuntimeError("no repr") + + +class ProbingAgent(Agent): + """Calls each instrumented helper once, then answers.""" + + async def __call__(self, run: Run) -> None: + probe = Probe() + await probe.lookup(list(range(20))) + probe.quiet("hunter2") + with pytest.raises(ValueError, match="bad input"): + probe.fail("bad input") + probe.opaque(Unprintable()) + run.trace.content = "ok" + + +async def test_instrumented_calls_inside_a_run_record_their_arguments_and_outcome() -> None: + run = await rollout(TASK, ProbingAgent(), runtime=LocalRuntime(_env())) + + assert run.reward == 1.0 + probes = { + span["name"]: { + "status": span["status_code"], + "message": span["status_message"], + "events": {event["name"]: event["attributes"] for event in span["events"]}, + } + for span in spans(run.trace_id) + if span["name"].startswith("probe.") + } + assert probes == snapshot( + { + "probe.lookup": { + "status": "OK", + "message": None, + "events": { + "hud.request": { + "hud.payload": {"items": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], "limit": 3} + }, + "hud.result": {"hud.payload": {"found": [0, 1, 2]}}, + }, + }, + "probe.quiet": {"status": "OK", "message": None, "events": {}}, + "probe.fail": { + "status": "ERROR", + "message": "ValueError: bad input", + "events": { + "hud.request": {"hud.payload": {"reason": "bad input"}}, + "exception": {"exception.message": "ValueError: bad input"}, + }, + }, + "probe.opaque": { + "status": "OK", + "message": None, + "events": { + "hud.request": {"hud.payload": {"value": ""}}, + "hud.result": {"hud.payload": ""}, + }, + }, + } + ) + + +async def test_instrumented_calls_outside_a_run_export_nothing(hud_env: HudEnv) -> None: + probe = Probe() + + assert await probe.lookup([1, 2]) == {"found": [1, 2]} + assert not (hud_env.home / ".hud" / "spans").exists() From f82c3ced349b1eaeb2d2815355ffc0452e315aca Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 018/248] test: query Trace on the traces real rollouts record Trace's final, collect, error, numbering and reload rows now run on a rollout's own trace instead of hand-built steps; the span emission rows that patched hud.types.queue_span moved to the telemetry scenarios. --- tests/test_trace.py | 181 +++++++++++++++----------------------------- 1 file changed, 59 insertions(+), 122 deletions(-) diff --git a/tests/test_trace.py b/tests/test_trace.py index 666999232..d802ef50a 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -1,131 +1,68 @@ -"""Core trajectory contract tests: ``Trace`` invariants + step span emission.""" +"""What a run's ``Trace`` answers about the steps it recorded.""" from __future__ import annotations -from typing import Any -from unittest.mock import patch - -from mcp import types as mcp_types - -from hud.telemetry.context import set_trace_context -from hud.types import Step, Trace - - -def test_trace_final_returns_newest_non_none_answer(): - """final() asks newest-first; None means "no answer", falsy answers win.""" - trace = Trace() - trace.record(Step(source="agent", extra={"note": "first"})) - trace.record(Step(source="agent", extra={"note": ""})) - trace.record(Step(source="tool")) - - assert trace.final(lambda s: s.extra.get("note")) == "" - assert trace.final(lambda s: s.error) is None - - -def test_trace_collect_gathers_answers_in_step_order(): - """collect() keeps step order and skips steps that answer None.""" - trace = Trace() - trace.record(Step(source="agent", extra={"n": 1})) - trace.record(Step(source="tool")) - trace.record(Step(source="agent", extra={"n": 2})) - - assert trace.collect(lambda s: s.extra.get("n")) == [1, 2] - - -def test_trace_append_numbers_steps(): - """Trace.append assigns sequential 1-based step ids.""" - trace = Trace() - trace.record(Step(source="user")) - trace.record(Step(source="agent")) - assert len(trace) == 2 - assert [step.step_id for step in trace.steps] == [1, 2] - - -def test_trace_validator_numbers_preloaded_steps(): - """Steps passed to the constructor are renumbered on validation.""" - trace = Trace(steps=[Step(source="user"), Step(source="agent"), Step(source="tool")]) +import pytest + +from hud import Environment, Trace +from hud.eval import LocalRuntime, Task, rollout +from tests.harness import ScriptedAgent + + +def _env() -> Environment: + env = Environment("trace") + + @env.template() + async def answer(broken: bool = False): + reply = yield "Reply with ok." + if broken: + raise RuntimeError("grader broke") + yield 1.0 if reply == "ok" else 0.0 + + return env + + +@pytest.mark.parametrize( + ("broken", "sources", "status", "error"), + [ + pytest.param(False, ["task", "user", "task"], "completed", None, id="graded"), + pytest.param( + True, + ["task", "user", "system"], + "error", + "[grading] hud.clients.client.HudProtocolError: hud rpc error -32000: grader broke", + id="broken", + ), + ], +) +async def test_a_runs_trace_answers_from_its_recorded_steps( + broken: bool, sources: list[str], status: str, error: str | None +) -> None: + task = Task(env="trace", id="answer", args={"broken": broken}) + + run = await rollout(task, ScriptedAgent("ok"), runtime=LocalRuntime(_env())) + + trace = run.trace + assert len(trace) == len(sources) + assert trace.collect(lambda step: step.source) == sources assert [step.step_id for step in trace.steps] == [1, 2, 3] + assert trace.final(lambda step: step.source) == sources[-1] + assert trace.final(lambda step: step.extra.get("missing")) is None + assert trace.status == status + assert trace.is_error is (status == "error") + assert trace.error == error + assert all(step.ended_at is not None for step in trace.steps) -def test_trace_error_surfaces_last_step_error(): - """Trace.error reads the most recent step error; is_error follows status.""" - trace = Trace() - assert trace.error is None - assert trace.is_error is False - - trace.record(Step(source="tool", error="first")) - trace.record(Step(source="agent")) - trace.record(Step(source="system", error="second")) - trace.status = "error" - - assert trace.error == "second" - assert trace.is_error is True - - -def test_step_emit_wraps_step_in_schema_tagged_span(): - captured: list[dict[str, Any]] = [] - step = Step( - source="user", - messages=[ - mcp_types.PromptMessage( - role="user", - content=mcp_types.TextContent(type="text", text="do the thing"), - ), - ], +async def test_a_dumped_trace_reloads_with_its_steps_renumbered() -> None: + run = await rollout( + Task(env="trace", id="answer"), ScriptedAgent("ok"), runtime=LocalRuntime(_env()) ) + dumped = run.trace.model_dump(mode="json") + for step in dumped["steps"]: + step.pop("step_id") - with ( - patch("hud.types.queue_span", side_effect=captured.append), - set_trace_context("run-1"), - ): - step.emit() - - (span,) = captured - assert span["name"] == "step.user" - assert span["attributes"]["hud.schema"] == "hud.step.v1" - assert span["attributes"]["hud.task_run_id"] == "run-1" - payload = span["attributes"]["hud.payload"] - assert payload["source"] == "user" - assert payload["messages"][0]["content"]["text"] == "do the thing" - assert span["status_code"] == "OK" - - -def test_step_emit_marks_error_status(): - captured: list[dict[str, Any]] = [] - - with ( - patch("hud.types.queue_span", side_effect=captured.append), - set_trace_context("run-1"), - ): - Step(source="system", error="boom").emit() - - (span,) = captured - assert span["status_code"] == "ERROR" - assert span["status_message"] == "boom" - - -def test_step_emit_without_context_is_noop(): - captured: list[dict[str, Any]] = [] - - with patch("hud.types.queue_span", side_effect=captured.append): - Step(source="system", error="boom").emit() - - assert captured == [] - - -def test_trace_record_emits_and_stamps_end(): - """record = number + stamp end + append + emit, in one call.""" - captured: list[dict[str, Any]] = [] - trace = Trace() - - with ( - patch("hud.types.queue_span", side_effect=captured.append), - set_trace_context("run-1"), - ): - trace.record(Step(source="user")) - trace.record(Step(source="agent", ended_at="2026-05-14T20:00:05Z")) + reloaded = Trace.model_validate(dumped) - assert [span["attributes"]["hud.payload"]["step_id"] for span in captured] == [1, 2] - assert trace.steps[0].ended_at is not None # stamped at record time - assert trace.steps[1].ended_at == "2026-05-14T20:00:05Z" # explicit timing kept - assert captured[1]["end_time"] == "2026-05-14T20:00:05Z" + assert [step.step_id for step in reloaded.steps] == [1, 2, 3] + assert reloaded.content == "ok" From ddadd86f6b8e43dc60cf3c3b856d22caa190a257 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 019/248] test: delete formatting-trivia, private-helper and duplicated tests test_types.py pinned str/rich formatting trivia, pydantic defaults on MCPToolCall.annotation and a private summary helper; test_serialization pinned a helper the instrument and grader scenarios exercise; the CLI init area owns environment-name normalization. --- tests/test_types.py | 164 ------------------------------ tests/utils/test_naming.py | 13 --- tests/utils/test_serialization.py | 31 ------ 3 files changed, 208 deletions(-) delete mode 100644 tests/test_types.py delete mode 100644 tests/utils/test_naming.py delete mode 100644 tests/utils/test_serialization.py diff --git a/tests/test_types.py b/tests/test_types.py deleted file mode 100644 index 00107901d..000000000 --- a/tests/test_types.py +++ /dev/null @@ -1,164 +0,0 @@ -from __future__ import annotations - -from unittest.mock import patch - -from mcp.types import ImageContent, TextContent - -from hud.types import MCPToolCall, MCPToolResult - - -def test_mcp_tool_call_str_long_args(): - """Test MCPToolCall __str__ truncates long arguments.""" - tool_call = MCPToolCall( - name="test_tool", - arguments={"very": "long" * 30 + " argument string that should be truncated"}, - ) - result = str(tool_call) - assert "..." in result - assert len(result) < 100 - - -def test_mcp_tool_call_str_invalid_json_args(): - """Test MCPToolCall __str__ handles non-JSON-serializable arguments.""" - tool_call = MCPToolCall(name="test_tool", arguments={"func": lambda x: x}) - result = str(tool_call) - assert "test_tool" in result - - -def test_mcp_tool_call_rich(): - """Test MCPToolCall __rich__ calls hud_console.""" - with patch("hud.utils.hud_console.hud_console") as mock_console: - mock_console.format_tool_call.return_value = "formatted" - tool_call = MCPToolCall(name="test", arguments={}) - result = tool_call.__rich__() - assert result == "formatted" - mock_console.format_tool_call.assert_called_once() - - -def test_mcp_tool_call_annotation_in_model_dump(): - """model_dump() includes annotation when set.""" - tool_call = MCPToolCall(name="click", arguments={"x": 100}, annotation="Navigate to login page") - data = tool_call.model_dump() - assert data["annotation"] == "Navigate to login page" - - -def test_mcp_tool_call_annotation_roundtrip(): - """Annotation survives serialize -> deserialize roundtrip.""" - original = MCPToolCall(name="click", arguments={"x": 100}, annotation="Step 1: open menu") - data = original.model_dump(mode="json") - restored = MCPToolCall(**data) - assert restored.annotation == "Step 1: open menu" - assert restored.name == original.name - assert restored.arguments == original.arguments - - -def test_mcp_tool_call_annotation_none_excluded(): - """model_dump(exclude_none=True) omits annotation when None.""" - tool_call = MCPToolCall(name="click", arguments={}) - data = tool_call.model_dump(exclude_none=True) - assert "annotation" not in data - - -def test_mcp_tool_call_annotation_defaults_to_none(): - """MCPToolCall without explicit annotation defaults to None.""" - tool_call = MCPToolCall(name="click", arguments={"x": 1}) - assert tool_call.annotation is None - - -def test_mcp_tool_call_str_with_annotation(): - """__str__ appends annotation comment when set.""" - tool_call = MCPToolCall(name="click", arguments={"x": 1}, annotation="Open the sidebar") - result = str(tool_call) - assert result.endswith(" # Open the sidebar") - assert "click" in result - - -def test_mcp_tool_call_str_without_annotation(): - """__str__ has no annotation comment when annotation is None.""" - tool_call = MCPToolCall(name="click", arguments={"x": 1}) - result = str(tool_call) - assert "#" not in result - - -def test_mcp_tool_call_rich_with_annotation(): - """__rich__ includes escaped annotation in bright_black markup.""" - with patch("hud.utils.hud_console.hud_console") as mock_console: - mock_console.format_tool_call.return_value = "formatted" - tool_call = MCPToolCall(name="test", arguments={}, annotation="has [brackets] & stuff") - result = tool_call.__rich__() - assert "[bright_black]" in result - assert "has \\[brackets] & stuff" in result - - -def test_mcp_tool_result_text_content(): - """Test MCPToolResult with text content.""" - result = MCPToolResult( - content=[TextContent(text="Test output", type="text")], - isError=False, - ) - assert "Test output" in str(result) - assert "✓" in str(result) - - -def test_mcp_tool_result_multiline_text(): - """Test MCPToolResult with multiline text uses first line.""" - result = MCPToolResult( - content=[TextContent(text="First line\nSecond line\nThird line", type="text")], - isError=False, - ) - assert "First line" in result._get_content_summary() - assert "Second line" not in result._get_content_summary() - - -def test_mcp_tool_result_image_content(): - """Test MCPToolResult with image content.""" - result = MCPToolResult( - content=[ImageContent(data="base64data", mimeType="image/png", type="image")], - isError=False, - ) - summary = result._get_content_summary() - assert "Image" in summary or "📷" in summary - - -def test_mcp_tool_result_structured_content(): - """Test MCPToolResult with structured content.""" - result = MCPToolResult( - content=[], - structuredContent={"key": "value", "nested": {"data": 123}}, - isError=False, - ) - summary = result._get_content_summary() - assert "key" in summary - - -def test_mcp_tool_result_structured_content_non_serializable(): - """Test MCPToolResult with non-JSON-serializable structured content.""" - result = MCPToolResult( - content=[], - structuredContent={"func": lambda x: x}, - isError=False, - ) - summary = result._get_content_summary() - assert summary # Should have some string representation - - -def test_mcp_tool_result_error(): - """Test MCPToolResult when isError is True.""" - result = MCPToolResult( - content=[TextContent(text="Error message", type="text")], - isError=True, - ) - assert "✗" in str(result) - - -def test_mcp_tool_result_rich(): - """Test MCPToolResult __rich__ calls hud_console.""" - with patch("hud.utils.hud_console.hud_console") as mock_console: - mock_console.format_tool_result.return_value = "formatted" - result = MCPToolResult( - content=[TextContent(text="Test", type="text")], - isError=False, - ) - rich_output = result.__rich__() - assert rich_output == "formatted" - mock_console.format_tool_result.assert_called_once() diff --git a/tests/utils/test_naming.py b/tests/utils/test_naming.py deleted file mode 100644 index 5edf49344..000000000 --- a/tests/utils/test_naming.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations - -from hud.utils.naming import normalize_environment_name - - -def test_normalize_environment_name() -> None: - assert normalize_environment_name("terminal-bench") == "terminal-bench" - assert normalize_environment_name("My Cool_Bench") == "my-cool-bench" - assert normalize_environment_name("bench@2.0!") == "bench20" - assert normalize_environment_name("--hello--") == "hello" - assert normalize_environment_name("a---b") == "a-b" - assert normalize_environment_name("@#$") == "environment" - assert normalize_environment_name("", default="converted") == "converted" diff --git a/tests/utils/test_serialization.py b/tests/utils/test_serialization.py deleted file mode 100644 index 1dfb92f25..000000000 --- a/tests/utils/test_serialization.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from hud.utils.serialization import json_safe_dict, json_safe_value - - -def test_json_safe_value_serializes_dataclass() -> None: - @dataclass - class Demo: - name: str - count: int - - result = json_safe_value(Demo(name="test", count=2)) - assert result == {"name": "test", "count": 2} - - -def test_json_safe_value_falls_back_for_unserializable_object() -> None: - class Weird: - def __init__(self) -> None: - raise RuntimeError("boom") - - value = Weird.__new__(Weird) - result = json_safe_value(value) - assert isinstance(result, str) - assert "Weird" in result - - -def test_json_safe_dict_serializes_each_value() -> None: - data = {"number": 1, "items": [1, 2, 3]} - assert json_safe_dict(data) == data From c07f49f15db7d015a8626b18980149042c3b838c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 020/248] test: pin settings precedence through the hud CLI hud set writes ~/.hud/.env, a project .env overrides it, and the process environment overrides both; the next hud command sends the winning key. Service URL defaults and HUD_FILE_TRACKING_ENABLED are observed through Settings and a served environment's manifest instead of pydantic field metadata. --- tests/test_settings.py | 160 +++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 78 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index ad75e85f8..c04de5333 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,81 +1,85 @@ -"""Tests for settings module.""" +"""Where the SDK's configuration comes from, and what a few settings switch.""" from __future__ import annotations -from hud.settings import Settings, get_settings, settings - - -def test_get_settings(): - """Test that get_settings returns the singleton settings instance.""" - result = get_settings() - assert isinstance(result, Settings) - assert result is settings # Should be the same singleton instance - - -def test_service_url_defaults(): - expected = { - "hud_telemetry_url": "https://telemetry.hud.ai/v3/api", - "hud_api_url": "https://api.hud.ai", - "hud_web_url": "https://hud.ai", - "hud_gateway_url": "https://inference.hud.ai", - "hud_runtime_url": "https://mcp.hud.ai", - "hud_rl_url": "https://rl.hud.ai", - } - - for name, default in expected.items(): - assert Settings.model_fields[name].default == default - - -def test_file_tracking_is_enabled_by_default(): - assert Settings.model_fields["file_tracking_enabled"].default is True - - -def test_file_tracking_can_be_disabled_by_env(monkeypatch): - monkeypatch.setenv("HUD_FILE_TRACKING_ENABLED", "false") - - assert Settings().file_tracking_enabled is False - - -def test_default_project_accepts_a_name_or_id(monkeypatch): - monkeypatch.setenv("HUD_DEFAULT_PROJECT", "browser-evals") - - assert Settings().default_project == "browser-evals" - - -def test_span_dir_defaults_to_home_when_uploads_are_off(monkeypatch, tmp_path): - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "false") - monkeypatch.setenv("HUD_TELEMETRY_LOCAL_DIR", "") - monkeypatch.setattr("hud.settings.Path.home", lambda: tmp_path) - - assert Settings().span_dir == str(tmp_path / ".hud" / "spans") - - -def test_span_dir_explicit_local_dir_wins_when_uploads_are_off(monkeypatch): - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "false") - monkeypatch.setenv("HUD_TELEMETRY_LOCAL_DIR", "./spans") - - assert Settings().span_dir == "./spans" - - -def test_span_dir_is_unset_when_uploads_are_on(monkeypatch): - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "true") - monkeypatch.setenv("HUD_TELEMETRY_LOCAL_DIR", "") - - assert Settings().span_dir is None - - -def test_cli_analytics_is_independent_of_trace_telemetry(monkeypatch): - monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "true") - monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "false") - configured = Settings() - - assert configured.telemetry_enabled is True - assert configured.cli_analytics_enabled is False - - -def test_settings_singleton(): - """Test that settings is a singleton.""" - s1 = get_settings() - s2 = get_settings() - assert s1 is s2 - assert s1 is settings +from typing import TYPE_CHECKING + +import pytest + +from hud import Environment +from hud.settings import Settings +from tests.harness import served + +if TYPE_CHECKING: + from pathlib import Path + + from tests.harness import FakeServices, Hud, HudEnv + +SERVICE_URLS = { + "HUD_TELEMETRY_URL": "https://telemetry.hud.ai/v3/api", + "HUD_API_URL": "https://api.hud.ai", + "HUD_WEB_URL": "https://hud.ai", + "HUD_GATEWAY_URL": "https://inference.hud.ai", + "HUD_RUNTIME_URL": "https://mcp.hud.ai", + "HUD_RL_URL": "https://rl.hud.ai", +} + + +@pytest.mark.parametrize( + ("process", "project", "user", "sent"), + [ + pytest.param(None, None, "from-hud-set", "from-hud-set", id="hud-set-only"), + pytest.param(None, "from-project", "from-hud-set", "from-project", id="project-wins"), + pytest.param("from-env", "from-project", "from-hud-set", "from-env", id="env-wins"), + ], +) +def test_the_api_key_comes_from_env_then_project_env_then_hud_set( + services: FakeServices, + hud_env: HudEnv, + hud: Hud, + process: str | None, + project: str | None, + user: str, + sent: str, +) -> None: + services.route("api", "GET", "/v2/jobs", json={"items": []}) + assert hud("set", f"HUD_API_KEY={user}").exit_code == 0 + if project is not None: + (hud.cwd / ".env").write_text(f"HUD_API_KEY={project}\n") + hud_env.set(HUD_API_KEY=process) + + result = hud("jobs", "list", "--json") + + assert result.exit_code == 0, result + assert [request.bearer for request in services.requests("api", "GET", "/v2/jobs")] == [sent] + + +def test_service_urls_default_to_the_hud_platform(hud_env: HudEnv) -> None: + hud_env.set(**dict.fromkeys(SERVICE_URLS)) + + configured = Settings(_env_file=None) + + assert { + variable: getattr(configured, variable.lower()) for variable in SERVICE_URLS + } == SERVICE_URLS + + +@pytest.mark.parametrize( + ("tracking", "published"), + [ + pytest.param(None, ["shell", "filetracking"], id="default-on"), + pytest.param("false", ["shell"], id="disabled"), + ], +) +async def test_file_tracking_follows_hud_file_tracking_enabled( + hud_env: HudEnv, tmp_path: Path, tracking: str | None, published: list[str] +) -> None: + hud_env.set(HUD_FILE_TRACKING_ENABLED=tracking) + env = Environment("tracked") + env.workspace(tmp_path / "workspace") + + async with served(env) as client: + assert client.manifest is not None + names = [binding.name for binding in client.manifest.bindings] + + assert names == published From 1cde0f4f822d12a80853dda8e44993a9a43a799d Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 021/248] test: grade templates in real rollouts and keep the text helpers as tables Combined BashGrader, combinator, custom-grader and LLM-judge grades now run inside templates of an in-process environment, asserting the run's reward, the evaluation summary tree and the evaluate step's details. The judge answers through the scripted model fake, so the requests it sends (bearer, model, criterion type, query) are pinned. Authoring mistakes fail the grade with their message or warn. The text helpers stay as one table through the public functions. --- tests/test_graders.py | 1266 +++++++++++++++++++++++------------------ 1 file changed, 717 insertions(+), 549 deletions(-) diff --git a/tests/test_graders.py b/tests/test_graders.py index 4b521cc49..5ca55848c 100644 --- a/tests/test_graders.py +++ b/tests/test_graders.py @@ -1,16 +1,17 @@ -"""Tests for hud.graders: comparison helpers, combine, graders.""" +"""Graders as task authors use them: text helpers, and graded templates in real rollouts.""" from __future__ import annotations -import json -import os +import time import warnings -from types import SimpleNamespace -from typing import Any -from unittest.mock import AsyncMock +from typing import TYPE_CHECKING, Any import pytest +from dirty_equals import IsStr +from inline_snapshot import snapshot +from hud import Environment +from hud.eval import LocalRuntime, Task, rollout from hud.graders import ( BashGrader, EvaluationResult, @@ -28,621 +29,788 @@ normalize, numeric_match, ) +from tests.harness import ScriptedAgent, Turn, say +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path -class TestResultShapes: - def test_subscore_score_aliases_value(self) -> None: - s = SubScore(name="acc", value=0.75, weight=1.0) - assert s.score == 0.75 + from hud.eval import Run + from tests.harness import HudEnv, ModelRequest, Models - def test_evaluation_result_from_float(self) -> None: - r = EvaluationResult.from_float(0.25) - assert r.reward == 0.25 - assert r.done is True +# ─── text helpers ──────────────────────────────────────────────────────────── - def test_evaluation_result_warns_when_subscores_disagree_with_reward(self) -> None: - with pytest.warns(UserWarning): - EvaluationResult(reward=1.0, subscores=[SubScore(name="a", value=0.5, weight=1.0)]) - def test_subscore_summary_omits_recursive_info(self) -> None: - node = SubScore( - name="any", - value=1.0, - children=[SubScore(name="tests", value=1.0, info={"exit_code": 0})], - ) - assert node.model_dump(mode="json")["children"][0]["info"] == {"exit_code": 0} +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("Hello World", "hello world"), + ("answer: 42!", "answer 42"), + ("The answer is a test", "answer is test"), + (" lots of space ", "lots of space"), + (42, "42"), + ], +) +def test_normalize_lowercases_and_strips_punctuation_articles_and_spaces( + text: Any, expected: str +) -> None: + assert normalize(text) == expected + + +@pytest.mark.parametrize( + ("score", "expected"), + [ + pytest.param(lambda: exact_match("The Answer!", "answer"), 1.0, id="exact-normalized"), + pytest.param(lambda: exact_match("Germany", "france"), 0.0, id="exact-differs"), + pytest.param(lambda: exact_match("Paris.", "Paris"), 1.0, id="exact-punctuation"), + pytest.param( + lambda: exact_match("The capital is Paris", "capital is Paris"), 1.0, id="exact-article" + ), + pytest.param( + lambda: exact_match("Paris", "Paris", normalize_text=False), 1.0, id="exact-raw-same" + ), + pytest.param( + lambda: exact_match("The Paris!", "Paris", normalize_text=False), + 0.0, + id="exact-raw-differs", + ), + pytest.param(lambda: exact_match(42, "42"), 1.0, id="exact-non-string"), + pytest.param(lambda: exact_match("", ""), 1.0, id="exact-empty"), + pytest.param(lambda: contains("The capital is Paris", "paris"), 1.0, id="contains"), + pytest.param(lambda: contains("The capital is Paris", "berlin"), 0.0, id="contains-not"), + pytest.param( + lambda: contains("Paris", "paris", case_sensitive=True), 0.0, id="contains-case" + ), + pytest.param( + lambda: contains("Paris", "Paris", case_sensitive=True), 1.0, id="contains-case-same" + ), + pytest.param( + lambda: contains_any("I like Toyota cars", ["toyota", "honda"]), 1.0, id="any-one" + ), + pytest.param( + lambda: contains_any("I like BMW cars", ["toyota", "honda"]), 0.0, id="any-none" + ), + pytest.param(lambda: contains_any("anything", []), 0.0, id="any-empty"), + pytest.param( + lambda: contains_any("Toyota", ["toyota"], case_sensitive=True), 0.0, id="any-case" + ), + pytest.param( + lambda: contains_all("Toyota and Honda", ["toyota", "honda"]), 1.0, id="all-present" + ), + pytest.param( + lambda: contains_all("Toyota is Japanese", ["toyota", "honda"]), 0.0, id="all-missing" + ), + pytest.param(lambda: contains_all("anything", []), 1.0, id="all-empty"), + pytest.param(lambda: numeric_match("The answer is 42", 42), 1.0, id="number-int"), + pytest.param(lambda: numeric_match("Result: 3.14", 3.14), 1.0, id="number-float"), + pytest.param(lambda: numeric_match("no numbers", 42), 0.0, id="number-none"), + pytest.param(lambda: numeric_match("The answer is 41", 42), 0.0, id="number-wrong"), + pytest.param( + lambda: numeric_match("It is 41.5", 42, tolerance=1.0), 1.0, id="number-tolerance" + ), + pytest.param( + lambda: numeric_match("It is 40", 42, tolerance=1.0), 0.0, id="number-out-of-tolerance" + ), + pytest.param(lambda: numeric_match("It is -5 degrees", -5), 1.0, id="number-negative"), + pytest.param(lambda: numeric_match("3 items and 5 left", 3), 1.0, id="number-first"), + pytest.param(lambda: f1_score("Paris", "Paris"), 1.0, id="f1-exact"), + pytest.param(lambda: f1_score("The capital is Paris, France", "Paris"), 0.4, id="f1-part"), + pytest.param(lambda: f1_score("Berlin", "Paris"), 0.0, id="f1-none"), + pytest.param(lambda: f1_score("", "Paris"), 0.0, id="f1-empty-prediction"), + pytest.param(lambda: f1_score("Paris", ""), 0.0, id="f1-empty-reference"), + pytest.param(lambda: f1_score("New York City", "New York City"), 1.0, id="f1-words"), + pytest.param(lambda: f1_score("I think 42 degrees", "42 degrees"), 2 / 3, id="f1-superset"), + pytest.param(lambda: f1_score("42", "42 degrees celsius"), 0.5, id="f1-subset"), + pytest.param(lambda: f1_score("The PARIS!", "paris"), 1.0, id="f1-normalized"), + ], +) +def test_text_comparisons_score_between_zero_and_one( + score: Callable[[], float], expected: float +) -> None: + assert score() == pytest.approx(expected) - dumped = node.to_summary() - assert dumped["children"][0]["name"] == "tests" - assert "info" not in dumped - assert "info" not in dumped["children"][0] - def test_evaluation_result_serializes_recursive_info(self) -> None: - result = EvaluationResult( - reward=1.0, - subscores=[ - SubScore( - name="judge", - value=1.0, - children=[ - SubScore( - name="Mentions Paris", - value=1.0, - info={"reason": "says Paris"}, - ) - ], - info={"model": "judge-model"}, - ) - ], - ) - dumped = result.model_dump(mode="json") - assert dumped["subscores"][0]["info"] == {"model": "judge-model"} - assert dumped["subscores"][0]["children"][0]["info"] == {"reason": "says Paris"} +# ─── graded templates ──────────────────────────────────────────────────────── - def test_subscore_accepts_metadata_alias(self) -> None: - subscore = SubScore.model_validate( - {"name": "legacy", "value": 1.0, "metadata": {"source": "legacy"}} - ) - assert subscore.info == {"source": "legacy"} - assert subscore.metadata == subscore.info - assert subscore.model_dump(mode="json")["info"] == {"source": "legacy"} - subscore.metadata = {"source": "updated"} - assert subscore.info == {"source": "updated"} +ANSWER = "Paris is the capital." -class TestNormalize: - def test_lowercases(self) -> None: - assert normalize("Hello World") == "hello world" +class Measured(Grader): + """A custom grader whose ``kind`` picks which result shape it returns.""" - def test_strips_punctuation(self) -> None: - assert normalize("answer: 42!") == "answer 42" + name = "measured" - def test_strips_articles(self) -> None: - assert normalize("The answer is a test") == "answer is test" + @classmethod + async def compute_score(cls, kind: str = "float", **kwargs: Any) -> Any: + del kwargs + if kind == "tuple": + return 0.75, {"source": "tuple"} + if kind == "subscore": + return SubScore(name="ignored", value=1.0, info={"reason": "did it"}) + return 0.75 - def test_collapses_whitespace(self) -> None: - assert normalize(" lots of space ") == "lots of space" - def test_non_string_input(self) -> None: - assert normalize(42) == "42" +def _graders_env(root: Path) -> Environment: + env = Environment("graders") + def task(grade: Callable[[str], Any]) -> None: + @env.template(id=grade.__name__) + async def template(): + answer = yield "Name the capital of France." + yield await grade(answer) -class TestExactMatch: - def test_match_normalized(self) -> None: - assert exact_match("The Answer!", "answer") == 1.0 + async def bash_pair(answer: str) -> EvaluationResult: + del answer + return await combine( + BashGrader.grade(weight=0.5, command="echo pass"), + BashGrader.grade(weight=0.5, command="echo oops >&2; false"), + ) - def test_no_match(self) -> None: - assert exact_match("Germany", "france") == 0.0 + async def any_tree(answer: str) -> EvaluationResult: + either = combine_any( + weight=0.5, + subscores=[ + await BashGrader.grade(weight=0.5, command="false"), + await BashGrader.grade(weight=0.5, command="true"), + ], + ) + both = combine_all( + weight=0.25, + subscores=[SubScore(name="lint", value=1.0), SubScore(name="types", value=0.0)], + name="checks", + ) + return await combine( + either, both, SubScore(name="format", value=exact_match(answer, ANSWER), weight=0.25) + ) - def test_punctuation_stripped(self) -> None: - assert exact_match("Paris.", "Paris") == 1.0 + async def weighted(answer: str) -> EvaluationResult: + return await combine( + SubScore(name="mentions", value=contains(answer, "paris"), weight=0.6), + SubScore(name="numbers", value=numeric_match(answer, 7), weight=0.4), + SubScore(name="penalty", value=1.0, weight=-0.2), + ) - def test_articles_stripped(self) -> None: - assert exact_match("The capital is Paris", "capital is Paris") == 1.0 + async def penalty_only_fails(answer: str) -> EvaluationResult: + del answer + return await combine( + SubScore(name="correct", value=0.0, weight=1.0), + SubScore(name="penalty", value=1.0, weight=-0.2), + ) - def test_no_normalize(self) -> None: - assert exact_match("Paris", "Paris", normalize_text=False) == 1.0 - assert exact_match("The Paris!", "Paris", normalize_text=False) == 0.0 + async def suffixed(answer: str) -> EvaluationResult: + del answer + return await combine( + SubScore(name="x-1", value=1.0, weight=0.3), + SubScore(name="x", value=1.0, weight=0.3), + SubScore(name="x", value=0.0, weight=0.4), + ) - def test_non_string_answer(self) -> None: - assert exact_match(42, "42") == 1.0 + async def custom(answer: str) -> EvaluationResult: + del answer + return await combine( + Measured.grade(weight=0.25, kind="float", payload=object()), + Measured.grade(weight=0.25, kind="tuple"), + Measured.grade(weight=0.25, kind="subscore", name="renamed"), + SubScore.model_validate( + {"name": "legacy", "value": 1.0, "weight": 0.25, "metadata": {"from": "metadata"}} + ), + ) - def test_empty_strings(self) -> None: - assert exact_match("", "") == 1.0 + async def concurrent(answer: str) -> EvaluationResult: + del answer + # Each command waits for the file the other writes, so they pass only in parallel. + wait = "for _ in $(seq 50); do [ -e {mine} ] && exit 0; sleep 0.1; done; exit 1" + return await combine( + BashGrader.grade( + weight=0.5, command=f"touch {root}/a; " + wait.format(mine=f"{root}/b") + ), + BashGrader.grade( + weight=0.5, command=f"touch {root}/b; " + wait.format(mine=f"{root}/a") + ), + ) - def test_different_content(self) -> None: - assert exact_match("hello", "world") == 0.0 + async def in_cwd(answer: str) -> EvaluationResult: + del answer + (root / "marker.txt").write_text("here") + return await combine( + BashGrader.grade(weight=1.0, command="cat marker.txt", cwd=str(root)), + ) + async def detached_child(answer: str) -> EvaluationResult: + del answer + return await combine( + BashGrader.grade(weight=1.0, command="echo started; sleep 30 & exit 0"), + ) -class TestContains: - def test_contains_substring(self) -> None: - assert contains("The capital of France is Paris", "paris") == 1.0 + async def overrun(answer: str) -> EvaluationResult: + del answer + return await combine( + BashGrader.grade( + weight=1.0, command="echo progress; echo stuck >&2; sleep 30", timeout_seconds=1 + ), + ) - def test_not_contains(self) -> None: - assert contains("The capital of France is Paris", "berlin") == 0.0 + async def chatty(answer: str) -> EvaluationResult: + del answer + return await combine( + BashGrader.grade(weight=1.0, command="yes hud | head -c 500000"), + ) - def test_case_sensitive(self) -> None: - assert contains("Paris", "paris", case_sensitive=True) == 0.0 - assert contains("Paris", "Paris", case_sensitive=True) == 1.0 + async def judged(answer: str) -> EvaluationResult: + return await combine( + LLMJudgeGrader.grade( + weight=1.0, + answer=answer, + criteria=["names Paris", ("cites a source", 2.0), ("invents facts", -1.0)], + question="Name the capital of France.", + ) + ) + async def judged_negative_only(answer: str) -> EvaluationResult: + return await combine( + LLMJudgeGrader.grade(weight=1.0, answer=answer, criteria=[("invents facts", -1.0)]) + ) -class TestContainsAny: - def test_matches_one(self) -> None: - assert contains_any("I like Toyota cars", ["toyota", "honda", "nissan"]) == 1.0 + async def judged_without_criteria(answer: str) -> EvaluationResult: + return await combine(LLMJudgeGrader.grade(weight=1.0, answer=answer, criteria=[])) + + for grade in ( + bash_pair, + any_tree, + weighted, + penalty_only_fails, + suffixed, + custom, + concurrent, + in_cwd, + detached_child, + overrun, + chatty, + judged, + judged_negative_only, + judged_without_criteria, + ): + task(grade) + return env + + +async def _grade(template: str, tmp_path: Path, answer: str = ANSWER) -> Run: + root = tmp_path / "grader" + root.mkdir(exist_ok=True) + task = Task(env="graders", id=template) + return await rollout(task, ScriptedAgent(answer), runtime=LocalRuntime(_graders_env(root))) + + +def _evaluated(run: Run) -> dict[str, Any]: + """The evaluation the platform receives: the evaluate step's full grade.""" + step = run.trace.steps[-1] + assert step.task_call is not None + assert step.task_call.phase == "evaluate" + result = step.task_call.result + assert isinstance(result, dict) + return result + + +@pytest.mark.parametrize( + ("template", "reward", "summary"), + [ + pytest.param( + "bash_pair", + 0.5, + snapshot( + [ + {"name": "BashGrader-1", "weight": 0.5, "value": 1.0, "children": None}, + {"name": "BashGrader-2", "weight": 0.5, "value": 0.0, "children": None}, + ] + ), + id="bash-pair", + ), + pytest.param( + "any_tree", + 0.75, + snapshot( + [ + { + "name": "any", + "weight": 0.5, + "value": 1.0, + "children": [ + {"name": "BashGrader-1", "weight": 0.5, "value": 0.0, "children": None}, + {"name": "BashGrader-2", "weight": 0.5, "value": 1.0, "children": None}, + ], + }, + { + "name": "checks", + "weight": 0.25, + "value": 0.0, + "children": [ + {"name": "lint", "weight": 1.0, "value": 1.0, "children": None}, + {"name": "types", "weight": 1.0, "value": 0.0, "children": None}, + ], + }, + {"name": "format", "weight": 0.25, "value": 1.0, "children": None}, + ] + ), + id="any-all-tree", + ), + pytest.param( + "weighted", + 0.4, + snapshot( + [ + {"name": "mentions", "weight": 0.6, "value": 1.0, "children": None}, + {"name": "numbers", "weight": 0.4, "value": 0.0, "children": None}, + {"name": "penalty", "weight": -0.2, "value": 1.0, "children": None}, + ] + ), + id="penalty", + ), + pytest.param( + "penalty_only_fails", + -0.2, + snapshot( + [ + {"name": "correct", "weight": 1.0, "value": 0.0, "children": None}, + {"name": "penalty", "weight": -0.2, "value": 1.0, "children": None}, + ] + ), + id="negative-reward", + ), + pytest.param( + "suffixed", + 0.6, + snapshot( + [ + {"name": "x-1", "weight": 0.3, "value": 1.0, "children": None}, + {"name": "x-2", "weight": 0.3, "value": 1.0, "children": None}, + {"name": "x-3", "weight": 0.4, "value": 0.0, "children": None}, + ] + ), + id="name-collisions", + ), + pytest.param( + "custom", + 0.875, + snapshot( + [ + {"name": "measured-1", "weight": 0.25, "value": 0.75, "children": None}, + {"name": "measured-2", "weight": 0.25, "value": 0.75, "children": None}, + {"name": "renamed", "weight": 0.25, "value": 1.0, "children": None}, + {"name": "legacy", "weight": 0.25, "value": 1.0, "children": None}, + ] + ), + id="custom-grader-shapes", + ), + ], +) +async def test_a_combined_grade_reaches_the_run_as_a_named_weighted_tree( + tmp_path: Path, template: str, reward: float, summary: list[dict[str, Any]] +) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("error") + run = await _grade(template, tmp_path) + + assert run.reward == pytest.approx(reward) + assert run.evaluation["subscores"] == summary + + +async def test_the_evaluate_step_keeps_each_graders_details(tmp_path: Path) -> None: + run = await _grade("bash_pair", tmp_path) + + assert [subscore["info"] for subscore in _evaluated(run)["subscores"]] == snapshot( + [ + { + "exit_code": 0, + "stdout": "pass\n", + "stderr": "", + "_parameters": {"command": "echo pass"}, + }, + { + "exit_code": 1, + "stdout": "", + "stderr": "oops\n", + "_parameters": {"command": "echo oops >&2; false"}, + }, + ] + ) + + +async def test_custom_graders_record_their_parameters_and_info(tmp_path: Path) -> None: + run = await _grade("custom", tmp_path) + + assert {s["name"]: s["info"] for s in _evaluated(run)["subscores"]} == snapshot( + { + "measured-1": { + "_parameters": {"kind": "float", "payload": ""} + }, + "measured-2": {"source": "tuple", "_parameters": {"kind": "tuple"}}, + "renamed": {"reason": "did it", "_parameters": {"kind": "subscore"}}, + "legacy": {"from": "metadata"}, + } + ) + + +@pytest.mark.parametrize( + ("template", "value", "info"), + [ + pytest.param( + "in_cwd", + 1.0, + {"exit_code": 0, "stdout": "here", "stderr": ""}, + id="cwd", + ), + pytest.param( + "detached_child", + 1.0, + {"exit_code": 0, "stdout": "started\n", "stderr": ""}, + id="background-child-does-not-hold-the-grade", + ), + pytest.param( + "overrun", + 0.0, + { + "exit_code": None, + "stdout": "progress\n", + "stderr": "stuck\n", + "timed_out": True, + "timeout": 1, + }, + id="overrun-keeps-its-output", + ), + ], +) +async def test_bash_grader_scores_by_exit_code_and_reports_what_ran( + tmp_path: Path, template: str, value: float, info: dict[str, Any] +) -> None: + started = time.monotonic() - def test_matches_none(self) -> None: - assert contains_any("I like BMW cars", ["toyota", "honda", "nissan"]) == 0.0 + run = await _grade(template, tmp_path) - def test_empty_list(self) -> None: - assert contains_any("anything", []) == 0.0 + (subscore,) = _evaluated(run)["subscores"] + assert run.reward == value + assert {key: subscore["info"][key] for key in info} == info + assert time.monotonic() - started < 15 - def test_case_sensitive(self) -> None: - assert contains_any("Toyota", ["toyota"], case_sensitive=True) == 0.0 +async def test_bash_grader_reads_a_large_output_without_blocking(tmp_path: Path) -> None: + run = await _grade("chatty", tmp_path) -class TestContainsAll: - def test_all_present(self) -> None: - assert contains_all("Toyota and Honda are Japanese", ["toyota", "honda"]) == 1.0 + (subscore,) = _evaluated(run)["subscores"] + assert run.reward == 1.0 + assert len(subscore["info"]["stdout"]) == 500_000 - def test_some_missing(self) -> None: - assert contains_all("Toyota is Japanese", ["toyota", "honda"]) == 0.0 - def test_empty_list(self) -> None: - assert contains_all("anything", []) == 1.0 +async def test_combined_graders_run_concurrently(tmp_path: Path) -> None: + run = await _grade("concurrent", tmp_path) + assert run.reward == 1.0 -class TestNumericMatch: - def test_exact_integer(self) -> None: - assert numeric_match("The answer is 42", 42) == 1.0 - def test_exact_float(self) -> None: - assert numeric_match("Result: 3.14", 3.14) == 1.0 +# ─── authoring mistakes ────────────────────────────────────────────────────── - def test_no_number(self) -> None: - assert numeric_match("no numbers", 42) == 0.0 - def test_wrong_number(self) -> None: - assert numeric_match("The answer is 41", 42) == 0.0 +NOT_SUBSCORES: list[Any] = [0.5] - def test_tolerance(self) -> None: - assert numeric_match("The answer is 41.5", 42, tolerance=1.0) == 1.0 - assert numeric_match("The answer is 40", 42, tolerance=1.0) == 0.0 - def test_negative_number(self) -> None: - assert numeric_match("Temperature is -5 degrees", -5) == 1.0 +def _mistakes_env() -> Environment: + env = Environment("mistakes") + cases: dict[str, Callable[[], Any]] = { + "empty_combine": lambda: combine(), + "no_positive_weight": lambda: combine(SubScore(name="p", value=1.0, weight=-1.0)), + "not_a_subscore": lambda: combine(*NOT_SUBSCORES), + "missing_command": lambda: combine(BashGrader.grade(weight=1.0)), + "empty_any": lambda: _awaitable(lambda: combine_any(weight=1.0, subscores=[])), + "empty_all": lambda: _awaitable(lambda: combine_all(weight=1.0, subscores=[])), + } + for name, grade in cases.items(): - def test_extracts_first_number(self) -> None: - assert numeric_match("There are 3 items and 5 left", 3) == 1.0 + def register(grade: Callable[[], Any] = grade, name: str = name) -> None: + @env.template(id=name) + async def template(): + yield "go" + yield await grade() + register() + return env -class TestF1Score: - def test_exact_match(self) -> None: - assert f1_score("Paris", "Paris") == pytest.approx(1.0) - def test_partial_overlap(self) -> None: - # pred=["capital", "is", "paris", "france"], ref=["paris"] - # common=1, precision=1/4, recall=1/1, f1=2*(0.25*1)/(0.25+1)=0.4 - assert f1_score("The capital is Paris, France", "Paris") == pytest.approx(0.4) +async def _awaitable(build: Callable[[], SubScore]) -> EvaluationResult: + return await combine(build()) - def test_no_overlap(self) -> None: - assert f1_score("Berlin", "Paris") == pytest.approx(0.0) - def test_empty_prediction(self) -> None: - assert f1_score("", "Paris") == pytest.approx(0.0) +@pytest.mark.parametrize( + ("template", "error"), + [ + ("empty_combine", "subscores must not be empty"), + ("no_positive_weight", "subscores must include at least one positive weight"), + ("not_a_subscore", "Expected SubScore or Awaitable[SubScore], got float"), + ("missing_command", "BashGrader requires command"), + ("empty_any", "subscores must not be empty"), + ("empty_all", "subscores must not be empty"), + ], +) +async def test_an_authoring_mistake_fails_the_grade_with_its_message( + template: str, error: str +) -> None: + task = Task(env="mistakes", id=template) - def test_empty_reference(self) -> None: - assert f1_score("Paris", "") == pytest.approx(0.0) + run = await rollout(task, ScriptedAgent("x"), runtime=LocalRuntime(_mistakes_env())) - def test_multi_word_match(self) -> None: - assert f1_score("New York City", "New York City") == pytest.approx(1.0) + assert run.trace.status == "error" + assert run.trace.error == IsStr(regex=rf"\[grading\] .*: {_escaped(error)}") - def test_superset_answer(self) -> None: - # pred has all ref tokens plus extras -> recall=1.0, precision<1.0 - result = f1_score("I believe the answer is 42 degrees", "42 degrees") - assert 0.0 < result < 1.0 - def test_subset_answer(self) -> None: - # pred has fewer tokens than ref -> precision=1.0, recall<1.0 - result = f1_score("42", "42 degrees celsius") - assert 0.0 < result < 1.0 +def _escaped(text: str) -> str: + return "".join(f"\\{char}" if char in "[]()." else char for char in text) - def test_normalization_applied(self) -> None: - assert f1_score("The PARIS!", "paris") == pytest.approx(1.0) +def _warning_env() -> Environment: + env = Environment("warnings") -class TestCombineParallelism: - async def test_combine_sync_subscores(self) -> None: - result = await combine( + @env.template() + async def misweighted(): + yield "go" + yield await combine( SubScore(name="a", value=1.0, weight=0.5), - SubScore(name="b", value=0.0, weight=0.5), + SubScore(name="b", value=1.0, weight=0.3), + SubScore(name="c", value=0.0, weight=0.3), ) - assert result.reward == pytest.approx(0.5) - - async def test_combine_with_awaitables(self) -> None: - import asyncio - - order: list[str] = [] - - async def slow_check_a() -> SubScore: - order.append("a_start") - await asyncio.sleep(0.05) - order.append("a_end") - return SubScore(name="a", value=1.0, weight=0.5) - - async def slow_check_b() -> SubScore: - order.append("b_start") - await asyncio.sleep(0.05) - order.append("b_end") - return SubScore(name="b", value=0.0, weight=0.5) - - result = await combine(slow_check_a(), slow_check_b()) - assert result.reward == pytest.approx(0.5) - assert order.index("b_start") < order.index("a_end") - - async def test_combine_mixed(self) -> None: - import asyncio - - async def async_score() -> SubScore: - await asyncio.sleep(0.01) - return SubScore(name="async", value=1.0, weight=0.5) - - result = await combine( - SubScore(name="sync", value=0.0, weight=0.5), - async_score(), - ) - assert result.reward == pytest.approx(0.5) - -#: ``BashGrader`` shells out to ``/bin/bash``; skip its tests where it's absent (Windows). -_HAS_BASH = os.path.exists("/bin/bash") + @env.template() + async def mismatched(): + yield "go" + yield EvaluationResult(reward=1.0, subscores=[SubScore(name="a", value=0.5, weight=1.0)]) + return env -class TestCombine: - async def test_combine_returns_evaluation_result(self) -> None: - result = await combine(SubScore(name="alpha", value=1.0, weight=1.0)) - assert isinstance(result, EvaluationResult) - assert result.reward == 1.0 - assert result.done is True - async def test_combine_normalizes_positive_weights(self) -> None: - result = await combine( - SubScore(name="alpha", value=1.0, weight=2.0), - SubScore(name="beta", value=0.0, weight=1.0), - ) - assert result.reward == pytest.approx(2.0 / 3.0) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - assert by_name["alpha"].weight == pytest.approx(2.0 / 3.0) - assert by_name["beta"].weight == pytest.approx(1.0 / 3.0) - - async def test_combine_preserves_negative_penalties(self) -> None: - result = await combine( - SubScore(name="correct", value=1.0, weight=1.0), - SubScore(name="penalty", value=1.0, weight=-0.2), - ) - assert result.reward == pytest.approx(0.8) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - assert by_name["correct"].weight == pytest.approx(1.0) - assert by_name["penalty"].weight == pytest.approx(-0.2) - - async def test_combine_duplicate_names_are_deduped(self) -> None: - result = await combine( - SubScore(name="same", value=1.0, weight=0.5), - SubScore(name="same", value=0.0, weight=0.5), - ) - assert result.subscores is not None - assert [subscore.name for subscore in result.subscores] == ["same-1", "same-2"] - - async def test_combine_duplicate_names_avoid_existing_suffix_collisions(self) -> None: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = await combine( - SubScore(name="x-1", value=1.0, weight=0.3), - SubScore(name="x", value=1.0, weight=0.4), - SubScore(name="x", value=0.0, weight=0.6), - ) - - assert result.subscores is not None - assert [subscore.name for subscore in result.subscores] == ["x-1", "x-2", "x-3"] - assert set(result.info) == set() - assert not [ - warning for warning in caught if "Duplicate subscore names" in str(warning.message) - ] - - async def test_combine_propagates_info(self) -> None: - info = {"stdout": "ok"} - result = await combine(SubScore(name="grader", value=1.0, weight=1.0, info=info)) - assert result.info == {} - assert result.subscores is not None - assert result.subscores[0].info == info - assert result.model_dump(mode="json")["subscores"][0]["info"] == info - - async def test_combine_keeps_rubric_children_on_subscores(self) -> None: - verdicts = [ - SubScore(name="Mentions Paris", value=1.0, info={"reason": "says Paris"}), - SubScore(name="Names the river", value=0.0, info={"reason": "no river"}), - ] - result = await combine( - SubScore(name="judge", value=0.5, weight=0.5, children=verdicts), - SubScore(name="tests", value=0.0, weight=0.5), - ) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - assert by_name["judge"].children == verdicts - assert by_name["tests"].children is None - - async def test_combine_preserves_combinator_children(self) -> None: - result = await combine( - combine_any( - weight=0.5, - subscores=[ - SubScore(name="pytest", value=1.0), - SubScore(name="make", value=0.0), - ], +@pytest.mark.parametrize( + ("template", "warning", "reward"), + [ + ("misweighted", "grader weights sum to 1.1000, not 1.0", 0.8 / 1.1), + ("mismatched", "Subscores don't match reward", 1.0), + ], +) +async def test_inconsistent_weights_warn_the_author( + template: str, warning: str, reward: float +) -> None: + task = Task(env="warnings", id=template) + + with pytest.warns(UserWarning, match=warning): + run = await rollout(task, ScriptedAgent("x"), runtime=LocalRuntime(_warning_env())) + + assert run.reward == pytest.approx(reward) + + +# ─── the LLM judge ─────────────────────────────────────────────────────────── + +VERDICTS = { + "json": '{"criterion_status": "MET", "explanation": "plainly stated"}', + "fenced": '```json\n{"criterion_status": "UNMET", "explanation": "no source"}\n```', + "prose": "Verdict: UNMET, although the word MET appears here.", + "prose-met": "The criterion is MET.", +} + + +def _judge(verdicts: dict[str, str]) -> Callable[[ModelRequest], Turn]: + def reply(request: ModelRequest) -> Turn: + prompt = request.body["messages"][1]["content"] + criterion = prompt.split("\n", 1)[1].split("\n", 1)[0] + return say(VERDICTS[verdicts[criterion]]) + + return reply + + +@pytest.mark.parametrize( + ("template", "verdicts", "reward", "children"), + [ + pytest.param( + "judged", + {"names Paris": "json", "cites a source": "fenced", "invents facts": "prose"}, + 1 / 3, + snapshot( + [ + { + "name": "names Paris", + "weight": 1.0, + "value": 1.0, + "children": None, + "info": {"reason": "plainly stated"}, + }, + { + "name": "cites a source", + "weight": 2.0, + "value": 0.0, + "children": None, + "info": {"reason": "no source"}, + }, + { + "name": "invents facts", + "weight": -1.0, + "value": 0.0, + "children": None, + "info": {"reason": "Verdict: UNMET, although the word MET appears here."}, + }, + ] ), - SubScore(name="format", value=1.0, weight=0.5), - ) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - any_node = by_name["any"] - assert any_node.children is not None - assert [child.name for child in any_node.children] == ["pytest", "make"] - assert by_name["format"].children is None - - async def test_combine_preserves_negative_reward_without_validator_warning(self) -> None: - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - result = await combine( - SubScore(name="correct", value=0.0, weight=1.0), - SubScore(name="penalty", value=1.0, weight=-0.2), - ) - - assert result.reward == pytest.approx(-0.2) - assert not [ - warning for warning in caught if "Subscores don't match reward" in str(warning.message) - ] - - -class TestGrader: - async def test_grade_wraps_float_and_stores_parameters(self) -> None: - class DummyGrader(Grader): - name = "DummyGrader" - - @classmethod - async def compute_score(cls, **kwargs: object) -> Any: - return 0.75 - - subscore = await DummyGrader.grade(weight=0.4, marker="ok", payload=object()) - assert isinstance(subscore, SubScore) - assert subscore.name == "DummyGrader" - assert subscore.value == pytest.approx(0.75) - assert subscore.weight == pytest.approx(0.4) - assert subscore.info is not None - assert subscore.info["_parameters"]["marker"] == "ok" - assert subscore.info["_parameters"]["payload"] == "" - - async def test_grade_coerces_legacy_tuple_metadata(self) -> None: - class TupleGrader(Grader): - name = "tuple" - - @classmethod - async def compute_score(cls, **kwargs: object) -> Any: - return 0.75, {"source": "released-contract"} - - subscore = await TupleGrader.grade(weight=0.4, marker="ok") - assert subscore.name == "tuple" - assert subscore.value == pytest.approx(0.75) - assert subscore.weight == pytest.approx(0.4) - assert subscore.info is not None - assert subscore.info["source"] == "released-contract" - assert subscore.info["_parameters"]["marker"] == "ok" - - async def test_grade_uses_grader_name_without_override(self) -> None: - class RubricGrader(Grader): - name = "rubric" - - @classmethod - async def compute_score(cls, **kwargs: object) -> SubScore: - return SubScore(name="specific-rubric", value=1.0) - - subscore = await RubricGrader.grade(weight=0.7) - assert subscore.name == "rubric" - assert subscore.weight == pytest.approx(0.7) - - async def test_grade_stamps_name_and_weight_on_returned_subscore(self) -> None: - class RubricGrader(Grader): - name = "rubric" - - @classmethod - async def compute_score(cls, **kwargs: object) -> SubScore: - return SubScore( - name="ignored", - value=1.0, - info={"reason": "did the thing", "extra": "kept"}, - ) - - subscore = await RubricGrader.grade(weight=0.7, name="my-rubric") - assert subscore.name == "my-rubric" - assert subscore.weight == pytest.approx(0.7) - assert subscore.info is not None - assert subscore.info["reason"] == "did the thing" - assert subscore.info["extra"] == "kept" - assert "_parameters" in subscore.info - - -class TestBooleanCombinators: - def test_combine_any_picks_max(self) -> None: - combined = combine_any( - weight=1.0, - subscores=[ - SubScore(name="a", value=1.0, weight=0.5), - SubScore(name="b", value=0.0, weight=0.5), - ], - ) - assert combined.name == "any" - assert combined.value == 1.0 - - def test_combine_any_keeps_inputs_as_children(self) -> None: - rubric = SubScore( - name="judge", - value=1.0, - weight=0.5, - children=[SubScore(name="c1", value=1.0, info={"reason": "met"})], - ) - combined = combine_any( - weight=1.0, - subscores=[rubric, SubScore(name="tests", value=0.0, weight=0.5)], - ) - assert combined.children is not None - assert combined.children[0].children == rubric.children - assert combined.children[1].children is None - - def test_combine_any_dedupes_child_names(self) -> None: - combined = combine_any( - weight=1.0, - subscores=[ - SubScore(name="BashGrader", value=1.0, weight=0.5, info={"exit_code": 0}), - SubScore(name="BashGrader", value=0.0, weight=0.5, info={"exit_code": 1}), - ], - ) - assert combined.children is not None - assert [child.name for child in combined.children] == ["BashGrader-1", "BashGrader-2"] - assert [child.info for child in combined.children] == [ - {"exit_code": 0}, - {"exit_code": 1}, + id="json-fenced-prose", + ), + pytest.param( + "judged_negative_only", + {"invents facts": "prose-met"}, + 0.0, + snapshot( + [ + { + "name": "invents facts", + "weight": -1.0, + "value": 1.0, + "children": None, + "info": {"reason": "The criterion is MET."}, + } + ] + ), + id="negative-only-met", + ), + pytest.param( + "judged_negative_only", + {"invents facts": "prose"}, + 1.0, + snapshot( + [ + { + "name": "invents facts", + "weight": -1.0, + "value": 0.0, + "children": None, + "info": {"reason": "Verdict: UNMET, although the word MET appears here."}, + } + ] + ), + id="negative-only-unmet", + ), + ], +) +async def test_the_judge_scores_each_criterion_by_weight( + models: Models, + hud_env: HudEnv, + tmp_path: Path, + template: str, + verdicts: dict[str, str], + reward: float, + children: list[dict[str, Any]], +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(_judge(verdicts)) + + run = await _grade(template, tmp_path) + + (judge,) = _evaluated(run)["subscores"] + assert run.reward == pytest.approx(reward) + assert judge["info"]["model"] == "claude-haiku-4-5" + assert judge["children"] == children + + +async def test_the_judge_sends_one_request_per_criterion( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond( + _judge({"names Paris": "json", "cites a source": "json", "invents facts": "prose"}) + ) + + await _grade("judged", tmp_path) + + requests = sorted(models.requests(), key=lambda request: request.body["messages"][1]["content"]) + assert [request.headers["authorization"] for request in requests] == ["Bearer k"] * 3 + assert {request.model for request in requests} == {"claude-haiku-4-5"} + assert {request.body["messages"][0]["content"][:40] for request in requests} == { + "You evaluate a response against a single" + } + assert [request.body["messages"][1]["content"] for request in requests] == snapshot( + [ + """\ + +negative + + + +invents facts + + + +Name the capital of France. + + + +Paris is the capital. +\ +""", + """\ + +positive + + + +cites a source + + + +Name the capital of France. + + + +Paris is the capital. +\ +""", + """\ + +positive + + + +names Paris + + + +Name the capital of France. + + + +Paris is the capital. +\ +""", ] - assert combined.info is None - - def test_combine_all_picks_min(self) -> None: - combined = combine_all( - weight=1.0, - subscores=[ - SubScore(name="a", value=1.0, weight=0.5), - SubScore(name="b", value=0.0, weight=0.5), - ], - name="tests_all", - ) - assert combined.name == "tests_all" - assert combined.value == 0.0 - - def test_nested_combinators_build_a_tree(self) -> None: - combined = combine_all( - weight=1.0, - subscores=[ - combine_any( - weight=1.0, - subscores=[ - SubScore(name="pytest", value=0.0), - SubScore(name="make", value=1.0), - ], - ), - SubScore(name="lint", value=1.0), - ], - ) - assert combined.value == 1.0 - assert combined.children is not None - inner = combined.children[0] - assert inner.children is not None - assert [child.name for child in inner.children] == ["pytest", "make"] + ) -class _FakeGatewayClient: - """Judges MET when the criterion text (inside the user prompt) contains 'met:'.""" +async def test_the_judge_without_a_question_omits_the_query( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(_judge({"invents facts": "prose"})) - def __init__(self) -> None: - self.chat = SimpleNamespace(completions=SimpleNamespace(create=self._create)) + await _grade("judged_negative_only", tmp_path) - async def _create(self, **kwargs: Any) -> Any: - user_prompt: str = kwargs["messages"][1]["content"] - status = "MET" if "met:" in user_prompt else "UNMET" - content = json.dumps({"criterion_status": status, "explanation": "because"}) - return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=content))]) + (request,) = models.requests() + assert "" not in request.body["messages"][1]["content"] -class TestLLMJudgeGrader: - async def test_compute_score_returns_rubric_node(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "hud.utils.gateway.build_gateway_client", lambda provider: _FakeGatewayClient() - ) - subscore = await LLMJudgeGrader.compute_score( - answer="some answer", - criteria=["met: mentions Paris", ("names the river", 3.0)], - ) - assert subscore.value == pytest.approx(0.25) - assert subscore.children is not None - assert [(c.name, c.value, c.weight, c.info) for c in subscore.children] == [ - ("met: mentions Paris", 1.0, 1.0, {"reason": "because"}), - ("names the river", 0.0, 3.0, {"reason": "because"}), - ] +async def test_the_judge_without_criteria_scores_zero_without_calling_a_model( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") - async def test_compute_score_applies_negative_criterion_penalty( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr( - "hud.utils.gateway.build_gateway_client", lambda provider: _FakeGatewayClient() - ) - subscore = await LLMJudgeGrader.compute_score( - answer="some answer", - criteria=[("met: positive", 1.0), ("met: error", -0.5)], - ) - assert subscore.value == pytest.approx(0.5) + run = await _grade("judged_without_criteria", tmp_path) - async def test_grade_puts_verdicts_on_children(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "hud.utils.gateway.build_gateway_client", lambda provider: _FakeGatewayClient() - ) - subscore = await LLMJudgeGrader.grade( - weight=1.0, answer="some answer", criteria=["met: correct"] - ) - assert subscore.children == [ - SubScore( - name="met: correct", - value=1.0, - weight=1.0, - info={"reason": "because"}, - ) - ] - assert subscore.info is not None - assert subscore.info["model"] == "claude-haiku-4-5" - - -@pytest.mark.skipif(not _HAS_BASH, reason="/bin/bash not available (e.g. Windows)") -class TestBashGrader: - async def test_compute_score_requires_command(self) -> None: - with pytest.raises(ValueError, match="requires command"): - await BashGrader.compute_score() - - async def test_compute_score_for_passing_command(self) -> None: - subscore = await BashGrader.compute_score(command="echo hello") - assert subscore.value == 1.0 - assert subscore.info is not None - assert subscore.info["exit_code"] == 0 - assert "hello" in subscore.info["stdout"] - - async def test_compute_score_for_failing_command(self) -> None: - subscore = await BashGrader.compute_score(command="echo oops >&2 && false") - assert subscore.value == 0.0 - assert subscore.info is not None - assert subscore.info["exit_code"] != 0 - assert "oops" in subscore.info["stderr"] - - async def test_compute_score_timeout(self) -> None: - subscore = await BashGrader.compute_score(command="sleep 2", timeout_seconds=1) - assert subscore.value == 0.0 - assert subscore.info is not None - assert subscore.info["timed_out"] is True - assert subscore.info["timeout"] == 1 - - async def test_compute_score_timeout_keeps_partial_output( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - from hud.graders import bash as bash_mod - from hud.utils.process import ProcessResult - - process = SimpleNamespace( - complete=AsyncMock( - return_value=ProcessResult(None, b"progress\n", b"still working\n", timed_out=True) - ) - ) - monkeypatch.setattr(bash_mod, "create_process_group_exec", AsyncMock(return_value=process)) + (judge,) = _evaluated(run)["subscores"] + assert run.reward == 0.0 + assert judge["info"]["error"] == "no criteria provided" + assert models.requests() == [] - subscore = await BashGrader.compute_score(command="slow", timeout_seconds=1) - assert subscore.info is not None - assert subscore.info["stdout"] == "progress\n" - assert subscore.info["stderr"] == "still working\n" +async def test_the_judge_without_a_hud_api_key_fails_the_grade(tmp_path: Path) -> None: + run = await _grade("judged", tmp_path) - async def test_grade_and_combine_compose(self) -> None: - result = await combine( - BashGrader.grade(weight=0.5, command="true"), - BashGrader.grade(weight=0.5, command="false"), - ) - assert result.reward == pytest.approx(0.5) - assert result.subscores is not None - by_name = {subscore.name: subscore for subscore in result.subscores} - assert by_name["BashGrader-1"].info is not None - assert by_name["BashGrader-1"].info["exit_code"] == 0 - assert by_name["BashGrader-2"].info is not None - assert by_name["BashGrader-2"].info["exit_code"] != 0 - assert result.info == {} + assert run.trace.status == "error" + assert run.trace.error == IsStr(regex=r".*HUD_API_KEY is required for HUD gateway clients") From 7c1605ab26cbd5f8b1190a7196d248acc9173b9e Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 022/248] test: pin the HUD API request policy against the fake services make_request(_sync) and every PlatformClient verb now talk to the fake API: the /v2 prefix, repeated query keys, the bearer, doubling backoff on 502/503, a single attempt for 4xx, typed errors for unreachable, slow and unparseable responses, and the once-per-process deprecation warning. Through the CLI, hud models list pages the catalog and API errors become JSON error documents. test_platform.py and the exception-string tests fold into this scenario. --- tests/utils/test_platform.py | 88 ------ tests/utils/test_requests.py | 563 +++++++++++++++++++---------------- 2 files changed, 304 insertions(+), 347 deletions(-) delete mode 100644 tests/utils/test_platform.py diff --git a/tests/utils/test_platform.py b/tests/utils/test_platform.py deleted file mode 100644 index f767b31cb..000000000 --- a/tests/utils/test_platform.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Generic platform transport in ``hud.utils.platform``.""" - -from __future__ import annotations - -from typing import Any - -import pytest - -from hud.utils.exceptions import HudAuthenticationError -from hud.utils.platform import PlatformClient - - -def test_url_prefixes_version_segment_and_joins_params() -> None: - """Feature modules pass version-free paths; the client prepends the - canonical ``/v2`` namespace (the default ``api_prefix``).""" - platform = PlatformClient("https://api.example/", "key") - - assert platform.base_url == "https://api.example/v2" - assert platform.url("/tasks/upload") == "https://api.example/v2/tasks/upload" - assert platform.url("/registry/envs", {"limit": 5}) == ( - "https://api.example/v2/registry/envs?limit=5" - ) - - -def test_url_encodes_repeated_query_parameters() -> None: - """List-valued FastAPI query parameters are emitted as repeated keys.""" - platform = PlatformClient("https://api.example", "key") - - assert platform.url( - "/qa-agents/results/resources", - {"subject_type": "taskset", "subject_ids": ["first", "second"]}, - ) == ( - "https://api.example/v2/qa-agents/results/resources?" - "subject_type=taskset&subject_ids=first&subject_ids=second" - ) - - -def test_http_methods_route_through_shared_requests(monkeypatch: pytest.MonkeyPatch) -> None: - calls: list[dict[str, object]] = [] - - def fake_request( - method: str, url: str, json: object = None, **kwargs: object - ) -> dict[str, Any]: - calls.append({"method": method, "url": url, "json": json, "api_key": kwargs.get("api_key")}) - return {"ok": True} - - monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) - platform = PlatformClient("https://api.example", "key") - - assert platform.get("/x", params={"a": 1}) == {"ok": True} - assert platform.post("/y", json={"b": 2}) == {"ok": True} - assert platform.patch("/z", json={"c": 3}) == {"ok": True} - assert platform.delete("/z") == {"ok": True} - assert calls == [ - {"method": "GET", "url": "https://api.example/v2/x?a=1", "json": None, "api_key": "key"}, - {"method": "POST", "url": "https://api.example/v2/y", "json": {"b": 2}, "api_key": "key"}, - {"method": "PATCH", "url": "https://api.example/v2/z", "json": {"c": 3}, "api_key": "key"}, - {"method": "DELETE", "url": "https://api.example/v2/z", "json": None, "api_key": "key"}, - ] - - -def test_requests_without_api_key_raise_authentication_error() -> None: - platform = PlatformClient("https://api.example", "") - - with pytest.raises(HudAuthenticationError): - platform.get("/tasks") - - -def test_from_settings_prepends_canonical_version(monkeypatch: pytest.MonkeyPatch) -> None: - from hud import settings as settings_module - - monkeypatch.setattr(settings_module.settings, "hud_api_url", "https://api.example") - monkeypatch.setattr(settings_module.settings, "api_key", "key") - - platform = PlatformClient.from_settings() - assert platform.url("/tasks/upload") == "https://api.example/v2/tasks/upload" - - -@pytest.mark.parametrize("api_key", [None, ""]) -def test_from_settings_requires_api_key( - monkeypatch: pytest.MonkeyPatch, api_key: str | None -) -> None: - from hud import settings as settings_module - - monkeypatch.setattr(settings_module.settings, "api_key", api_key) - - with pytest.raises(HudAuthenticationError, match="HUD_API_KEY is required"): - PlatformClient.from_settings() diff --git a/tests/utils/test_requests.py b/tests/utils/test_requests.py index 49c32c009..9b03307f4 100644 --- a/tests/utils/test_requests.py +++ b/tests/utils/test_requests.py @@ -1,15 +1,17 @@ -"""Tests for the HTTP request utilities in the HUD API.""" +"""How the SDK talks to the HUD API: the platform client and the shared request policy.""" from __future__ import annotations +import time import uuid -from http import HTTPStatus +from itertools import pairwise from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, Mock, patch import httpx import pytest +from inline_snapshot import snapshot +from hud.utils import PlatformClient, make_request, make_request_sync from hud.utils.exceptions import ( HudAuthenticationError, HudDeprecationWarning, @@ -17,305 +19,348 @@ HudRequestError, HudTimeoutError, ) -from hud.utils.requests import ( - make_request, - make_request_sync, -) +from tests.harness import Reply if TYPE_CHECKING: - from collections.abc import Callable + from tests.harness import FakeServices, Hud, HudEnv, Request + + +async def _request(asynchronous: bool, url: str, **kwargs: Any) -> Any: + if asynchronous: + return await make_request("GET", url, **kwargs) + return make_request_sync("GET", url, **kwargs) + +SYNC_AND_ASYNC = pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) -@pytest.mark.parametrize("asynchronous", [False, True]) -async def test_requests_retry_transient_responses_with_shared_backoff(asynchronous): - calls = 0 - def handle(request): - nonlocal calls - calls += 1 - return httpx.Response(503 if calls < 3 else 200, json={"ok": True}) +@SYNC_AND_ASYNC +async def test_transient_responses_are_retried_with_doubling_backoff( + services: FakeServices, asynchronous: bool +) -> None: + arrivals: list[float] = [] + replies = [Reply(status=503), Reply(status=502), Reply(json={"ok": True})] + + def answer(request: Request) -> Reply: + del request + arrivals.append(time.monotonic()) + return replies[len(arrivals) - 1] + + services.route("api", "GET", "/v2/data", handler=answer) + + result = await _request( + asynchronous, f"{services.url('api')}/v2/data", api_key="k", retry_delay=0.1 + ) - transport = httpx.MockTransport(handle) - with ( - patch("hud.utils.requests.time.sleep") as sync_sleep, - patch("hud.utils.requests.asyncio.sleep", new_callable=AsyncMock) as async_sleep, - ): + assert result == {"ok": True} + gaps = [later - earlier for earlier, later in pairwise(arrivals)] + assert len(gaps) == 2 + assert gaps[0] >= 0.1 + assert gaps[1] >= 0.2 + + +@SYNC_AND_ASYNC +@pytest.mark.parametrize( + ("replies", "outcome", "attempts"), + [ + pytest.param([Reply(json={"id": "1"})], {"id": "1"}, 1, id="ok"), + pytest.param([Reply(status=204)], {}, 1, id="no-content"), + pytest.param( + [Reply(status=502), Reply(json={"id": "1"})], {"id": "1"}, 2, id="502-then-ok" + ), + pytest.param( + [Reply(status=404, json={"detail": "nope"})], + ("Request failed: nope", 404, {"detail": "nope"}), + 1, + id="404-detail", + ), + pytest.param( + [Reply(status=400, json={"code": "bad"})], + ( + "Request failed with status 400 - JSON response: {'code': 'bad'}", + 400, + {"code": "bad"}, + ), + 1, + id="400-without-detail", + ), + pytest.param( + [Reply(status=402, json={"detail": "no credits"})], + ("Request failed: no credits", 402, {"detail": "no credits"}), + 1, + id="402-once", + ), + pytest.param( + [Reply(status=429, json={"detail": "slow down"})], + ("Request failed: slow down", 429, {"detail": "slow down"}), + 1, + id="429-once", + ), + pytest.param( + [Reply(status=500, body="oops", content_type="text/plain")], + ("Request failed with status 500", 500, None), + 1, + id="500-once", + ), + ], +) +async def test_each_response_status_has_one_outcome( + services: FakeServices, + asynchronous: bool, + replies: list[Reply], + outcome: Any, + attempts: int, +) -> None: + services.route("api", "GET", "/v2/data", *replies) + url = f"{services.url('api')}/v2/data" + + if isinstance(outcome, tuple): + with pytest.raises(HudRequestError) as raised: + await _request(asynchronous, url, api_key="k", retry_delay=0.01) + message, status, body = outcome + assert (raised.value.message, raised.value.status_code) == (message, status) + assert raised.value.response_json == body + assert f"Status: {status}" in str(raised.value) + else: + assert await _request(asynchronous, url, api_key="k", retry_delay=0.01) == outcome + sent = services.requests("api", "GET", "/v2/data") + assert len(sent) == attempts + assert {request.headers["authorization"] for request in sent} == {"Bearer k"} + + +@SYNC_AND_ASYNC +@pytest.mark.parametrize( + ("url", "kwargs", "error", "message"), + [ + pytest.param( + "{api}/v2/data", + {"api_key": None}, + HudAuthenticationError, + "API key is required", + id="no-key", + ), + pytest.param( + "http://127.0.0.1:9/v2/data", + {"api_key": "k", "max_retries": 2, "retry_delay": 0.01}, + HudNetworkError, + "Network error", + id="unreachable", + ), + pytest.param( + "{api}/v2/slow", + {"api_key": "k"}, + HudTimeoutError, + "Request timed out", + id="timeout", + ), + pytest.param( + "{api}/v2/not-json", + {"api_key": "k"}, + HudRequestError, + "Unexpected error", + id="unparseable-body", + ), + ], +) +async def test_requests_that_never_get_an_answer_raise_typed_errors( + services: FakeServices, + asynchronous: bool, + url: str, + kwargs: dict[str, Any], + error: type[Exception], + message: str, +) -> None: + services.route("api", "GET", "/v2/slow", Reply(json={}, delay=2.0)) + services.route("api", "GET", "/v2/not-json", Reply(body="", content_type="text/html")) + target = url.format(api=services.url("api")) + timeout = httpx.Timeout(0.3) + + with pytest.raises(error, match=message): if asynchronous: - async with httpx.AsyncClient(transport=transport) as client: - result = await make_request( - "GET", "https://test/data", api_key="key", client=client - ) + async with httpx.AsyncClient(timeout=timeout) as client: + await make_request("GET", target, client=client, **kwargs) else: - with httpx.Client(transport=transport) as client: - result = make_request_sync("GET", "https://test/data", api_key="key", client=client) - sleep = async_sleep if asynchronous else sync_sleep - assert [call.args[0] for call in sleep.call_args_list] == [2.0, 4.0] - assert result == {"ok": True} - assert calls == 3 + with httpx.Client(timeout=timeout) as client: + make_request_sync("GET", target, client=client, **kwargs) -def test_requests_warn_once_per_deprecation_notice(): +def test_a_deprecation_notice_warns_once_per_process(services: FakeServices) -> None: successor = f"https://api.test/v2/{uuid.uuid4()}" headers = { "Deprecation": "@1790985600", "Sunset": "Sat, 10 Oct 2026 00:00:00 GMT", "Link": f'<{successor}>; rel="successor-version"', } - transport = httpx.MockTransport(lambda request: httpx.Response(200, json={}, headers=headers)) - with httpx.Client(transport=transport) as client, pytest.warns(HudDeprecationWarning) as caught: - for url in ("https://api.test/v2/old/1", "https://api.test/v2/old/2"): - make_request_sync("GET", url, api_key="key", client=client) + services.route("api", "GET", "/v2/old/{id}", Reply(json={}, headers=headers)) + + with pytest.warns(HudDeprecationWarning) as caught: + for item in ("1", "2"): + make_request_sync("GET", f"{services.url('api')}/v2/old/{item}", api_key="k") assert [str(warning.message) for warning in caught] == [ - "GET /v2/old/1 is deprecated by the HUD API and stops being served on " + "GET /api/v2/old/1 is deprecated by the HUD API and stops being served on " f"Sat, 10 Oct 2026 00:00:00 GMT; its replacement is {successor}. " "If a hud command printed this, upgrade hud." ] -def _create_mock_response( - status_code: int = 200, - json_data: dict[str, Any] | None = None, - raise_exception: Exception | None = None, -) -> Callable[[httpx.Request], httpx.Response]: - """Create a mock response handler for httpx.MockTransport.""" - - def handler(request: httpx.Request) -> httpx.Response: - if "Authorization" not in request.headers: - return httpx.Response(HTTPStatus.UNAUTHORIZED, json={"error": "Unauthorized"}) - - if raise_exception: - raise raise_exception - - return httpx.Response(status_code, json=json_data or {"result": "success"}, request=request) - - return handler - - -@pytest.mark.asyncio -async def test_make_request_success(): - """Test successful async request.""" - expected_data = {"id": "123", "name": "test"} - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(200, expected_data)) - ) - result = await make_request( - "GET", "https://api.test.com/data", api_key="test-key", client=async_client - ) - assert result == expected_data - - -@pytest.mark.asyncio -async def test_make_request_no_api_key(): - """Test request without API key.""" - with pytest.raises(HudAuthenticationError): - await make_request("GET", "https://api.test.com/data", api_key=None) - - -@pytest.mark.asyncio -async def test_make_request_http_error(): - """Test HTTP error handling.""" - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(404, {"error": "Not found"})) - ) - - with pytest.raises(HudRequestError) as excinfo: - await make_request( - "GET", "https://api.test.com/data", api_key="test-key", client=async_client - ) - - assert "404" in str(excinfo.value) - - -@pytest.mark.asyncio -async def test_make_request_network_error(): - """Test network error handling with retry exhaustion.""" - request_error = httpx.RequestError( - "Connection error", request=httpx.Request("GET", "https://api.test.com") - ) - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(raise_exception=request_error)) - ) - - with patch("hud.utils.requests.asyncio.sleep", AsyncMock()) as mock_retry: - mock_retry.return_value = None - - with pytest.raises(HudNetworkError) as excinfo: - await make_request( - "GET", - "https://api.test.com/data", - api_key="test-key", - max_retries=2, - retry_delay=0.01, - client=async_client, - ) - - assert "Connection error" in str(excinfo.value) - - -@pytest.mark.asyncio -async def test_make_request_timeout(): - """Test timeout error handling.""" - timeout_error = httpx.TimeoutException( - "Request timed out", request=httpx.Request("GET", "https://api.test.com") - ) - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(raise_exception=timeout_error)) - ) +# ─── the platform client ───────────────────────────────────────────────────── - with pytest.raises(HudTimeoutError) as excinfo: - await make_request( - "GET", "https://api.test.com/data", api_key="test-key", client=async_client - ) - assert "timed out" in str(excinfo.value) +async def test_the_platform_client_sends_each_verb_under_v2_with_the_bearer( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k") + for method in ("GET", "POST", "PUT", "PATCH", "DELETE"): + services.route("api", method, "/v2/{rest:path}", json={"method": method}) + platform = PlatformClient.from_settings() + answers = [ + platform.get("/tasks", params={"limit": 5, "ids": ["a", "b"]}), + platform.post("/tasks", json={"b": 2}), + platform.put("/tasks/1", json={"c": 3}), + platform.patch("/tasks/1", json={"d": 4}), + platform.delete("/tasks/1"), + await platform.aget("/tasks", params={"limit": 1}), + await platform.apost("/tasks", json={"e": 5}), + ] -@pytest.mark.asyncio -async def test_make_request_unexpected_error(): - """Test handling of unexpected errors.""" - unexpected_error = ValueError("Unexpected error") - async_client = httpx.AsyncClient( - transport=httpx.MockTransport(_create_mock_response(raise_exception=unexpected_error)) + assert [answer["method"] for answer in answers] == [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE", + "GET", + "POST", + ] + assert [ + (request.method, request.path, request.query, request.json, request.bearer) + for request in services.requests("api") + ] == snapshot( + [ + ("GET", "/v2/tasks", {"limit": ["5"], "ids": ["a", "b"]}, None, "k"), + ("POST", "/v2/tasks", {}, {"b": 2}, "k"), + ("PUT", "/v2/tasks/1", {}, {"c": 3}, "k"), + ("PATCH", "/v2/tasks/1", {}, {"d": 4}, "k"), + ("DELETE", "/v2/tasks/1", {}, None, "k"), + ("GET", "/v2/tasks", {"limit": ["1"]}, None, "k"), + ("POST", "/v2/tasks", {}, {"e": 5}, "k"), + ] ) - with pytest.raises(HudRequestError) as excinfo: - await make_request( - "GET", "https://api.test.com/data", api_key="test-key", client=async_client - ) - - assert "Unexpected error" in str(excinfo.value) - -@pytest.mark.asyncio -async def test_make_request_auto_client_creation(): - """Test automatic client creation when not provided.""" - with patch("hud.utils.requests._create_default_async_client") as mock_create_client: - mock_client = AsyncMock() - mock_client.request.return_value = httpx.Response( - 200, json={"result": "success"}, request=httpx.Request("GET", "https://api.test.com") - ) - mock_client.aclose = AsyncMock() - mock_create_client.return_value = mock_client - result = await make_request("GET", "https://api.test.com/data", api_key="test-key") - - assert result == {"result": "success"} - mock_client.aclose.assert_awaited_once() - - -def test_make_request_sync_success(): - """Test successful sync request.""" - expected_data = {"id": "123", "name": "test"} - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(200, expected_data)) - ) +@pytest.mark.parametrize("api_key", [None, ""]) +def test_the_platform_client_needs_an_api_key(hud_env: HudEnv, api_key: str | None) -> None: + hud_env.set(HUD_API_KEY=api_key) - result = make_request_sync( - "GET", "https://api.test.com/data", api_key="test-key", client=sync_client - ) + with pytest.raises(HudAuthenticationError, match="HUD_API_KEY is required"): + PlatformClient.from_settings() - assert result == expected_data +def test_a_platform_client_built_without_a_key_refuses_to_send(services: FakeServices) -> None: + platform = PlatformClient(services.url("api"), "") -def test_make_request_sync_no_api_key(): - """Test sync request without API key.""" with pytest.raises(HudAuthenticationError): - make_request_sync("GET", "https://api.test.com/data", api_key=None) - - -def test_make_request_sync_http_error(): - """Test HTTP error handling.""" - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(404, {"error": "Not found"})) - ) - with pytest.raises(HudRequestError) as excinfo: - make_request_sync( - "GET", "https://api.test.com/data", api_key="test-key", client=sync_client - ) - - assert "404" in str(excinfo.value) - - -def test_make_request_sync_network_error(): - """Test network error handling with retry exhaustion.""" - request_error = httpx.RequestError( - "Connection error", request=httpx.Request("GET", "https://api.test.com") - ) - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(raise_exception=request_error)) - ) - with patch("time.sleep", lambda _: None): - with pytest.raises(HudNetworkError) as excinfo: - make_request_sync( - "GET", - "https://api.test.com/data", - api_key="test-key", - max_retries=2, - retry_delay=0.01, - client=sync_client, - ) - - assert "Connection error" in str(excinfo.value) - - -def test_make_request_sync_timeout(): - """Test timeout error handling.""" - timeout_error = httpx.TimeoutException( - "Request timed out", request=httpx.Request("GET", "https://api.test.com") - ) - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(raise_exception=timeout_error)) - ) - with pytest.raises(HudTimeoutError) as excinfo: - make_request_sync( - "GET", "https://api.test.com/data", api_key="test-key", client=sync_client - ) - - assert "timed out" in str(excinfo.value) + platform.get("/tasks") + assert services.requests() == [] -def test_make_request_sync_unexpected_error(): - """Test handling of unexpected errors.""" - unexpected_error = ValueError("Unexpected error") - sync_client = httpx.Client( - transport=httpx.MockTransport(_create_mock_response(raise_exception=unexpected_error)) - ) - - with pytest.raises(HudRequestError) as excinfo: - make_request_sync( - "GET", "https://api.test.com/data", api_key="test-key", client=sync_client - ) +# ─── through the CLI ───────────────────────────────────────────────────────── - assert "Unexpected error" in str(excinfo.value) +def test_hud_models_list_reads_every_catalog_page( + services: FakeServices, hud_env: HudEnv, hud: Hud +) -> None: + hud_env.set(HUD_API_KEY="k") + rows = [{"id": f"id-{index:03}", "name": f"Model {index:03}"} for index in range(103)] -def test_make_request_sync_auto_client_creation(): - """Test automatic client creation when not provided.""" - with patch("hud.utils.requests._create_default_sync_client") as mock_create_client: - mock_client = Mock() - mock_client.request.return_value = httpx.Response( - 200, json={"result": "success"}, request=httpx.Request("GET", "https://api.test.com") - ) - mock_client.close = Mock() - mock_create_client.return_value = mock_client + def page(request: Request) -> Reply: + offset, limit = int(request.query["offset"][0]), int(request.query["limit"][0]) + return Reply(json={"items": rows[offset : offset + limit], "total": len(rows)}) - result = make_request_sync("GET", "https://api.test.com/data", api_key="test-key") + services.route("api", "GET", "/v2/models", handler=page) - assert result == {"result": "success"} - mock_client.close.assert_called_once() + result = hud("models", "list", "--json") + assert result.exit_code == 0, result + assert [model["id"] for model in result.json] == [row["id"] for row in rows] + assert [request.query for request in services.requests("api", "GET", "/v2/models")] == [ + {"limit": ["100"], "offset": ["0"]}, + {"limit": ["100"], "offset": ["100"]}, + ] -def test_make_request_sync_empty_204() -> None: - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(204, request=request) - sync_client = httpx.Client(transport=httpx.MockTransport(handler)) - assert ( - make_request_sync( - "DELETE", - "https://api.test.com/data", - api_key="test-key", - client=sync_client, - ) - == {} +@pytest.mark.parametrize( + ("reply", "exit_code", "document"), + [ + pytest.param( + Reply(status=404, json={"detail": "nope"}), + 1, + snapshot( + { + "error": "not_found", + "message": "Request failed: nope", + "suggestion": "Check the resource id, or list existing ones.", + } + ), + id="not-found", + ), + pytest.param( + Reply(status=429, json={"detail": "slow down"}), + 1, + snapshot( + { + "error": "rate_limited", + "message": "Request failed: slow down", + "suggestion": "Retry after a short delay.", + } + ), + id="rate-limited", + ), + pytest.param( + Reply(status=403, json={"detail": "forbidden"}), + 1, + snapshot( + { + "error": "permission_denied", + "message": "Request failed: forbidden", + "suggestion": "Check that this API key can access the resource.", + } + ), + id="permission-denied", + ), + ], +) +def test_an_api_error_becomes_a_json_error_document( + services: FakeServices, + hud_env: HudEnv, + hud: Hud, + reply: Reply, + exit_code: int, + document: dict[str, Any], +) -> None: + hud_env.set(HUD_API_KEY="k") + services.route("api", "GET", "/v2/jobs", reply) + + result = hud("jobs", "list", "--json") + + assert result.exit_code == exit_code, result + assert result.json == document + assert len(services.requests("api", "GET", "/v2/jobs")) == 1 + + +def test_a_deprecated_route_prints_one_warning_line( + services: FakeServices, hud_env: HudEnv, hud: Hud +) -> None: + hud_env.set(HUD_API_KEY="k") + headers = {"Deprecation": "@1790985600", "Sunset": "Sat, 10 Oct 2026 00:00:00 GMT"} + services.route("api", "GET", "/v2/jobs", Reply(json={"items": []}, headers=headers)) + + result = hud("jobs", "list", "--json") + + assert result.exit_code == 0, result + assert " ".join(result.stderr.split()) == ( + "⚠ GET /api/v2/jobs is deprecated by the HUD API and stops being served on " + "Sat, 10 Oct 2026 00:00:00 GMT. If a hud command printed this, upgrade hud." ) From 49773c2710f181a6e6ee037e0b6aa82c276d8a0d Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 023/248] test: observe model routing as the requests that reach each server Each agent family runs a real rollout with a provider key, the HUD key alone, or a forced gateway, against the scripted gateway and a second fake standing in for the provider's own API; the test asserts which server answered and with which credential. Bedrock profiles reach the Bedrock endpoint signed with AWS credentials, unroutable models are refused when the agent is built, gateway requests carry their run's Trace-Id, and a nested rollout names its parent trace. The tests that patched hud.utils.gateway and asyncio.sleep are removed. --- tests/utils/conftest.py | 27 +++ tests/utils/test_gateway.py | 462 ++++++++++++++---------------------- 2 files changed, 205 insertions(+), 284 deletions(-) create mode 100644 tests/utils/conftest.py diff --git a/tests/utils/conftest.py b/tests/utils/conftest.py new file mode 100644 index 000000000..73ce899f5 --- /dev/null +++ b/tests/utils/conftest.py @@ -0,0 +1,27 @@ +"""A model provider reached directly, apart from the HUD gateway.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from tests.harness import FakeServices, Models + +if TYPE_CHECKING: + from collections.abc import Iterator + + +@pytest.fixture(scope="session") +def provider_server() -> Iterator[FakeServices]: + server = FakeServices() + server.start() + yield server + server.stop() + + +@pytest.fixture +def provider(provider_server: FakeServices) -> Models: + """Scripted models served as a provider's own API (not behind the HUD gateway).""" + provider_server.reset() + return Models(provider_server) diff --git a/tests/utils/test_gateway.py b/tests/utils/test_gateway.py index f452994f6..3524f6d82 100644 --- a/tests/utils/test_gateway.py +++ b/tests/utils/test_gateway.py @@ -1,319 +1,213 @@ +"""Which server a model request reaches, with which credentials and trace headers.""" + from __future__ import annotations -import asyncio -from functools import partial -from typing import TYPE_CHECKING, cast -from unittest.mock import AsyncMock, MagicMock +from typing import TYPE_CHECKING, Any -import httpx import pytest -from hud.settings import settings -from hud.telemetry.context import set_trace_context -from hud.utils import gateway +from hud import Environment +from hud.agents import ClaudeAgent, GeminiAgent, OpenAIAgent, OpenAIChatAgent +from hud.agents.base import Agent +from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig +from hud.eval import LocalRuntime, Task, Taskset, rollout from hud.utils.exceptions import HudAuthenticationError +from tests.harness import Reply, say if TYPE_CHECKING: - from google.genai import Client as GenaiClient - from openai import AsyncOpenAI - - -@pytest.fixture(autouse=True) -def _gateway_settings(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", "sk-hud-test") - monkeypatch.setattr(settings, "hud_gateway_url", "https://gateway.test") - - -@pytest.mark.asyncio -async def test_openai_client_resolves_trace_id_per_request( - monkeypatch: pytest.MonkeyPatch, -) -> None: - seen_headers: dict[str, str] = {} - - async def handler(request: httpx.Request) -> httpx.Response: - trace_id = request.headers["Trace-Id"] - await asyncio.sleep(0) - seen_headers[request.url.path] = trace_id - return httpx.Response(200, json={"object": "list", "data": []}) - - transport = httpx.MockTransport(handler) - real_client_factory = gateway.DefaultAsyncHttpxClient - monkeypatch.setattr( - gateway, - "DefaultAsyncHttpxClient", - partial(real_client_factory, transport=transport), - ) - client = cast("AsyncOpenAI", gateway.build_gateway_client("openai")) - - async def request_in_trace(trace_id: str) -> None: - with set_trace_context(trace_id): - await client.get(f"/models/{trace_id}", cast_to=object) - - try: - await asyncio.gather( - request_in_trace("11111111-1111-4111-8111-111111111111"), - request_in_trace("22222222-2222-4222-8222-222222222222"), - ) - finally: - await client.close() - - assert seen_headers == { - "/models/11111111-1111-4111-8111-111111111111": ("11111111-1111-4111-8111-111111111111"), - "/models/22222222-2222-4222-8222-222222222222": ("22222222-2222-4222-8222-222222222222"), - } - - -@pytest.mark.asyncio -async def test_openai_client_trace_context_overrides_empty_header( - monkeypatch: pytest.MonkeyPatch, -) -> None: - seen_trace_id: str | None = None - - async def handler(request: httpx.Request) -> httpx.Response: - nonlocal seen_trace_id - seen_trace_id = request.headers["Trace-Id"] - return httpx.Response(200, json={"object": "list", "data": []}) - - transport = httpx.MockTransport(handler) - real_client_factory = gateway.DefaultAsyncHttpxClient - monkeypatch.setattr( - gateway, - "DefaultAsyncHttpxClient", - partial(real_client_factory, transport=transport), + from collections.abc import Callable + + from hud.eval import Run + from tests.harness import FakeServices, HudEnv, Models + +BEDROCK_ARN = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/anthropic.claude" +AWS = { + "AWS_ACCESS_KEY_ID": "AKIATEST", + "AWS_SECRET_ACCESS_KEY": "secret", + "AWS_REGION": "us-east-1", +} + + +def _env() -> Environment: + env = Environment("routing") + + @env.template() + async def answer(): + reply = yield "Reply with ok." + yield 1.0 if reply == "ok" else 0.0 + + return env + + +TASK = Task(env="routing", id="answer") + +AGENTS: dict[str, Callable[[bool], Agent]] = { + "openai": lambda gateway: OpenAIAgent(OpenAIConfig(model="m", max_steps=2, gateway=gateway)), + "anthropic": lambda gateway: ClaudeAgent( + ClaudeConfig(model="claude-m", max_steps=2, gateway=gateway) + ), + "gemini": lambda gateway: GeminiAgent( + GeminiConfig(model="gemini-m", max_steps=2, gateway=gateway) + ), +} +PROVIDER_KEYS = { + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "gemini": "GEMINI_API_KEY", +} +AUTH_HEADERS = {"openai": "authorization", "anthropic": "x-api-key", "gemini": "x-goog-api-key"} + + +def _point_provider_sdks_at(hud_env: HudEnv, provider: Models) -> None: + hud_env.set( + OPENAI_BASE_URL=provider.url, + ANTHROPIC_BASE_URL=provider.url, + GOOGLE_GEMINI_BASE_URL=provider.url, ) - client = cast("AsyncOpenAI", gateway.build_gateway_client("openai")) - trace_id = "11111111-1111-4111-8111-111111111111" - - try: - with set_trace_context(trace_id): - await client.get( - "/models/explicit", - cast_to=object, - options={"headers": {"Trace-Id": ""}}, - ) - finally: - await client.close() - assert seen_trace_id == trace_id - -@pytest.mark.asyncio -async def test_openai_client_sends_child_and_parent_trace_ids( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize( + ("family", "provider_key", "gateway", "reached"), + [ + pytest.param(family, key, gateway, reached, id=f"{family}-{label}") + for family in AGENTS + for key, gateway, reached, label in [ + (True, False, "provider", "own-key"), + (False, False, "gateway", "hud-key-only"), + (True, True, "gateway", "forced-gateway"), + ] + ], +) +async def test_a_provider_key_routes_direct_unless_the_gateway_is_forced( + models: Models, + provider: Models, + hud_env: HudEnv, + family: str, + provider_key: bool, + gateway: bool, + reached: str, ) -> None: - seen_headers: dict[str, str] = {} - - async def handler(request: httpx.Request) -> httpx.Response: - seen_headers["Trace-Id"] = request.headers["Trace-Id"] - seen_headers["X-HUD-Parent-Trace-Id"] = request.headers["X-HUD-Parent-Trace-Id"] - return httpx.Response(200, json={"object": "list", "data": []}) - - transport = httpx.MockTransport(handler) - real_client_factory = gateway.DefaultAsyncHttpxClient - monkeypatch.setattr( - gateway, - "DefaultAsyncHttpxClient", - partial(real_client_factory, transport=transport), + _point_provider_sdks_at(hud_env, provider) + hud_env.set( + HUD_API_KEY="hud-key", **{PROVIDER_KEYS[family]: "own-key" if provider_key else None} ) - client = cast("AsyncOpenAI", gateway.build_gateway_client("openai")) + models.script([say("ok")]) + provider.script([say("ok")]) - try: - with set_trace_context("child", parent_trace_id="parent"): - await client.get("/models/nested", cast_to=object) - finally: - await client.close() + run = await rollout(TASK, AGENTS[family](gateway), runtime=LocalRuntime(_env())) - assert seen_headers == { - "Trace-Id": "child", - "X-HUD-Parent-Trace-Id": "parent", + served = {"gateway": models.requests(), "provider": provider.requests()} + assert run.reward == 1.0 + assert {name: len(requests) for name, requests in served.items()} == { + name: int(name == reached) for name in served } + (request,) = served[reached] + credential = request.headers[AUTH_HEADERS[family]].removeprefix("Bearer ") + assert credential == ("hud-key" if reached == "gateway" else "own-key") + + +@pytest.mark.parametrize( + ("build", "settings", "error", "message"), + [ + pytest.param( + lambda: OpenAIAgent(OpenAIConfig(model="m")), + {}, + HudAuthenticationError, + "No API key for openai", + id="no-key-at-all", + ), + pytest.param( + lambda: ClaudeAgent(ClaudeConfig(model=BEDROCK_ARN)), + {"HUD_API_KEY": "hud-key"}, + HudAuthenticationError, + "AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_REGION are required", + id="bedrock-without-aws-credentials", + ), + pytest.param( + lambda: ClaudeAgent(ClaudeConfig(model=BEDROCK_ARN, gateway=True)), + {"HUD_API_KEY": "hud-key", **AWS}, + ValueError, + "is a Bedrock inference profile; it cannot use the HUD gateway", + id="bedrock-through-the-gateway", + ), + ], +) +def test_an_unroutable_model_is_refused_when_the_agent_is_built( + hud_env: HudEnv, + build: Callable[[], Agent], + settings: dict[str, str], + error: type[Exception], + message: str, +) -> None: + hud_env.set(**settings) - -_BEDROCK_ARN = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/anthropic.claude" - - -def test_list_gateway_models_reads_every_page(monkeypatch: pytest.MonkeyPatch) -> None: - gateway.list_gateway_models.cache_clear() - rows = [{"id": f"m{i}", "model_name": f"m{i}"} for i in range(103)] - - def get(path: str, *, params: dict[str, int]) -> dict[str, object]: - assert path == "/models" - start = params["offset"] - return {"items": rows[start : start + params["limit"]], "total": len(rows)} - - platform = MagicMock() - platform.get.side_effect = get - monkeypatch.setattr(gateway.PlatformClient, "from_settings", lambda: platform) - try: - models = gateway.list_gateway_models() - finally: - gateway.list_gateway_models.cache_clear() - - assert [m.model_name for m in models] == [f"m{i}" for i in range(103)] - assert platform.get.call_count == 2 + with pytest.raises(error, match=message): + build() -def test_model_recency_prefers_release_over_catalog_date() -> None: - released = gateway.GatewayModelInfo( - id="released", created_at="2026-09-01T00:00:00Z", released_at="2026-01-01T00:00:00Z" - ) - added = gateway.GatewayModelInfo(id="added", created_at="2026-06-01T00:00:00Z") - undated = gateway.GatewayModelInfo(id="undated") - assert sorted([added, released, undated], key=lambda m: m.recency) == [ - undated, - released, - added, - ] - - -def test_bedrock_arn_model_gets_a_bedrock_client(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "aws_access_key_id", "AKIATEST") - monkeypatch.setattr(settings, "aws_secret_access_key", "secret") - monkeypatch.setattr(settings, "aws_region", "us-east-1") - bedrock = MagicMock(return_value=object()) - monkeypatch.setattr("anthropic.AsyncAnthropicBedrock", bedrock) - gateway_client = MagicMock() - monkeypatch.setattr(gateway, "build_gateway_client", gateway_client) - - client = gateway.build_model_client("anthropic", model=_BEDROCK_ARN) - - assert client is bedrock.return_value - bedrock.assert_called_once_with( - aws_access_key="AKIATEST", aws_secret_key="secret", aws_region="us-east-1" +async def test_a_bedrock_profile_is_sent_to_bedrock_signed_with_aws_credentials( + models: Models, provider_server: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="hud-key", ANTHROPIC_BEDROCK_BASE_URL=provider_server.url("api"), **AWS) + provider_server.reset() + provider_server.route("api", "POST", "/model/{rest:path}", Reply(status=400, json={})) + + run = await rollout( + TASK, + ClaudeAgent(ClaudeConfig(model=BEDROCK_ARN, max_steps=1)), + runtime=LocalRuntime(_env()), ) - gateway_client.assert_not_called() - -def test_bedrock_arn_model_requires_aws_credentials(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "aws_access_key_id", None) - monkeypatch.setattr(settings, "aws_secret_access_key", None) - monkeypatch.setattr(settings, "aws_region", None) + (request, *_) = provider_server.requests("api", "POST") + assert run.reward == 0.0 + assert models.requests() == [] + assert request.params["rest"] == f"{BEDROCK_ARN}/invoke" + assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 Credential=AKIATEST/") - with pytest.raises(HudAuthenticationError, match="AWS Bedrock"): - gateway.build_model_client("anthropic", model=_BEDROCK_ARN) - -def test_bedrock_arn_model_cannot_be_forced_through_the_gateway( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize("family", list(AGENTS)) +async def test_each_gateway_request_carries_its_own_runs_trace_id( + models: Models, hud_env: HudEnv, family: str ) -> None: - monkeypatch.setattr(settings, "aws_access_key_id", "AKIATEST") - monkeypatch.setattr(settings, "aws_secret_access_key", "secret") - monkeypatch.setattr(settings, "aws_region", "us-east-1") - bedrock = MagicMock() - monkeypatch.setattr("anthropic.AsyncAnthropicBedrock", bedrock) - - with pytest.raises(ValueError, match="cannot use the HUD gateway"): - gateway.build_model_client("anthropic", model=_BEDROCK_ARN, gateway=True) - bedrock.assert_not_called() - - -def test_provider_key_wins_over_hud_key(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "anthropic_api_key", "sk-ant-test") - direct = MagicMock(return_value=object()) - monkeypatch.setattr("anthropic.AsyncAnthropic", direct) - gateway_client = MagicMock() - monkeypatch.setattr(gateway, "build_gateway_client", gateway_client) + hud_env.set(HUD_API_KEY="hud-key") + models.script([say("ok")]) - client = gateway.build_model_client("anthropic", model="claude-sonnet-4-6") - - assert client is direct.return_value - direct.assert_called_once_with(api_key="sk-ant-test") - gateway_client.assert_not_called() - - -def test_hud_key_alone_routes_through_gateway(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "anthropic_api_key", None) - gateway_client = MagicMock(return_value=object()) - monkeypatch.setattr(gateway, "build_gateway_client", gateway_client) - - client = gateway.build_model_client("anthropic", model="claude-sonnet-4-6") + job = await Taskset("t", [TASK]).run( + AGENTS[family](True), runtime=LocalRuntime(_env()), group=3 + ) - assert client is gateway_client.return_value - gateway_client.assert_called_once_with("anthropic") + assert sorted(request.headers["trace-id"] for request in models.requests()) == sorted( + str(run.trace_id) for run in job.runs + ) + assert all("x-hud-parent-trace-id" not in r.headers for r in models.requests()) -def test_no_key_at_all_is_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", None) - monkeypatch.setattr(settings, "openai_api_key", None) - with pytest.raises(HudAuthenticationError, match="No API key for openai"): - gateway.build_model_client("openai") +class Delegating(Agent): + """Answers by running a nested rollout, as a subagent would.""" + def __init__(self) -> None: + super().__init__() + self.inner: list[Run] = [] -@pytest.mark.asyncio -async def test_anthropic_client_receives_trace_aware_http_client( - monkeypatch: pytest.MonkeyPatch, -) -> None: - client_factory = MagicMock(return_value=object()) - monkeypatch.setattr("anthropic.AsyncAnthropic", client_factory) + async def __call__(self, run: Run) -> None: + agent = OpenAIChatAgent(OpenAIChatConfig(model="m", max_steps=2)) + inner = await rollout(TASK, agent, runtime=LocalRuntime(_env())) + self.inner.append(inner) + run.trace.content = inner.trace.content - gateway.build_gateway_client("anthropic") - kwargs = client_factory.call_args.kwargs - http_client = kwargs["http_client"] - assert http_client.event_hooks["request"] - await http_client.aclose() +async def test_a_nested_rollout_names_its_parent_trace(models: Models, hud_env: HudEnv) -> None: + hud_env.set(HUD_API_KEY="hud-key") + models.script([say("ok")]) + agent = Delegating() + outer = await rollout(TASK, agent, runtime=LocalRuntime(_env())) -@pytest.mark.asyncio -@pytest.mark.parametrize("initial_status", [200, 429, 503]) -async def test_gemini_async_request_includes_trace_id( - monkeypatch: pytest.MonkeyPatch, - initial_status: int, -) -> None: - sleep = AsyncMock() - monkeypatch.setattr(asyncio, "sleep", sleep) - seen_trace_id: str | None = None - attempts = 0 - - async def handler(request: httpx.Request) -> httpx.Response: - nonlocal seen_trace_id, attempts - seen_trace_id = request.headers["Trace-Id"] - attempts += 1 - if attempts == 1 and initial_status != 200: - return httpx.Response( - initial_status, json={"error": {"code": initial_status, "message": "busy"}} - ) - return httpx.Response( - 200, - json={ - "candidates": [ - { - "content": {"parts": [{"text": "ok"}], "role": "model"}, - "finishReason": "STOP", - "index": 0, - } - ], - "modelVersion": "gemini-test", - "usageMetadata": { - "candidatesTokenCount": 1, - "promptTokenCount": 1, - "totalTokenCount": 2, - }, - }, - ) - - monkeypatch.setattr( - gateway.httpx, - "AsyncHTTPTransport", - MagicMock(return_value=httpx.MockTransport(handler)), + (request,) = models.requests() + (inner,) = agent.inner + headers: dict[str, Any] = request.headers + assert outer.reward == 1.0 + assert (headers["trace-id"], headers["x-hud-parent-trace-id"]) == ( + str(inner.trace_id), + str(outer.trace_id), ) - client = cast("GenaiClient", gateway.build_gateway_client("gemini")) - trace_id = "11111111-1111-4111-8111-111111111111" - - try: - with set_trace_context(trace_id): - response = await client.aio.models.generate_content( - model="gemini-test", - contents="hi", - ) - finally: - await client.aio.aclose() - - assert response.text == "ok" - assert seen_trace_id == trace_id - assert attempts == (1 if initial_status == 200 else 2) - if initial_status != 200: - sleep.assert_awaited_once() - assert 10 <= sleep.await_args_list[0].args[0] <= 11 - else: - sleep.assert_not_awaited() From b93b9059ccdb0a79af9ded4a735c2e03b99d9d7e Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 024/248] test: pin the training wire against the fake catalog and RL service Runs come from real rollouts: a trainable model behind the gateway returns token ids, so forward_backward sends inline trajectories, and plain runs train by trace id. The scenario pins the request bodies, the catalog lookup made once with the client's own key, a single optim-step POST even on 502/503/504, the custom-loss backward gradients, the no-gradient warning, and that untrainable batches are refused before any request. --- tests/train/test_client.py | 391 +++++++++++++++++++++++++++++-------- 1 file changed, 308 insertions(+), 83 deletions(-) diff --git a/tests/train/test_client.py b/tests/train/test_client.py index e166a8bff..e56c32698 100644 --- a/tests/train/test_client.py +++ b/tests/train/test_client.py @@ -1,112 +1,337 @@ +"""What TrainingClient sends to the HUD training service, and what it refuses to send.""" + from __future__ import annotations +import itertools import logging -from typing import Any +from typing import TYPE_CHECKING, Any import pytest +import torch +from dirty_equals import IsStr +from inline_snapshot import snapshot + +from hud import Environment +from hud.agents import OpenAIChatAgent +from hud.agents.types import OpenAIChatConfig +from hud.eval import LocalRuntime, Run, Task, Taskset +from hud.train import DatumTensors, TrainingClient +from hud.utils.exceptions import HudRequestError +from tests.harness import Reply, ScriptedAgent + +if TYPE_CHECKING: + from hud.agents.base import Agent + from tests.harness import FakeServices, HudEnv, Request + +MODEL_ID = "00000000-0000-4000-8000-000000000001" +TRAIN = f"/v1/models/{MODEL_ID}/train" +OPTIM = { + "step": 1, + "checkpoint_id": "ckpt-1", + "sampler_path": "s", + "state_path": "t", + "model": "owner/model", +} +TRACE_ID = IsStr(regex=r"[0-9a-f]{32}") + + +def _env() -> Environment: + env = Environment("train") + + @env.template() + async def answer(target: str = "ok"): + reply = yield "Reply with ok." + yield 1.0 if reply == target else 0.0 + + return env + + +@pytest.fixture +def service(services: FakeServices, hud_env: HudEnv) -> FakeServices: + """The model catalog and the training service for ``owner/model``.""" + hud_env.set(HUD_API_KEY="hud-key") + services.route( + "api", + "GET", + "/v2/models", + json={"items": [{"id": MODEL_ID, "model_name": "owner/model"}], "total": 1}, + ) + services.route("rl", "POST", f"{TRAIN}/forward-backward", json={"metrics": {}, "num_datums": 2}) + services.route("rl", "POST", f"{TRAIN}/optim-step", json=OPTIM) + return services + + +def _sampling_gateway(services: FakeServices) -> None: + """A trainable model behind the gateway: every reply carries token ids and logprobs.""" + turns = itertools.count() + + def complete(request: Request) -> Reply: + assert request.json["return_token_ids"] is True + content = "ok" if next(turns) % 2 == 0 else "no" + return Reply( + json={ + "id": "c", + "object": "chat.completion", + "created": 0, + "model": "owner/model", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": content}, + "logprobs": { + "content": [ + {"token": content, "logprob": -0.5, "bytes": [], "top_logprobs": []} + ] + }, + "prompt_token_ids": [1, 2, 3], + "token_ids": [7], + } + ], + } + ) + + services.route("gateway", "POST", "/chat/completions", handler=complete) + + +async def _runs(agent: Agent, *, tasks: list[Task] | None = None, group: int = 2) -> list[Run]: + job = await Taskset("t", tasks or [Task(env="train", id="answer")]).run( + agent, runtime=LocalRuntime(_env()), group=group, max_concurrent=1 + ) + return job.runs + + +def _trainable_agent() -> OpenAIChatAgent: + return OpenAIChatAgent( + OpenAIChatConfig( + model="owner/model", + max_steps=1, + completion_kwargs={"extra_body": {"return_token_ids": True}}, + ) + ) + + +async def test_a_step_sends_sampled_runs_inline_then_one_optimizer_step( + service: FakeServices, +) -> None: + _sampling_gateway(service) + runs = await _runs(_trainable_agent()) + client = TrainingClient("owner/model", api_key="team-key") + + result = await client.step(runs, learning_rate=1e-5, group_size=2) + + assert result.checkpoint_id == "ckpt-1" + assert [run.reward for run in runs] == [1.0, 0.0] + (forward_backward,) = service.requests("rl", "POST", f"{TRAIN}/forward-backward") + assert forward_backward.bearer == "team-key" + assert forward_backward.json == snapshot( + { + "inputs": [ + { + "samples": [ + { + "prompt_token_ids": [1, 2, 3], + "output_token_ids": [7], + "output_logprobs": [-0.5], + } + ], + "reward": 1.0, + "trace_id": TRACE_ID, + }, + { + "samples": [ + { + "prompt_token_ids": [1, 2, 3], + "output_token_ids": [7], + "output_logprobs": [-0.5], + } + ], + "reward": 0.0, + "trace_id": TRACE_ID, + }, + ], + "loss_fn": "importance_sampling", + "group_size": 2, + "reward_scale": 1.0, + "num_substeps": 1, + } + ) + (optim,) = service.requests("rl", "POST", f"{TRAIN}/optim-step") + assert optim.json == snapshot( + {"learning_rate": 1e-05, "beta1": 0.9, "beta2": 0.95, "eps": 1e-08, "weight_decay": 0.0} + ) -from hud.eval.run import Run -from hud.train import TrainingClient -from hud.utils.gateway import list_gateway_models - -_MODEL_ID = "00000000-0000-0000-0000-000000000001" - - -async def test_training_rejects_timed_out_run_without_explicit_reward() -> None: - run = Run.failed("rollout timed out") - run.trace.stop_reason = "timeout" - - with pytest.raises(ValueError, match="explicit training reward"): - await TrainingClient("test-model").forward_backward([run], group_size=1) +async def test_runs_without_token_samples_train_by_trace_id(service: FakeServices) -> None: + runs = await _runs(ScriptedAgent("ok")) -async def test_training_rejects_incomplete_run_groups() -> None: - runs = [Run(None, "", {}) for _ in range(4)] - for index, run in enumerate(runs): - run.trace.trace_id = str(index) - run.group_id = "a" if index < 3 else "b" + await TrainingClient("owner/model").forward_backward(runs, group_size=2) - with pytest.raises(ValueError, match="incomplete GRPO groups"): - await TrainingClient("test-model").forward_backward(runs, group_size=2) + (request,) = service.requests("rl", "POST", f"{TRAIN}/forward-backward") + assert request.json["inputs"] == [str(run.trace_id) for run in runs] + assert request.bearer == "hud-key" -async def test_slug_resolves_through_the_clients_own_credentials( - monkeypatch: pytest.MonkeyPatch, +async def test_the_model_slug_resolves_once_with_the_clients_own_key( + service: FakeServices, ) -> None: - monkeypatch.setattr("hud.settings.settings.api_key", None) - list_gateway_models.cache_clear() - seen: list[tuple[str, str | None]] = [] - - def catalog(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - seen.append((url, kwargs.get("api_key"))) - return { - "items": [{"id": _MODEL_ID, "model_name": "owner/model"}], - "total": 1, - } - - monkeypatch.setattr("hud.utils.platform.make_request_sync", catalog) - client = TrainingClient("owner/model", api_key="team-key", api_url="https://api.example") - try: - url = await client._train_url("optim_step") - finally: - list_gateway_models.cache_clear() - assert url.startswith(f"{client._base_url}/v1/models/{_MODEL_ID}/") - assert seen and seen[0][0].startswith("https://api.example/v2/models") - assert seen[0][1] == "team-key" - - -def _grpo_runs(groups: list[str], rewards: list[float]) -> list[Run]: - runs = [Run(None, "", {}) for _ in rewards] - for index, (run, group_id, reward) in enumerate(zip(runs, groups, rewards, strict=True)): - run.trace.trace_id = str(index) - run.group_id = group_id - run.grade.reward = reward - return runs - + runs = await _runs(ScriptedAgent("ok")) + client = TrainingClient("owner/model", api_key="team-key") + + await client.forward_backward(runs, group_size=2) + await client.optim_step(learning_rate=1e-5) + + (catalog,) = service.requests("api", "GET", "/v2/models") + assert catalog.bearer == "team-key" + assert [request.path for request in service.requests("rl")] == [ + f"{TRAIN}/forward-backward", + f"{TRAIN}/optim-step", + ] + + +@pytest.mark.parametrize("status", [502, 503, 504]) +async def test_an_optimizer_step_is_never_retried(service: FakeServices, status: int) -> None: + service.route("rl", "POST", f"{TRAIN}/optim-step", Reply(status=status), Reply(json=OPTIM)) + + with pytest.raises(HudRequestError) as raised: + await TrainingClient(MODEL_ID).optim_step(learning_rate=1e-5) + + assert raised.value.status_code == status + assert len(service.requests("rl", "POST", f"{TRAIN}/optim-step")) == 1 + + +async def test_a_custom_loss_sends_its_gradients_to_backward(service: FakeServices) -> None: + runs = await _runs(ScriptedAgent("ok")) + service.route( + "rl", + "POST", + f"{TRAIN}/forward", + json={ + "forward_id": "fwd-1", + "data": [ + { + "reward": reward, + "traj_idx": index, + "logprobs": [-0.5, -1.0], + "sampling_logprobs": [-0.5, -1.0], + "mask": [1.0, 1.0], + } + for index, reward in enumerate([1.0, 0.5]) + ], + }, + ) + service.route("rl", "POST", f"{TRAIN}/backward", json={"metrics": {}, "num_datums": 2}) + + def loss( + data: list[DatumTensors], logprobs: list[torch.Tensor] + ) -> tuple[torch.Tensor, dict[str, float]]: + total = sum(-datum.reward * lp.sum() for datum, lp in zip(data, logprobs, strict=True)) + assert isinstance(total, torch.Tensor) + return total, {"loss": total.item()} + + await TrainingClient(MODEL_ID).forward_backward_custom(runs, loss, group_size=2) + + (forward,) = service.requests("rl", "POST", f"{TRAIN}/forward") + (backward,) = service.requests("rl", "POST", f"{TRAIN}/backward") + assert forward.json == { + "inputs": [str(run.trace_id) for run in runs], + "group_size": 2, + "reward_scale": 1.0, + } + assert backward.json == { + "forward_id": "fwd-1", + "weights": [[1.0, 1.0], [0.5, 0.5]], + "metrics": {"loss": 2.25}, + } + + +async def _timed_out() -> list[Run]: + job = await Taskset("t", [Task(env="train", id="answer")]).run( + ScriptedAgent("ok", delay=5), runtime=LocalRuntime(_env()), rollout_timeout=0.2 + ) + return job.runs + + +@pytest.mark.parametrize( + ("runs", "group_size", "message"), + [ + pytest.param( + _timed_out, None, "timed-out runs require an explicit training reward", id="timeout" + ), + pytest.param( + lambda: _first(3, _runs(ScriptedAgent("ok"), group=2, tasks=_two_tasks())), + 2, + "3 trajectories do not divide evenly into groups of 2", + id="uneven", + ), + pytest.param( + lambda: _split_two_groups(_runs(ScriptedAgent("ok"), group=2, tasks=_three_tasks())), + 2, + "incomplete GRPO groups", + id="incomplete-group", + ), + pytest.param( + lambda: _single(Run.failed("never launched")), + None, + "run carries neither token-level samples nor a trace_id", + id="no-trace", + ), + ], +) +async def test_an_untrainable_batch_is_refused_before_anything_is_sent( + service: FakeServices, runs: Any, group_size: int | None, message: str +) -> None: + batch = await runs() -async def _forward_backward(runs: list[Run], monkeypatch: pytest.MonkeyPatch) -> None: - """Run forward_backward against a stubbed training service (a UUID model id - resolves without a catalog lookup).""" + with pytest.raises(ValueError, match=message): + await TrainingClient("owner/model").forward_backward(batch, group_size=group_size) + assert service.requests("rl") == [] + assert service.requests("api", "GET", "/v2/models") == [] - async def fake_request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: - return {"metrics": {}, "num_datums": len(runs)} - monkeypatch.setattr("hud.train.base.make_request", fake_request) - await TrainingClient(_MODEL_ID).forward_backward(runs, group_size=2) +def _two_tasks() -> list[Task]: + return [Task(env="train", id="answer", args={"target": target}) for target in ("ok", "no")] -async def test_training_warns_when_no_group_has_reward_spread( - caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch -) -> None: - # Identical rewards within every group zero every GRPO advantage, so the - # pass accumulates nothing and the later optim_step fails confusingly. - runs = _grpo_runs(["a", "a", "b", "b"], [1.0, 1.0, 0.0, 0.0]) +def _three_tasks() -> list[Task]: + return [Task(env="train", id="answer", args={"target": t}) for t in ("ok", "no", "maybe")] - with caplog.at_level(logging.WARNING, logger="hud.train.client"): - await _forward_backward(runs, monkeypatch) - assert "no gradient" in caplog.text +async def _first(count: int, runs: Any) -> list[Run]: + return (await runs)[:count] -async def test_training_groups_rewards_by_group_id_not_position( - caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch -) -> None: - # An interleaved batch is valid (groups are keyed by group_id), and its - # positional slices have spread even when the real groups have none. - runs = _grpo_runs(["a", "b", "a", "b"], [1.0, 0.0, 1.0, 0.0]) +async def _split_two_groups(runs: Any) -> list[Run]: + batch = await runs + return [batch[0], batch[2], batch[4], batch[5]] - with caplog.at_level(logging.WARNING, logger="hud.train.client"): - await _forward_backward(runs, monkeypatch) - assert "no gradient" in caplog.text +async def _single(run: Run) -> list[Run]: + return [run] -async def test_training_is_quiet_when_a_group_has_reward_spread( - caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + ("answers", "order", "warns"), + [ + pytest.param(["ok", "ok", "ok", "ok"], [0, 1, 2, 3], True, id="no-spread"), + pytest.param(["ok", "no", "ok", "no"], [0, 1, 2, 3], False, id="spread"), + pytest.param(["ok", "ok", "ok", "ok"], [0, 2, 1, 3], True, id="interleaved-groups"), + ], +) +async def test_a_batch_with_no_reward_spread_warns_that_it_trains_nothing( + service: FakeServices, + caplog: pytest.LogCaptureFixture, + answers: list[str], + order: list[int], + warns: bool, ) -> None: - runs = _grpo_runs(["a", "a", "b", "b"], [0.0, 1.0, 0.0, 1.0]) + replies = iter(answers) + runs = await _runs(ScriptedAgent(lambda _: next(replies)), tasks=_two_tasks()) with caplog.at_level(logging.WARNING, logger="hud.train.client"): - await _forward_backward(runs, monkeypatch) + await TrainingClient("owner/model").forward_backward( + [runs[index] for index in order], group_size=2 + ) - assert "no gradient" not in caplog.text + assert ("accumulates no gradient" in caplog.text) is warns + assert len(service.requests("rl")) == 1 From f1fe268719007d0ba12001bfacfc13a400fea0e6 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 025/248] test: cover the MCP transport and validation patches hud/patches/mcp_patches.py had no tests. A scripted streamable-HTTP MCP server shows a capability call retrying 502/503/504 and then succeeding, resolving with a -32000 transport error once HUD_CLIENT_TIMEOUT passes, on unparseable JSON, and on a dropped connection instead of hanging, and accepting structured output outside the tool's schema. A real FastMCP server shows plain results wrapped as {"result": ...} without output validation. Every row fails with the patches disabled. --- tests/patches/test_mcp_patches.py | 204 ++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 tests/patches/test_mcp_patches.py diff --git a/tests/patches/test_mcp_patches.py b/tests/patches/test_mcp_patches.py new file mode 100644 index 000000000..2016c1a2c --- /dev/null +++ b/tests/patches/test_mcp_patches.py @@ -0,0 +1,204 @@ +"""MCP calls over streamable HTTP resolve instead of hanging, and tolerate off-schema output.""" + +from __future__ import annotations + +import asyncio +import socket +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +import pytest +import uvicorn +from fastmcp import FastMCP +from mcp.shared.exceptions import McpError + +from hud.capabilities import Capability, MCPClient +from tests.harness import Reply + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Iterator + + from tests.harness import FakeServices, HudEnv, Request + +OUTPUT_SCHEMA = { + "type": "object", + "properties": {"n": {"type": "integer"}}, + "required": ["n"], +} + + +def _result(request: Request, result: dict[str, Any]) -> Reply: + return Reply(json={"jsonrpc": "2.0", "id": request.json["id"], "result": result}) + + +class FakeMCPServer: + """A streamable-HTTP MCP server whose ``tools/call`` answers come from a script.""" + + def __init__(self, services: FakeServices, calls: list[Callable[[Request], Reply]]) -> None: + self._calls = calls + self.url = f"{services.url('runtime')}/mcp" + services.route("runtime", "POST", "/mcp", handler=self._post) + services.route("runtime", "GET", "/mcp", Reply(status=405)) + services.route("runtime", "DELETE", "/mcp", Reply(json={})) + self.services = services + + def _post(self, request: Request) -> Reply: + method = request.json.get("method") + if method == "initialize": + reply = _result( + request, + { + "protocolVersion": request.json["params"]["protocolVersion"], + "capabilities": {"tools": {}}, + "serverInfo": {"name": "fake", "version": "1"}, + }, + ) + reply.headers["mcp-session-id"] = "session-1" + return reply + if "id" not in request.json: + return Reply(status=202, body=b"") + if method == "tools/list": + tool = {"name": "count", "inputSchema": {"type": "object"}} + return _result(request, {"tools": [{**tool, "outputSchema": OUTPUT_SCHEMA}]}) + answer = self._calls.pop(0) if len(self._calls) > 1 else self._calls[0] + return answer(request) + + def attempts(self) -> int: + return sum( + request.json.get("method") == "tools/call" + for request in self.services.requests("runtime", "POST", "/mcp") + ) + + +def _ok(request: Request) -> Reply: + return _result( + request, + {"content": [{"type": "text", "text": "3"}], "structuredContent": {"n": 3}}, + ) + + +def _status(status: int) -> Callable[[Request], Reply]: + return lambda request: Reply(status=status, json={}) + + +def _off_schema(request: Request) -> Reply: + return _result( + request, + {"content": [{"type": "text", "text": "x"}], "structuredContent": {"n": "x"}}, + ) + + +def _unparseable(request: Request) -> Reply: + del request + return Reply(body=b"{not json", content_type="application/json") + + +def _dropped(request: Request) -> Reply: + del request + + def chunks() -> Iterator[bytes]: + yield b'{"jsonrpc": "2.0", ' + raise ConnectionResetError("server went away") + + return Reply(stream=chunks(), content_type="application/json") + + +async def _call(server: FakeMCPServer) -> Any: + client = await MCPClient.connect(Capability.mcp(url=server.url, transport="streamable-http")) + try: + return await asyncio.wait_for(client.call_tool("count", {}), timeout=30) + finally: + await client.close() + + +@pytest.mark.parametrize( + ("calls", "structured", "attempts"), + [ + pytest.param([_status(502), _ok], {"n": 3}, 2, id="502-retried"), + pytest.param([_status(503), _ok], {"n": 3}, 2, id="503-retried"), + pytest.param([_status(504), _ok], {"n": 3}, 2, id="504-retried"), + pytest.param([_off_schema], {"n": "x"}, 1, id="output-outside-its-schema-accepted"), + ], +) +async def test_a_tool_call_succeeds_after_transient_gateway_errors( + services: FakeServices, + calls: list[Callable[[Request], Reply]], + structured: dict[str, Any], + attempts: int, +) -> None: + server = FakeMCPServer(services, calls) + + result = await _call(server) + + assert result.isError is False + assert result.structuredContent == structured + assert server.attempts() == attempts + + +@pytest.mark.parametrize( + ("calls", "error_type", "attempts"), + [ + pytest.param([_status(503)], "HTTPStatusError", 3, id="503-until-the-deadline"), + pytest.param([_unparseable], "ValidationError", 1, id="unparseable-json"), + pytest.param([_dropped], "RemoteProtocolError", 1, id="connection-dropped"), + ], +) +async def test_a_failing_tool_call_resolves_with_a_transport_error( + services: FakeServices, + hud_env: HudEnv, + calls: list[Callable[[Request], Reply]], + error_type: str, + attempts: int, +) -> None: + hud_env.set(HUD_CLIENT_TIMEOUT="1") + server = FakeMCPServer(services, calls) + + with pytest.raises(McpError) as raised: + await _call(server) + + assert raised.value.error.code == -32000 + assert raised.value.error.message == f"Transport error: {error_type}" + assert server.attempts() == attempts + + +@asynccontextmanager +async def _serving(mcp: FastMCP) -> AsyncIterator[str]: + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + server = uvicorn.Server(uvicorn.Config(mcp.http_app(), log_level="error")) + serving = asyncio.create_task(server.serve(sockets=[listener])) + for _ in range(1000): + if server.started: + break + await asyncio.sleep(0.01) + else: + raise TimeoutError("the MCP server did not start") + try: + yield f"http://127.0.0.1:{listener.getsockname()[1]}/mcp" + finally: + server.should_exit = True + await serving + + +async def test_a_fastmcp_tool_wraps_plain_results_and_skips_output_validation() -> None: + mcp = FastMCP("wrapped") + + @mcp.tool() + def numbers() -> list[int]: + return [1, 2] + + @mcp.tool() + def mistyped() -> int: + wrong: Any = "not an int" + return wrong + + async with _serving(mcp) as url: + client = await MCPClient.connect(Capability.mcp(url=url, transport="streamable-http")) + try: + wrapped = await client.call_tool("numbers", {}) + off_schema = await client.call_tool("mistyped", {}) + finally: + await client.close() + + assert (wrapped.isError, wrapped.structuredContent) == (False, {"result": [1, 2]}) + assert (off_schema.isError, off_schema.structuredContent) == (False, {"result": "not an int"}) From 73644fde2c89274ffd1cb5a3141c8bc28f4c405f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 026/248] test: drive HUDConsole.select with keystrokes only The select rows that patched questionary to inspect its choice list become keystroke rows; the smoke tests that only checked output methods do not raise are removed. --- tests/utils/test_hud_console.py | 156 ++++++-------------------------- 1 file changed, 26 insertions(+), 130 deletions(-) diff --git a/tests/utils/test_hud_console.py b/tests/utils/test_hud_console.py index 7508a5d31..a5e72fa29 100644 --- a/tests/utils/test_hud_console.py +++ b/tests/utils/test_hud_console.py @@ -1,149 +1,45 @@ -"""``HUDConsole`` — smoke-exercise the output methods + check the pure formatters. - -These mostly assert "doesn't raise" (output goes to a Rich console), which still -exercises the formatting branches; the ``format_*`` helpers return values -we can assert directly. -""" +"""``HUDConsole.select``, driven by keystrokes the way a user picks from a list.""" from __future__ import annotations -import logging from typing import Any import pytest import typer from prompt_toolkit.application import create_app_session from prompt_toolkit.input.defaults import create_pipe_input -from prompt_toolkit.key_binding import KeyBindings from prompt_toolkit.output import DummyOutput -from hud.utils.hud_console import HUDConsole - - -def _select_with_keystrokes( - keys: str, choices: list[str | dict[str, Any]], *, spaced: bool = False -) -> str: - """Drive ``HUDConsole.select`` with scripted keystrokes through a prompt_toolkit - pipe input (no real TTY), returning the selected value.""" - with create_pipe_input() as inp: - inp.send_text(keys) - with create_app_session(input=inp, output=DummyOutput()): - return HUDConsole().select("pick", choices, spaced=spaced) - - -def test_output_methods_do_not_raise() -> None: - c = HUDConsole() - c.header("Title") - c.section_title("Section") - c.success("ok") - c.error("bad") - c.warning("warn") - c.info("info") - c.print("plain") - c.dim_info("key", "value") - c.link("https://example.com") - c.progress_message("working") - c.hint("a hint") - c.status_item("label", "value") - c.command_example("hud eval tasks.json") - c.key_value_table({"key": "value"}) - - -def test_debug_respects_logger_level() -> None: - logger = logging.getLogger("test_hud_console_debug") - c = HUDConsole(logger=logger) - logger.setLevel(logging.DEBUG) - c.debug("debug visible") - logger.setLevel(logging.WARNING) - c.debug("debug hidden") # no-op when not verbose - - -def test_format_helpers_return_strings() -> None: - c = HUDConsole() - assert isinstance(c.format_tool_call("bash", {"command": "ls"}), str) - assert isinstance(c.format_tool_result("output text"), str) - assert isinstance(c.format_tool_result("error text", is_error=True), str) - - -def _capture_select_choices(monkeypatch: pytest.MonkeyPatch, returns: str) -> dict[str, Any]: - """Patch the ``questionary`` boundary so ``HUDConsole.select`` runs without a - TTY, recording the choice list it builds.""" - import questionary - - captured: dict[str, Any] = {} +from hud.utils import HUDConsole - class _App: - def __init__(self) -> None: - self.key_bindings = KeyBindings() +CHOICES: list[str | dict[str, Any]] = [ + {"name": "A", "value": "a"}, + {"name": "B", "value": "b"}, + {"name": "C", "value": "c"}, +] - class _Stub: - def __init__(self) -> None: - self.application = _App() - def ask(self) -> str: - return returns +def _select(keys: str, *, spaced: bool) -> str: + with create_pipe_input() as keyboard: + keyboard.send_text(keys) + with create_app_session(input=keyboard, output=DummyOutput()): + return HUDConsole().select("pick", CHOICES, spaced=spaced) - def _fake_select(message: str, **kwargs: Any) -> _Stub: - captured["choices"] = kwargs["choices"] - return _Stub() - monkeypatch.setattr(questionary, "select", _fake_select) - return captured +@pytest.mark.parametrize( + ("keys", "spaced", "picked"), + [ + pytest.param("\r", False, "a", id="enter-picks-the-first"), + pytest.param("\x1b[B\r", False, "b", id="down-then-enter"), + pytest.param("\x1b[B\r", True, "b", id="spaced-down-skips-the-separator"), + pytest.param("\x1b[B\x1b[B\r", True, "c", id="spaced-down-twice"), + ], +) +def test_arrow_keys_and_enter_pick_a_choice(keys: str, spaced: bool, picked: str) -> None: + assert _select(keys, spaced=spaced) == picked -def test_select_spaced_interleaves_blank_separators(monkeypatch: pytest.MonkeyPatch) -> None: - from questionary import Choice, Separator - - captured = _capture_select_choices(monkeypatch, returns="b") - - result = HUDConsole().select( - "pick", - [ - {"name": "A", "value": "a"}, - {"name": "B", "value": "b"}, - {"name": "C", "value": "c"}, - ], - spaced=True, - ) - - assert result == "b" - choices = captured["choices"] - # A blank separator sits between consecutive choices, but never leads or trails. - # (questionary's Separator subclasses Choice, so compare exact types here.) - assert [type(x).__name__ for x in choices] == [ - "Choice", - "Separator", - "Choice", - "Separator", - "Choice", - ] - assert sum(type(x) is Choice for x in choices) == 3 - assert sum(isinstance(x, Separator) for x in choices) == 2 - - -def test_select_without_spaced_has_no_separators(monkeypatch: pytest.MonkeyPatch) -> None: - from questionary import Separator - - captured = _capture_select_choices(monkeypatch, returns="a") - - HUDConsole().select("pick", [{"name": "A", "value": "a"}, {"name": "B", "value": "b"}]) - - assert not any(isinstance(x, Separator) for x in captured["choices"]) - - -def test_select_escape_cancels() -> None: - # questionary cancels on Ctrl+C only; we additionally bind Esc to cancel, - # which surfaces as a typer.Exit through select's None handling. +@pytest.mark.parametrize("spaced", [False, True]) +def test_escape_cancels_the_selection(spaced: bool) -> None: with pytest.raises(typer.Exit): - _select_with_keystrokes("\x1b", [{"name": "A", "value": "a"}, {"name": "B", "value": "b"}]) - - -def test_select_arrow_then_enter_selects_skipping_separators() -> None: - # Down + Enter lands on the second real choice; the blank separator between - # spaced choices is skipped, and the Esc binding doesn't break arrow keys. - result = _select_with_keystrokes( - "\x1b[B\r", - [{"name": "A", "value": "a"}, {"name": "B", "value": "b"}, {"name": "C", "value": "c"}], - spaced=True, - ) - assert result == "b" + _select("\x1b", spaced=spaced) From afa54960dba841b43f6af3ef5271aefe2d566965 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 027/248] test: cover robot video teardown and per-episode robot traces A lossy encoder finalizes on time with a full queue and a stalled callback, and a lossless submit stops waiting once its encoder thread has died. A wrapped two-env sim reports one trace per episode: done closes the slot's trace with env success outranking the shaped reward, skips the post-reset observation, and opens a fresh trace for the next episode. --- tests/telemetry/test_robot_recorder.py | 112 +++++++++++++++++++++++++ tests/telemetry/test_robot_video.py | 49 ++++++++++- 2 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 tests/telemetry/test_robot_recorder.py diff --git a/tests/telemetry/test_robot_recorder.py b/tests/telemetry/test_robot_recorder.py new file mode 100644 index 000000000..daf28b23e --- /dev/null +++ b/tests/telemetry/test_robot_recorder.py @@ -0,0 +1,112 @@ +"""A wrapped vectorized sim reports one trace per episode per recorded slot.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar + +import numpy as np +from inline_snapshot import snapshot + +from hud.environment.robot import wrap + +if TYPE_CHECKING: + from tests.harness import FakeServices, HudEnv + + +class TwoArmSim: + """Two envs: env 0 finishes on the second step (succeeding with a shaped reward of + 0.2), env 1 on the third (failing with a shaped reward of 5.0).""" + + num_envs = 2 + metadata: ClassVar[dict[str, int]] = {"render_fps": 10} + DONES = ([False, False], [True, False], [False, True]) + + def __init__(self) -> None: + self.t = 0 + + def _obs(self) -> dict[str, Any]: + return { + "state": np.full((2, 3), self.t, dtype=np.float32), + "cam": np.full((2, 16, 16, 3), self.t, dtype=np.uint8), + } + + def reset(self, **kwargs: Any) -> tuple[dict[str, Any], dict[str, Any]]: + del kwargs + return self._obs(), {} + + def step(self, action: Any) -> tuple[Any, ...]: + del action + done = np.array(self.DONES[self.t]) + self.t += 1 + info = {"success": np.array([True, False])} + return self._obs(), np.array([0.2, 5.0]), done, np.array([False, False]), info + + def close(self) -> None: + pass + + +def test_each_episode_reports_its_own_trace_and_outcome( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1") + for path in ("/v2/trace/job/{id}/enter", "/v2/trace/{id}/enter", "/v2/trace/{id}/exit"): + services.route("api", "POST", path, json={}) + services.route("telemetry", "POST", "/trace/{id}/telemetry-upload", json={}) + + env = wrap(TwoArmSim(), contract=None) + env.reset() + for _ in TwoArmSim.DONES: + env.step(np.zeros((2, 2))) + env.close() + + opened = [ + request.params["id"] for request in services.requests("api", "POST", "/v2/trace/{id}/enter") + ] + label = {trace_id: f"trace-{index}" for index, trace_id in enumerate(opened)} + exits = { + label[request.params["id"]]: request.json + for request in services.requests("api", "POST", "/v2/trace/{id}/exit") + } + spans: dict[str, list[dict[str, Any]]] = {} + for upload in services.requests("telemetry", "POST", "/trace/{id}/telemetry-upload"): + spans.setdefault(label[upload.params["id"]], []).extend(upload.json["telemetry"]) + steps = { + trace: [ + (span["name"], span["attributes"]["hud.payload"]["tick"]) + for span in sorted(recorded, key=lambda span: span["start_time"]) + if span["name"] != "step.video_segment" + ] + for trace, recorded in spans.items() + } + assert exits == snapshot( + { + "trace-0": { + "status": "completed", + "reward": 1.0, + "metadata": {"env_index": 0, "episode_index": 0, "seed": None, "success": True}, + }, + "trace-1": { + "status": "completed", + "reward": 0.0, + "metadata": {"env_index": 1, "episode_index": 0, "seed": None, "success": False}, + }, + "trace-2": { + "status": "completed", + "reward": 0.2, + "metadata": {"env_index": 0, "episode_index": 1, "seed": None}, + }, + } + ) + assert dict(sorted(steps.items())) == snapshot( + { + "trace-0": [("step.observation", 0), ("step.inference", 0), ("step.inference", 1)], + "trace-1": [ + ("step.observation", 0), + ("step.inference", 0), + ("step.observation", 1), + ("step.inference", 1), + ("step.inference", 2), + ], + "trace-2": [("step.observation", 0), ("step.inference", 0)], + } + ) diff --git a/tests/telemetry/test_robot_video.py b/tests/telemetry/test_robot_video.py index c900775d6..dccd68b39 100644 --- a/tests/telemetry/test_robot_video.py +++ b/tests/telemetry/test_robot_video.py @@ -1,15 +1,20 @@ -"""Tests for the robot trace video encoder.""" +"""The robot trace video encoder: lossless frame keeping, and teardown that cannot hang.""" from __future__ import annotations import io import threading +import time +from typing import TYPE_CHECKING import av import numpy as np from hud.telemetry.robot.video import SegmentEncoder +if TYPE_CHECKING: + import pytest + def test_a_lossless_encoder_keeps_every_frame_while_its_queue_is_full() -> None: frames = 60 @@ -28,3 +33,45 @@ def on_segment(_index: int, data: bytes) -> None: with av.open(io.BytesIO(b"".join(segments)), mode="r") as container: assert sum(1 for _ in container.decode(video=0)) == frames + + +def test_a_lossy_encoder_finalizes_on_time_with_a_full_queue_and_a_stalled_encoder() -> None: + released = threading.Event() + encoder = SegmentEncoder( + "cam", + lambda _index, _data: released.wait(), + fps=10, + segment_seconds=0.1, + max_queued_frames=1, + ) + for value in range(40): + encoder.submit(np.full((16, 16, 3), value, dtype=np.uint8)) + + started = time.monotonic() + encoder.finalize(timeout=0.5) + elapsed = time.monotonic() - started + released.set() + + assert elapsed < 5 + + +def test_a_lossless_submit_stops_waiting_once_the_encoder_has_died( + caplog: pytest.LogCaptureFixture, +) -> None: + encoder = SegmentEncoder( + "cam", lambda _index, _data: None, fps=10, max_queued_frames=1, lossless=True + ) + finished = threading.Event() + + def feed() -> None: + # An empty image fails the encoder, which ends its thread with the queue unread. + encoder.submit(np.zeros((0, 16, 3), dtype=np.uint8)) + for _ in range(20): + encoder.submit(np.zeros((16, 16, 3), dtype=np.uint8)) + encoder.finalize(timeout=1) + finished.set() + + threading.Thread(target=feed, daemon=True).start() + + assert finished.wait(timeout=30) + assert "video encode failed (camera cam)" in caplog.text From 1ee3e34b06c113c08afedc2a1f55de7736ddd3ad Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 028/248] refactor: drop the exporter's guard against non-string span directories settings.span_dir is typed str | None and only a test that patched an object() into settings ever reached the TypeError branch. --- hud/telemetry/exporter.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hud/telemetry/exporter.py b/hud/telemetry/exporter.py index de395419b..885be614d 100644 --- a/hud/telemetry/exporter.py +++ b/hud/telemetry/exporter.py @@ -77,10 +77,6 @@ def _export_local(span: dict[str, Any], local_dir: str | None) -> None: """ if not local_dir: return - # Only I/O is best-effort: a config-shape error (e.g. a mock or wrong type - # reaching settings) must fail loudly, not write to a repr-named path. - if not isinstance(local_dir, str): - raise TypeError(f"telemetry_local_dir must be a str path, got {type(local_dir).__name__}") try: path = Path(local_dir) path.mkdir(parents=True, exist_ok=True) From f54f64073707a6e5c835786e4c31520e9cc70bfb Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 029/248] test: type the grader templates and the stalled video callback ty rejected reading __name__ off a Callable and a lambda returning the Event's bool where the segment callback returns None. --- tests/telemetry/test_robot_video.py | 6 +++- tests/test_graders.py | 51 ++++++++++++++++------------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/tests/telemetry/test_robot_video.py b/tests/telemetry/test_robot_video.py index dccd68b39..035e9c8f7 100644 --- a/tests/telemetry/test_robot_video.py +++ b/tests/telemetry/test_robot_video.py @@ -37,9 +37,13 @@ def on_segment(_index: int, data: bytes) -> None: def test_a_lossy_encoder_finalizes_on_time_with_a_full_queue_and_a_stalled_encoder() -> None: released = threading.Event() + + def stall(_index: int, _data: bytes) -> None: + released.wait() + encoder = SegmentEncoder( "cam", - lambda _index, _data: released.wait(), + stall, fps=10, segment_seconds=0.1, max_queued_frames=1, diff --git a/tests/test_graders.py b/tests/test_graders.py index 5ca55848c..6caf99c55 100644 --- a/tests/test_graders.py +++ b/tests/test_graders.py @@ -32,8 +32,11 @@ from tests.harness import ScriptedAgent, Turn, say if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Awaitable, Callable from pathlib import Path + from typing import TypeAlias + + Grade: TypeAlias = Callable[[str], Awaitable[EvaluationResult]] from hud.eval import Run from tests.harness import HudEnv, ModelRequest, Models @@ -153,12 +156,18 @@ async def compute_score(cls, kind: str = "float", **kwargs: Any) -> Any: def _graders_env(root: Path) -> Environment: env = Environment("graders") - def task(grade: Callable[[str], Any]) -> None: - @env.template(id=grade.__name__) - async def template(): - answer = yield "Name the capital of France." - yield await grade(answer) + def graded(name: str) -> Callable[[Grade], Grade]: + def register(grade: Grade) -> Grade: + @env.template(id=name) + async def template(): + answer = yield "Name the capital of France." + yield await grade(answer) + + return grade + + return register + @graded("bash_pair") async def bash_pair(answer: str) -> EvaluationResult: del answer return await combine( @@ -166,6 +175,7 @@ async def bash_pair(answer: str) -> EvaluationResult: BashGrader.grade(weight=0.5, command="echo oops >&2; false"), ) + @graded("any_tree") async def any_tree(answer: str) -> EvaluationResult: either = combine_any( weight=0.5, @@ -183,6 +193,7 @@ async def any_tree(answer: str) -> EvaluationResult: either, both, SubScore(name="format", value=exact_match(answer, ANSWER), weight=0.25) ) + @graded("weighted") async def weighted(answer: str) -> EvaluationResult: return await combine( SubScore(name="mentions", value=contains(answer, "paris"), weight=0.6), @@ -190,6 +201,7 @@ async def weighted(answer: str) -> EvaluationResult: SubScore(name="penalty", value=1.0, weight=-0.2), ) + @graded("penalty_only_fails") async def penalty_only_fails(answer: str) -> EvaluationResult: del answer return await combine( @@ -197,6 +209,7 @@ async def penalty_only_fails(answer: str) -> EvaluationResult: SubScore(name="penalty", value=1.0, weight=-0.2), ) + @graded("suffixed") async def suffixed(answer: str) -> EvaluationResult: del answer return await combine( @@ -205,6 +218,7 @@ async def suffixed(answer: str) -> EvaluationResult: SubScore(name="x", value=0.0, weight=0.4), ) + @graded("custom") async def custom(answer: str) -> EvaluationResult: del answer return await combine( @@ -216,6 +230,7 @@ async def custom(answer: str) -> EvaluationResult: ), ) + @graded("concurrent") async def concurrent(answer: str) -> EvaluationResult: del answer # Each command waits for the file the other writes, so they pass only in parallel. @@ -229,6 +244,7 @@ async def concurrent(answer: str) -> EvaluationResult: ), ) + @graded("in_cwd") async def in_cwd(answer: str) -> EvaluationResult: del answer (root / "marker.txt").write_text("here") @@ -236,12 +252,14 @@ async def in_cwd(answer: str) -> EvaluationResult: BashGrader.grade(weight=1.0, command="cat marker.txt", cwd=str(root)), ) + @graded("detached_child") async def detached_child(answer: str) -> EvaluationResult: del answer return await combine( BashGrader.grade(weight=1.0, command="echo started; sleep 30 & exit 0"), ) + @graded("overrun") async def overrun(answer: str) -> EvaluationResult: del answer return await combine( @@ -250,12 +268,14 @@ async def overrun(answer: str) -> EvaluationResult: ), ) + @graded("chatty") async def chatty(answer: str) -> EvaluationResult: del answer return await combine( BashGrader.grade(weight=1.0, command="yes hud | head -c 500000"), ) + @graded("judged") async def judged(answer: str) -> EvaluationResult: return await combine( LLMJudgeGrader.grade( @@ -266,31 +286,16 @@ async def judged(answer: str) -> EvaluationResult: ) ) + @graded("judged_negative_only") async def judged_negative_only(answer: str) -> EvaluationResult: return await combine( LLMJudgeGrader.grade(weight=1.0, answer=answer, criteria=[("invents facts", -1.0)]) ) + @graded("judged_without_criteria") async def judged_without_criteria(answer: str) -> EvaluationResult: return await combine(LLMJudgeGrader.grade(weight=1.0, answer=answer, criteria=[])) - for grade in ( - bash_pair, - any_tree, - weighted, - penalty_only_fails, - suffixed, - custom, - concurrent, - in_cwd, - detached_child, - overrun, - chatty, - judged, - judged_negative_only, - judged_without_criteria, - ): - task(grade) return env From 0c15a0bdfe398e692ac7a3e7940b11f850be8124 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:15 -0700 Subject: [PATCH 030/248] test: give the escaped-child completion row a load-proof margin The row allowed one second for completion while the escaped child slept two, so it timed out under coverage with eight workers. The child now outlives a ten-second budget by twenty seconds, which still proves completion does not wait for it. --- tests/utils/test_process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/test_process.py b/tests/utils/test_process.py index 1fdbca0bc..87b4709c3 100644 --- a/tests/utils/test_process.py +++ b/tests/utils/test_process.py @@ -100,8 +100,8 @@ def write(self, value: str) -> int: async def test_a_child_outside_the_group_cannot_hold_completion_open() -> None: child = shlex.quote(sys.executable) result = await asyncio.wait_for( - _run(f"{child} -c 'import os, time; os.setsid(); time.sleep(2)' & echo retained"), - 1, + _run(f"{child} -c 'import os, time; os.setsid(); time.sleep(30)' & echo retained"), + 10, ) assert result.returncode == 0 From fc346c698e56afdfcd42c27712a644b2cf92a6f5 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:21:03 -0700 Subject: [PATCH 031/248] fix(robot): never show negative zero in tool observations A value that rounds to zero printed as -0.0000 on some platforms and 0.0000 on others, so the text a model reads, and the scenarios that pin it, depended on the machine. Observations now round first and drop the sign of zero. --- hud/environment/robot/control.py | 11 ++++++++--- tests/environment/test_robot_control.py | 14 +++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/hud/environment/robot/control.py b/hud/environment/robot/control.py index 8243eb226..86d7384c2 100644 --- a/hud/environment/robot/control.py +++ b/hud/environment/robot/control.py @@ -659,7 +659,7 @@ def _off(self, proprio: NDArray[np.float64], goal: NDArray[np.float64]) -> str: for name in self._dims: axis = self._axes[name] if axis.kind == "scalar": - parts.append(f"{name}={float(proprio[axis.index] - goal[axis.index]):.4f}") + parts.append(f"{name}={_decimal(float(proprio[axis.index] - goal[axis.index]))}") continue assert axis.block is not None key = id(axis.block) @@ -667,7 +667,7 @@ def _off(self, proprio: NDArray[np.float64], goal: NDArray[np.float64]) -> str: continue reported.add(key) angle = angular_distance(axis.block.read(proprio), axis.block.read(goal)) - parts.append(f"orientation={angle:.4f} rad") + parts.append(f"orientation={_decimal(angle)} rad") return ", ".join(parts) @@ -686,7 +686,12 @@ def _tool_order(names: list[str], blocks: list[Orientation]) -> list[str]: def _labeled(pairs: Iterable[tuple[str, Any]]) -> str: - return ", ".join(f"{name}={value:.4f}" for name, value in pairs) + return ", ".join(f"{name}={_decimal(float(value))}" for name, value in pairs) + + +def _decimal(value: float) -> str: + """Four decimals, unsigned when the value rounds to zero, so ``-0.0000`` never shows.""" + return f"{round(value, 4) + 0.0:.4f}" def _png(frame: NDArray[Any]) -> str: diff --git a/tests/environment/test_robot_control.py b/tests/environment/test_robot_control.py index 99094ffc4..84856e1ac 100644 --- a/tests/environment/test_robot_control.py +++ b/tests/environment/test_robot_control.py @@ -537,8 +537,8 @@ def quaternion_alignment(target: list[float]) -> Callable[[NDArray[Any]], float] "tcp_quat.qz=0.7071", "tcp_quat.qw=0.7071", ], - ["pose: tcp.roll=0.0000", "tcp.pitch=-0.0000", "tcp.yaw=1.5708"], - ["commanded: tcp.roll=0.0000", "tcp.pitch=-0.0000", "tcp.yaw=1.5708"], + ["pose: tcp.roll=0.0000", "tcp.pitch=0.0000", "tcp.yaw=1.5708"], + ["commanded: tcp.roll=0.0000", "tcp.pitch=0.0000", "tcp.yaw=1.5708"], ["off: orientation=0.0000 rad"], ["camera observation/image:"], ], @@ -572,8 +572,8 @@ def quaternion_alignment(target: list[float]) -> Callable[[NDArray[Any]], float] "tcp_quat.qy=0.0000", "tcp_quat.qz=0.0000", ], - ["pose: tcp.roll=1.5708", "tcp.pitch=-0.0000", "tcp.yaw=0.0000"], - ["commanded: tcp.roll=1.5708", "tcp.pitch=-0.0000", "tcp.yaw=0.0000"], + ["pose: tcp.roll=1.5708", "tcp.pitch=0.0000", "tcp.yaw=0.0000"], + ["commanded: tcp.roll=1.5708", "tcp.pitch=0.0000", "tcp.yaw=0.0000"], ["off: orientation=0.0000 rad"], ["camera observation/image:"], ], @@ -732,7 +732,7 @@ def quaternion_alignment(target: list[float]) -> Callable[[NDArray[Any]], float] ], [ "commanded: target_eef.roll=1.5708", - "target_eef.pitch=-0.0000", + "target_eef.pitch=0.0000", "target_eef.yaw=1.5708", ], ["off: orientation=0.0000 rad"], @@ -784,7 +784,7 @@ def quaternion_alignment(target: list[float]) -> Callable[[NDArray[Any]], float] "ee_pos.y=0.0000", "ee_pos.z=1.2000", "ee.roll=0.0000", - "ee.pitch=-0.0000", + "ee.pitch=0.0000", "ee.yaw=0.4000", "gripper=0.0400", ], @@ -793,7 +793,7 @@ def quaternion_alignment(target: list[float]) -> Callable[[NDArray[Any]], float] "ee_pos.y=0.0000", "ee_pos.z=1.2000", "ee.roll=0.0000", - "ee.pitch=-0.0000", + "ee.pitch=0.0000", "ee.yaw=0.4000", "gripper=0.0400", ], From 435ba86eff17fefc9826b0ad1b7f59a80f7fa6bf Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:21:39 -0700 Subject: [PATCH 032/248] test: let the black-box policy ratchet files that predate it A file listed in scripts/check_tests_baseline.json may keep the number of reaches into hud internals it holds today, and that number may only fall; every other file must have none. This lets suites be migrated one at a time without letting any file get worse in the meantime. --- scripts/check_tests.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/scripts/check_tests.py b/scripts/check_tests.py index 0f7cebfcd..8c45c7191 100644 --- a/scripts/check_tests.py +++ b/scripts/check_tests.py @@ -9,18 +9,26 @@ - importing a private name or module from hud; - touching a private attribute of anything other than ``self`` or ``cls``. +Files that predate the rule are listed in ``check_tests_baseline.json`` with the +number of findings they still hold. That number may only fall: a file above its +baseline fails, and so does one below it until the baseline is lowered to match. +Every other file must have none. + Usage: uv run python scripts/check_tests.py [path ...] """ from __future__ import annotations import ast +import json import sys +from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parents[1] DEFAULT_PATHS = [ROOT / "tests", *sorted(ROOT.glob("environments/*/tests"))] HARNESS = ROOT / "tests" / "harness" +BASELINE = Path(__file__).with_name("check_tests_baseline.json") PATCHERS = {"setattr", "delattr", "patch"} PATCH_METHODS = {"object", "dict", "multiple"} @@ -110,12 +118,27 @@ def main(argv: list[str]) -> int: for file in ([root] if root.is_file() else root.rglob("*.py")) if HARNESS not in file.parents and "tasks" not in file.relative_to(ROOT).parts ) - problems = [problem for file in files for problem in check(file)] - for problem in problems: - print(problem) - if problems: - print(f"\n{len(problems)} reach(es) into hud internals; see Testing in AGENTS.md.") - return 1 if problems else 0 + baseline: dict[str, int] = json.loads(BASELINE.read_text("utf-8")) if BASELINE.exists() else {} + findings = {str(file.relative_to(ROOT)): check(file) for file in files} + failed = False + for name, problems in findings.items(): + allowed = baseline.get(name, 0) + if len(problems) > allowed: + failed = True + for problem in problems: + print(problem) + print(f"{name}: {len(problems)} reach(es) into hud internals, {allowed} allowed\n") + elif len(problems) < allowed: + failed = True + print(f"{name}: {len(problems)} reach(es) left; lower its baseline from {allowed}\n") + counts = Counter({name: len(problems) for name, problems in findings.items()}) + if failed: + print("See Testing in AGENTS.md; baselines live in scripts/check_tests_baseline.json.") + elif baseline: + print( + f"{sum(counts[name] for name in baseline)} known reach(es) in {len(baseline)} file(s)" + ) + return 1 if failed else 0 if __name__ == "__main__": From 676a54dccda7a1c2fb872e6b780b5b336e37624b Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 033/248] test(harness): script provider-native items, finish reasons and stream faults Agent scenarios need provider turns the scripted models could not produce: Responses computer_call and shell_call items, Anthropic citations, refusal details and malformed tool JSON, Gemini grounding metadata, Chat token ids, and Anthropic streams that report an error event or drop mid-message. Turn gains native entries appended as they are, a finish-reason override and envelope extras; computer_call, shell_call, stream_error and interrupted build the common cases. Existing turns encode as before. --- tests/harness/__init__.py | 19 ++++- tests/harness/models.py | 146 ++++++++++++++++++++++++++++++++------ 2 files changed, 143 insertions(+), 22 deletions(-) diff --git a/tests/harness/__init__.py b/tests/harness/__init__.py index 4858ef700..b1f222517 100644 --- a/tests/harness/__init__.py +++ b/tests/harness/__init__.py @@ -9,7 +9,20 @@ from .cli import Hud, Result, scrub from .config import HudEnv from .docker import FakeDocker -from .models import ModelRequest, Models, ToolCall, Turn, call, calls, fail, say +from .models import ( + ModelRequest, + Models, + ToolCall, + Turn, + call, + calls, + computer_call, + fail, + interrupted, + say, + shell_call, + stream_error, +) from .rfb import FakeScreen, KeyEvent, PointerEvent, fake_screen from .scenario import RecordingProvider, ScriptedAgent, served, task_row from .services import FakeServices, Reply, Request @@ -35,12 +48,16 @@ "Turn", "call", "calls", + "computer_call", "fail", "fake_screen", + "interrupted", "say", "scrub", "served", + "shell_call", "spans", "steps", + "stream_error", "task_row", ] diff --git a/tests/harness/models.py b/tests/harness/models.py index 66d8e0e01..099104d09 100644 --- a/tests/harness/models.py +++ b/tests/harness/models.py @@ -38,13 +38,23 @@ class ToolCall: @dataclass(frozen=True) class Turn: - """One assistant turn. ``fail`` answers with an HTTP error instead, such as a 529 or 400.""" + """One assistant turn. ``fail`` answers with an HTTP error instead, such as a 529 or 400. + + ``native`` holds provider-native entries appended after the tool calls as they + are: Responses output items, Anthropic content blocks, Gemini parts or Chat + Completions tool calls. ``finish`` replaces the protocol's finish reason, and + ``extra`` is merged into the turn's envelope: the Chat Completions choice, the + Responses response, the Anthropic message or the Gemini candidate. + """ text: str | None = None tool_calls: tuple[ToolCall, ...] = () reasoning: str | None = None truncated: bool = False fail: Reply | None = None + native: tuple[dict[str, Any], ...] = () + finish: str | None = None + extra: dict[str, Any] = field(default_factory=dict[str, Any]) def say(text: str, *, reasoning: str | None = None) -> Turn: @@ -63,6 +73,60 @@ def fail(status: int, message: str = "scripted provider error") -> Turn: return Turn(fail=Reply(status=status, json={"error": {"type": "error", "message": message}})) +def computer_call( + *actions: dict[str, Any], + id: str = "call_computer", + pending_safety_checks: Sequence[dict[str, Any]] = (), +) -> Turn: + """A Responses ``computer_call`` running ``actions`` in order.""" + return Turn( + native=( + { + "type": "computer_call", + "id": f"cu_{id}", + "call_id": id, + "actions": list(actions), + "pending_safety_checks": list(pending_safety_checks), + "status": "completed", + }, + ) + ) + + +def shell_call(*commands: str, id: str = "call_shell", **action: Any) -> Turn: + """A Responses ``shell_call``; ``action`` adds ``timeout_ms`` or ``max_output_length``.""" + return Turn( + native=( + { + "type": "shell_call", + "id": f"sh_{id}", + "call_id": id, + "action": {"commands": list(commands), **action}, + "status": "completed", + }, + ) + ) + + +def stream_error(kind: str, *, headers: dict[str, str] | None = None) -> Turn: + """An Anthropic stream that opens and then reports an ``error`` event of type ``kind``.""" + events = [ + _sse("message_start", {"message": _message_start("m")}), + _sse("error", {"error": {"type": kind, "message": f"scripted {kind}"}}), + ] + return Turn(fail=Reply(stream=events, headers=headers or {}, content_type="text/event-stream")) + + +def interrupted() -> Turn: + """An Anthropic stream whose connection drops after the message starts.""" + + def events() -> Iterator[bytes]: + yield _sse("message_start", {"message": _message_start("m")}) + raise ConnectionResetError("scripted interruption") + + return Turn(fail=Reply(stream=events(), content_type="text/event-stream")) + + @dataclass(frozen=True) class ModelRequest: """A model request as the provider received it, with the turn it asks for.""" @@ -221,21 +285,23 @@ def _chat(self, raw: Request) -> Reply: } for tool in turn.tool_calls ] + tool_calls.extend(turn.native) message: dict[str, Any] = {"role": "assistant", "content": turn.text} if tool_calls: message["tool_calls"] = tool_calls if turn.reasoning is not None: message["reasoning_content"] = turn.reasoning - finish = "length" if turn.truncated else ("tool_calls" if tool_calls else "stop") + finish = turn.finish or ( + "length" if turn.truncated else ("tool_calls" if tool_calls else "stop") + ) + choice = {"index": 0, "message": message, "finish_reason": finish, "logprobs": None} return Reply( json={ "id": self._id("chatcmpl"), "object": "chat.completion", "created": 0, "model": body.get("model", ""), - "choices": [ - {"index": 0, "message": message, "finish_reason": finish, "logprobs": None} - ], + "choices": [{**choice, **turn.extra}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, } ) @@ -279,13 +345,15 @@ def _responses_api(self, raw: Request) -> Reply: } for tool in turn.tool_calls ) + output.extend(turn.native) + incomplete = turn.finish or ("max_output_tokens" if turn.truncated else None) return Reply( json={ "id": response_id, "object": "response", "created_at": 0, - "status": "incomplete" if turn.truncated else "completed", - "incomplete_details": {"reason": "max_output_tokens"} if turn.truncated else None, + "status": "incomplete" if incomplete else "completed", + "incomplete_details": {"reason": incomplete} if incomplete else None, "model": body.get("model", ""), "output": output, "parallel_tool_calls": True, @@ -298,6 +366,7 @@ def _responses_api(self, raw: Request) -> Reply: "input_tokens_details": {"cached_tokens": 0}, "output_tokens_details": {"reasoning_tokens": 0}, }, + **turn.extra, } ) @@ -321,7 +390,10 @@ def _anthropic(self, raw: Request) -> Reply: } for tool in turn.tool_calls ) - stop = "max_tokens" if turn.truncated else ("tool_use" if turn.tool_calls else "end_turn") + blocks.extend(turn.native) + stop = turn.finish or ( + "max_tokens" if turn.truncated else ("tool_use" if turn.tool_calls else "end_turn") + ) message = { "id": self._id("msg"), "type": "message", @@ -331,6 +403,7 @@ def _anthropic(self, raw: Request) -> Reply: "stop_reason": stop, "stop_sequence": None, "usage": {"input_tokens": 1, "output_tokens": 1}, + **turn.extra, } if not body.get("stream"): return Reply(json=message) @@ -349,15 +422,15 @@ def _gemini(self, raw: Request) -> Reply: {"functionCall": {"name": tool.name, "args": tool.arguments}} for tool in turn.tool_calls ) + parts.extend(turn.native) + candidate = { + "content": {"role": "model", "parts": parts}, + "finishReason": turn.finish or ("MAX_TOKENS" if turn.truncated else "STOP"), + "index": 0, + } return Reply( json={ - "candidates": [ - { - "content": {"role": "model", "parts": parts}, - "finishReason": "MAX_TOKENS" if turn.truncated else "STOP", - "index": 0, - } - ], + "candidates": [{**candidate, **turn.extra}], "usageMetadata": { "promptTokenCount": 1, "candidatesTokenCount": 1, @@ -368,11 +441,27 @@ def _gemini(self, raw: Request) -> Reply: ) -def _anthropic_events(message: dict[str, Any]) -> Iterator[bytes]: - def event(kind: str, data: dict[str, Any]) -> bytes: - return f"event: {kind}\ndata: {json.dumps({'type': kind, **data})}\n\n".encode() +def _sse(kind: str, data: dict[str, Any]) -> bytes: + return f"event: {kind}\ndata: {json.dumps({'type': kind, **data})}\n\n".encode() + +def _message_start(model: str) -> dict[str, Any]: + return { + "id": "msg_stream", + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + +def _anthropic_events(message: dict[str, Any]) -> Iterator[bytes]: + event = _sse start = {**message, "content": [], "stop_reason": None} + start.pop("stop_details", None) yield event("message_start", {"message": start}) for index, block in enumerate(message["content"]): if block["type"] == "text": @@ -384,6 +473,11 @@ def event(kind: str, data: dict[str, Any]) -> bytes: "content_block_delta", {"index": index, "delta": {"type": "text_delta", "text": block["text"]}}, ) + for citation in block.get("citations") or []: + yield event( + "content_block_delta", + {"index": index, "delta": {"type": "citations_delta", "citation": citation}}, + ) elif block["type"] == "thinking": yield event( "content_block_start", @@ -406,7 +500,9 @@ def event(kind: str, data: dict[str, Any]) -> bytes: "delta": {"type": "signature_delta", "signature": block["signature"]}, }, ) - else: + elif "input" in block: + # A string input streams verbatim, so a script can send malformed tool JSON. + partial = block["input"] yield event( "content_block_start", {"index": index, "content_block": {**block, "input": {}}}, @@ -417,15 +513,23 @@ def event(kind: str, data: dict[str, Any]) -> bytes: "index": index, "delta": { "type": "input_json_delta", - "partial_json": json.dumps(block["input"]), + "partial_json": partial + if isinstance(partial, str) + else json.dumps(partial), }, }, ) + else: + yield event("content_block_start", {"index": index, "content_block": block}) yield event("content_block_stop", {"index": index}) yield event( "message_delta", { - "delta": {"stop_reason": message["stop_reason"], "stop_sequence": None}, + "delta": { + "stop_reason": message["stop_reason"], + "stop_sequence": None, + "stop_details": message.get("stop_details"), + }, "usage": {"output_tokens": 1}, }, ) From 2559e39f35b19bbe2fef7e5842bf78be7590a7c3 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 034/248] test(agents): drive each provider agent through a served workspace Every provider agent (Claude, OpenAI Responses, Gemini, OpenAI-compatible chat) now runs a real rollout against an environment serving a workspace over SSH, talking to the scripted providers behind the fake gateway. The scenarios pin the reward, the exported steps, and the request bodies each provider received: tools per model, system prompt, cache_control, previous_response_id chaining, replayed reasoning, hosted tools, every kind of MCP tool result, Claude's image preparation for viewed files, and the step payloads (citations, refusals, usage, training samples) the platform reads. --- tests/agents/support.py | 170 ++++ tests/agents/test_provider_requests.py | 1019 ++++++++++++++++++++++++ tests/agents/test_step_spans.py | 566 +++++++++++++ tests/agents/test_tool_results.py | 722 +++++++++++++++++ 4 files changed, 2477 insertions(+) create mode 100644 tests/agents/support.py create mode 100644 tests/agents/test_provider_requests.py create mode 100644 tests/agents/test_step_spans.py create mode 100644 tests/agents/test_tool_results.py diff --git a/tests/agents/support.py b/tests/agents/support.py new file mode 100644 index 000000000..f5d37ed8d --- /dev/null +++ b/tests/agents/support.py @@ -0,0 +1,170 @@ +"""Fixture environments and request views shared by the agent scenarios. + +Every scenario runs a real rollout: a :class:`~hud.Environment` serving a real +workspace over SSH (and a fake screen or MCP servers when the row needs them), +placed by :class:`~hud.eval.LocalRuntime`, with the agent talking to the +scripted providers behind the fake gateway. +""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import re +import socket +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any, Literal + +import uvicorn +from PIL import Image + +from hud import Environment +from hud.capabilities import Capability +from hud.eval import LocalRuntime, Task, rollout + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Mapping + from pathlib import Path + + from fastmcp import FastMCP + + from hud.agents.base import Agent + from hud.eval import Run + +PROMPT = "Do the task." +DATA_URL = re.compile(r"^data:(?P[\w/+.-]+);base64,(?P.*)$", re.DOTALL) + + +def workspace_env( + root: Path, + *, + files: Mapping[str, bytes | str] | None = None, + passes: Callable[[Path], bool] = lambda _root: True, + capabilities: tuple[Capability, ...] = (), + name: str = "ws", + **workspace: Any, +) -> Environment: + """An environment serving ``root`` over SSH with one template, ``task``. + + ``files`` seed the workspace before each run; the task scores 1.0 when + ``passes(root)`` holds after the agent finishes. ``workspace`` goes to + :meth:`Environment.workspace` (``env=`` for the shell environment). + """ + env = Environment(name, capabilities=capabilities) + env.workspace(root, guest_path=str(root), **workspace) + + @env.initialize + async def seed() -> None: + write_files(root, files or {}) + + @env.template() + async def task(prompt: str = PROMPT): + yield prompt + yield 1.0 if passes(root) else 0.0 + + return env + + +async def run_task(env: Environment, agent: Agent, *, prompt: str = PROMPT, **options: Any) -> Run: + """Roll ``agent`` out on ``env``'s ``task`` in this process.""" + args = {} if prompt == PROMPT else {"prompt": prompt} + return await rollout( + Task(env=env.name, id="task", args=args), agent, runtime=LocalRuntime(env), **options + ) + + +def write_files(root: Path, files: Mapping[str, bytes | str]) -> None: + root.mkdir(parents=True, exist_ok=True) + for relative, content in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, str): + path.write_text(content) + else: + path.write_bytes(content) + + +@asynccontextmanager +async def mcp_server( + server: FastMCP, + *, + name: str = "tools", + transport: Literal["streamable-http", "sse"] = "streamable-http", +) -> AsyncIterator[Capability]: + """Serve ``server`` over HTTP on loopback; yield the capability that reaches it.""" + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + path = "/mcp" if transport == "streamable-http" else "/sse" + app = server.http_app(path=path, transport=transport) + http = uvicorn.Server(uvicorn.Config(app, log_level="error", lifespan="on")) + serving = asyncio.create_task(http.serve(sockets=[sock])) + while not http.started: + if serving.done(): + await serving + raise RuntimeError("MCP server stopped before it started") + await asyncio.sleep(0.01) + try: + yield Capability.mcp(name=name, url=f"http://127.0.0.1:{port}{path}", transport=transport) + finally: + http.should_exit = True + await serving + + +def image( + size: tuple[int, int], + *, + format: str = "PNG", + mode: str = "RGB", + color: tuple[int, ...] | str = "white", + exif: Image.Exif | None = None, +) -> bytes: + buffer = io.BytesIO() + options: dict[str, Any] = {} if exif is None else {"exif": exif} + Image.new(mode, size, color).save(buffer, format=format, **options) + return buffer.getvalue() + + +def describe_image(mime: str, data: str) -> str: + """```` for base64 image data, so a snapshot pins type and size, not bytes.""" + try: + raw = base64.b64decode(data, altchars=b"-_" if "-" in data or "_" in data else None) + with Image.open(io.BytesIO(raw)) as decoded: + return f"<{mime} {decoded.width}x{decoded.height}>" + except (OSError, ValueError): + return f"<{mime} undecodable>" + + +def wire(value: Any, *paths: Path, descriptions: bool = False) -> Any: + """A request body as a snapshot compares it. + + Base64 images become ````, each of ``paths`` becomes ````, + ````, ... wherever it appears in a string, and tool and schema + ``description`` prose is dropped unless ``descriptions`` is set. + """ + if isinstance(value, dict): + mime = value.get("media_type") or value.get("mimeType") or value.get("mime_type") + return { + key: ( + describe_image(mime, item) + if key == "data" and isinstance(mime, str) and mime.startswith("image/") + else wire(item, *paths, descriptions=descriptions) + ) + for key, item in value.items() + if descriptions or key != "description" or not isinstance(item, str) + } + if isinstance(value, list): + return [wire(item, *paths, descriptions=descriptions) for item in value] + if isinstance(value, str): + if (match := DATA_URL.match(value)) and match["mime"].startswith("image/"): + return describe_image(match["mime"], match["data"]) + for index, path in enumerate(paths): + value = value.replace(str(path), f"") + return value + return value + + +def json_lines(text: str) -> list[dict[str, Any]]: + return [json.loads(line) for line in text.splitlines() if line.strip()] diff --git a/tests/agents/test_provider_requests.py b/tests/agents/test_provider_requests.py new file mode 100644 index 000000000..39acd7fd1 --- /dev/null +++ b/tests/agents/test_provider_requests.py @@ -0,0 +1,1019 @@ +"""Each provider agent drives a real workspace through its own wire protocol. + +A scripted provider behind the fake gateway answers the agent; the workspace is +served over real SSH. Rows assert the reward (the grader reads the file the +agent wrote), the step sequence the run exported, and the last request +body the provider received, which carries the whole conversation (or, for the +Responses API, the items new since the previous response). +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Any + +import pytest +from fastmcp import FastMCP +from inline_snapshot import snapshot + +from hud.agents import ClaudeAgent, GeminiAgent, OpenAIAgent, OpenAIChatAgent +from hud.agents.claude import ClaudeToolSearchTool, ClaudeWebFetchTool, ClaudeWebSearchTool +from hud.agents.gemini.tools import ( + GeminiCodeExecutionTool, + GeminiGoogleSearchTool, + GeminiUrlContextTool, +) +from hud.agents.openai.tools import OpenAICodeInterpreterTool, OpenAIToolSearchTool +from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig +from tests.agents.support import image, mcp_server, run_task, wire, workspace_env +from tests.harness import call, say, shell_call, steps + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hud.agents.base import Agent + from tests.harness import HudEnv, Models, Turn + + +@dataclass(frozen=True) +class Row: + agent: Callable[[], Agent] + script: Callable[[Path], list[Turn]] + last_request: Any + steps: Any + + +ROWS = { + "claude": Row( + agent=lambda: ClaudeAgent( + ClaudeConfig(model="claude-sonnet-4-6", system_prompt="Be brief.", max_tokens=1024) + ), + script=lambda root: [ + replace( + call( + "str_replace_based_edit_tool", + command="create", + path=f"{root}/a.txt", + file_text="PASS", + ), + reasoning="plan", + ), + call("str_replace_based_edit_tool", command="view", path=f"{root}/shot.png"), + call("bash", command="cat a.txt"), + say("done", reasoning="checked"), + ], + last_request=snapshot( + { + "max_tokens": 1024, + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "Do the task."}]}, + { + "role": "assistant", + "content": [ + {"signature": "sig", "thinking": "plan", "type": "thinking"}, + { + "id": "toolu_1", + "input": { + "command": "create", + "path": "/a.txt", + "file_text": "PASS", + }, + "name": "str_replace_based_edit_tool", + "type": "tool_use", + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": [ + {"type": "text", "text": "wrote 4 bytes to /a.txt"} + ], + "is_error": False, + } + ], + }, + { + "role": "assistant", + "content": [ + { + "id": "toolu_3", + "input": {"command": "view", "path": "/shot.png"}, + "name": "str_replace_based_edit_tool", + "type": "tool_use", + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_3", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "", + }, + } + ], + "is_error": False, + } + ], + }, + { + "role": "assistant", + "content": [ + { + "id": "toolu_5", + "input": {"command": "cat a.txt"}, + "name": "bash", + "type": "tool_use", + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_5", + "content": [ + { + "type": "text", + "text": """\ +$ cat a.txt +PASS +(exit 0)\ +""", + } + ], + "is_error": False, + "cache_control": {"type": "ephemeral"}, + } + ], + }, + ], + "model": "claude-sonnet-4-6", + "system": "Be brief.", + "tools": [ + {"type": "bash_20250124", "name": "bash"}, + {"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, + ], + "tool_choice": {"type": "auto", "disable_parallel_tool_use": True}, + "stream": True, + } + ), + steps=snapshot( + ["task", "user", "agent", "tool", "agent", "tool", "agent", "tool", "agent", "task"] + ), + ), + "openai": Row( + agent=lambda: OpenAIAgent( + OpenAIConfig(model="gpt-5.6", system_prompt="Be brief.", prompt_cache_key="cache-1") + ), + script=lambda _root: [ + replace(shell_call("printf PASS > a.txt", "cat a.txt"), reasoning="plan"), + say("done", reasoning="checked"), + ], + last_request=snapshot( + { + "input": [ + { + "type": "shell_call_output", + "call_id": "call_shell", + "status": "completed", + "output": [ + { + "stdout": "", + "stderr": "", + "outcome": {"type": "exit", "exit_code": 0}, + }, + { + "stdout": "PASS", + "stderr": "", + "outcome": {"type": "exit", "exit_code": 0}, + }, + ], + "max_output_length": 10485760, + } + ], + "instructions": "Be brief.", + "max_output_tokens": None, + "model": "gpt-5.6", + "parallel_tool_calls": None, + "previous_response_id": "resp_1", + "prompt_cache_key": "cache-1", + "temperature": None, + "tools": [{"type": "shell", "environment": {"type": "local"}}], + } + ), + steps=snapshot(["task", "user", "agent", "tool", "agent", "task"]), + ), + "gemini": Row( + agent=lambda: GeminiAgent( + GeminiConfig(model="gemini-3.1-pro-preview", system_prompt="Be brief.") + ), + script=lambda root: [ + replace( + call("write_file", file_path=f"{root}/a.txt", content="PASS"), reasoning="plan" + ), + call("run_shell_command", command="cat a.txt", dir_path=str(root)), + call("list_directory", dir_path=str(root)), + say("done", reasoning="checked"), + ], + last_request=snapshot( + { + "contents": [ + {"parts": [{"text": "Do the task."}], "role": "user"}, + { + "parts": [ + {"text": "plan", "thought": True}, + { + "functionCall": { + "args": {"file_path": "/a.txt", "content": "PASS"}, + "name": "write_file", + } + }, + ], + "role": "model", + }, + { + "parts": [ + { + "functionResponse": { + "name": "write_file", + "response": { + "success": True, + "output": "wrote 4 bytes to /a.txt", + }, + } + } + ], + "role": "user", + }, + { + "parts": [ + { + "functionCall": { + "args": {"command": "cat a.txt", "dir_path": ""}, + "name": "run_shell_command", + } + } + ], + "role": "model", + }, + { + "parts": [ + { + "functionResponse": { + "name": "run_shell_command", + "response": { + "success": True, + "output": """\ +$ cd && cat a.txt +PASS +(exit 0)\ +""", + }, + } + } + ], + "role": "user", + }, + { + "parts": [ + { + "functionCall": { + "args": {"dir_path": ""}, + "name": "list_directory", + } + } + ], + "role": "model", + }, + { + "parts": [ + { + "functionResponse": { + "name": "list_directory", + "response": { + "success": True, + "output": """\ +a.txt +shot.png\ +""", + }, + } + } + ], + "role": "user", + }, + ], + "systemInstruction": {"parts": [{"text": "Be brief."}], "role": "user"}, + "tools": [ + { + "functionDeclarations": [ + { + "name": "run_shell_command", + "parameters_json_schema": { + "type": "object", + "properties": { + "command": {"type": "string"}, + "description": {"type": "string"}, + "dir_path": {"type": "string"}, + }, + "required": ["command"], + }, + }, + { + "name": "replace", + "parameters_json_schema": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "instruction": {"type": "string"}, + "old_string": {"type": "string"}, + "new_string": {"type": "string"}, + }, + "required": ["file_path", "old_string", "new_string"], + }, + }, + { + "name": "write_file", + "parameters_json_schema": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "content": {"type": "string"}, + }, + "required": ["file_path", "content"], + }, + }, + { + "name": "read_file", + "parameters_json_schema": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "start_line": {"type": "integer"}, + "end_line": {"type": "integer"}, + }, + "required": ["file_path"], + }, + }, + { + "name": "grep_search", + "parameters_json_schema": { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "dir_path": {"type": "string"}, + "include_pattern": {"type": "string"}, + }, + "required": ["pattern"], + }, + }, + { + "name": "glob", + "parameters_json_schema": { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "dir_path": {"type": "string"}, + }, + "required": ["pattern"], + }, + }, + { + "name": "list_directory", + "parameters_json_schema": { + "type": "object", + "properties": {"dir_path": {"type": "string"}}, + "required": ["dir_path"], + }, + }, + ] + } + ], + "generationConfig": { + "temperature": 1.0, + "topP": 0.95, + "topK": 40.0, + "maxOutputTokens": 8192, + "thinkingConfig": {"include_thoughts": True}, + }, + } + ), + steps=snapshot( + ["task", "user", "agent", "tool", "agent", "tool", "agent", "tool", "agent", "task"] + ), + ), + "chat": Row( + agent=lambda: OpenAIChatAgent( + OpenAIChatConfig(model="qwen3.6-plus", system_prompt="Be brief.") + ), + script=lambda root: [ + replace(call("write", filePath=f"{root}/a.txt", content="PASS"), reasoning="plan"), + call("read", filePath=f"{root}/shot.png"), + call("bash", command="cat a.txt"), + say("done", reasoning="checked"), + ], + last_request=snapshot( + { + "messages": [ + {"role": "system", "content": "Be brief."}, + {"role": "user", "content": [{"type": "text", "text": "Do the task."}]}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "write", + "arguments": '{"filePath": "/a.txt", "content": "PASS"}', + }, + } + ], + "reasoning_content": "plan", + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "wrote 4 bytes to /a.txt", + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_3", + "type": "function", + "function": { + "name": "read", + "arguments": '{"filePath": "/shot.png"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_3", + "content": "Tool executed successfully", + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Tool returned the following:"}, + {"type": "image_url", "image_url": {"url": ""}}, + ], + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_5", + "type": "function", + "function": { + "name": "bash", + "arguments": '{"command": "cat a.txt"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_5", + "content": """\ +$ cat a.txt +PASS +(exit 0)\ +""", + }, + ], + "model": "qwen3.6-plus", + "stream": False, + "tools": [ + { + "type": "function", + "function": { + "name": "bash", + "parameters": { + "type": "object", + "properties": { + "command": {"type": "string"}, + "timeout": {"type": "integer", "minimum": 1}, + "workdir": {"type": "string"}, + }, + "required": ["command"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read", + "parameters": { + "type": "object", + "properties": { + "filePath": {"type": "string"}, + "offset": {"type": "integer", "minimum": 0}, + "limit": {"type": "integer", "minimum": 1}, + }, + "required": ["filePath"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "glob", + "parameters": { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "path": {"type": "string"}, + }, + "required": ["pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "grep", + "parameters": { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "path": {"type": "string"}, + "include": {"type": "string"}, + }, + "required": ["pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "edit", + "parameters": { + "type": "object", + "properties": { + "filePath": {"type": "string"}, + "oldString": {"type": "string"}, + "newString": {"type": "string"}, + "replaceAll": {"type": "boolean"}, + }, + "required": ["filePath", "oldString", "newString"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "write", + "parameters": { + "type": "object", + "properties": { + "content": {"type": "string"}, + "filePath": {"type": "string"}, + }, + "required": ["content", "filePath"], + }, + }, + }, + ], + } + ), + steps=snapshot( + ["task", "user", "agent", "tool", "agent", "tool", "agent", "tool", "agent", "task"] + ), + ), +} + + +@pytest.mark.parametrize("row", ROWS.values(), ids=ROWS.keys()) +async def test_a_provider_agent_works_the_workspace_over_its_wire_protocol( + row: Row, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + root = tmp_path / "ws" + env = workspace_env( + root, + files={"shot.png": image((40, 30))}, + passes=lambda root: (root / "a.txt").read_text() == "PASS", + ) + models.script(row.script(root)) + + run = await run_task(env, row.agent()) + + assert run.reward == 1.0 + assert wire(models.requests()[-1].body, root) == row.last_request + assert [step["source"] for step in steps(run.trace_id)] == row.steps + + +def lookup_tools() -> FastMCP: + server = FastMCP("lookup") + + @server.tool() + def lookup(key: str) -> str: + """Look a key up.""" + return key + + @server.tool() + def store(key: str, value: str) -> str: + """Store a value.""" + return value + + return server + + +@dataclass(frozen=True) +class HostedRow: + agent: Callable[[], Agent] + tools: Any + + +HOSTED = { + "claude": HostedRow( + agent=lambda: ClaudeAgent( + ClaudeConfig( + model="claude-sonnet-4-6", + hosted_tools=[ + ClaudeWebSearchTool(max_uses=2, allowed_domains=["example.com"]), + ClaudeWebFetchTool(max_content_tokens=100, citations_enabled=True), + ClaudeWebSearchTool(supported_models=("claude-opus-*",)), + ], + ) + ), + tools=snapshot( + [ + {"type": "bash_20250124", "name": "bash"}, + {"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, + { + "name": "lookup", + "input_schema": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "eager_input_streaming": True, + }, + { + "name": "store", + "input_schema": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}, "value": {"type": "string"}}, + "required": ["key", "value"], + "type": "object", + }, + "eager_input_streaming": True, + }, + { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 2, + "allowed_domains": ["example.com"], + }, + { + "type": "web_fetch_20250910", + "name": "web_fetch", + "max_content_tokens": 100, + "citations": {"enabled": True}, + }, + ] + ), + ), + "claude-tool-search-under-threshold": HostedRow( + agent=lambda: ClaudeAgent( + ClaudeConfig( + model="claude-sonnet-4-6", hosted_tools=[ClaudeToolSearchTool(threshold=5)] + ) + ), + tools=snapshot( + [ + {"type": "bash_20250124", "name": "bash"}, + {"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, + { + "name": "lookup", + "input_schema": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "eager_input_streaming": True, + }, + { + "name": "store", + "input_schema": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}, "value": {"type": "string"}}, + "required": ["key", "value"], + "type": "object", + }, + "eager_input_streaming": True, + }, + {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}, + ] + ), + ), + "openai": HostedRow( + agent=lambda: OpenAIAgent( + OpenAIConfig( + model="gpt-5.6", + hosted_tools=[OpenAICodeInterpreterTool(container={"type": "auto"})], + ) + ), + tools=snapshot( + [ + {"type": "shell", "environment": {"type": "local"}}, + { + "type": "function", + "name": "lookup", + "parameters": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "strict": True, + }, + { + "type": "function", + "name": "store", + "parameters": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}, "value": {"type": "string"}}, + "required": ["key", "value"], + "type": "object", + }, + "strict": True, + }, + {"type": "code_interpreter", "container": {"type": "auto"}}, + ] + ), + ), + "openai-tool-search-over-threshold": HostedRow( + agent=lambda: OpenAIAgent( + OpenAIConfig(model="gpt-5.6", hosted_tools=[OpenAIToolSearchTool(threshold=1)]) + ), + tools=snapshot( + [ + {"type": "shell", "environment": {"type": "local"}}, + { + "type": "function", + "name": "lookup", + "parameters": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "strict": True, + "defer_loading": True, + }, + { + "type": "function", + "name": "store", + "parameters": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}, "value": {"type": "string"}}, + "required": ["key", "value"], + "type": "object", + }, + "strict": True, + "defer_loading": True, + }, + {"type": "tool_search"}, + ] + ), + ), + "openai-tool-search-under-threshold": HostedRow( + agent=lambda: OpenAIAgent( + OpenAIConfig(model="gpt-5.6", hosted_tools=[OpenAIToolSearchTool(threshold=5)]) + ), + tools=snapshot( + [ + {"type": "shell", "environment": {"type": "local"}}, + { + "type": "function", + "name": "lookup", + "parameters": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "strict": True, + }, + { + "type": "function", + "name": "store", + "parameters": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}, "value": {"type": "string"}}, + "required": ["key", "value"], + "type": "object", + }, + "strict": True, + }, + {"type": "tool_search"}, + ] + ), + ), + "gemini": HostedRow( + agent=lambda: GeminiAgent( + GeminiConfig( + model="gemini-3.1-pro-preview", + hosted_tools=[ + GeminiGoogleSearchTool(), + GeminiUrlContextTool(), + GeminiCodeExecutionTool(), + ], + ) + ), + tools=snapshot( + [ + {"googleSearch": {}}, + {"urlContext": {}}, + {"codeExecution": {}}, + { + "functionDeclarations": [ + { + "name": "run_shell_command", + "parameters_json_schema": { + "type": "object", + "properties": { + "command": {"type": "string"}, + "description": {"type": "string"}, + "dir_path": {"type": "string"}, + }, + "required": ["command"], + }, + }, + { + "name": "replace", + "parameters_json_schema": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "instruction": {"type": "string"}, + "old_string": {"type": "string"}, + "new_string": {"type": "string"}, + }, + "required": ["file_path", "old_string", "new_string"], + }, + }, + { + "name": "write_file", + "parameters_json_schema": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "content": {"type": "string"}, + }, + "required": ["file_path", "content"], + }, + }, + { + "name": "read_file", + "parameters_json_schema": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "start_line": {"type": "integer"}, + "end_line": {"type": "integer"}, + }, + "required": ["file_path"], + }, + }, + { + "name": "grep_search", + "parameters_json_schema": { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "dir_path": {"type": "string"}, + "include_pattern": {"type": "string"}, + }, + "required": ["pattern"], + }, + }, + { + "name": "glob", + "parameters_json_schema": { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "dir_path": {"type": "string"}, + }, + "required": ["pattern"], + }, + }, + { + "name": "list_directory", + "parameters_json_schema": { + "type": "object", + "properties": {"dir_path": {"type": "string"}}, + "required": ["dir_path"], + }, + }, + { + "name": "lookup", + "parameters_json_schema": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + }, + { + "name": "store", + "parameters_json_schema": { + "additionalProperties": False, + "properties": { + "key": {"type": "string"}, + "value": {"type": "string"}, + }, + "required": ["key", "value"], + "type": "object", + }, + }, + ] + }, + ] + ), + ), +} + + +@pytest.mark.parametrize("row", HOSTED.values(), ids=HOSTED.keys()) +async def test_configured_hosted_tools_join_the_request( + row: HostedRow, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + """A hosted tool the model does not support stays out; tool search defers MCP tools + once there are more of them than its threshold.""" + hud_env.set(HUD_API_KEY="k") + models.script([say("done")]) + + async with mcp_server(lookup_tools()) as tools: + await run_task(workspace_env(tmp_path / "ws", capabilities=(tools,)), row.agent()) + + (request,) = models.requests() + assert wire(request.body["tools"]) == row.tools + + +INVALID_HOSTED = { + "claude-both-domain-lists": ( + ClaudeAgent, + ClaudeConfig, + ClaudeWebSearchTool(allowed_domains=["a.com"], blocked_domains=["b.com"]), + "[agent loop] ValueError: Use either allowed_domains or blocked_domains, not both.", + ), + "claude-fetch-both-domain-lists": ( + ClaudeAgent, + ClaudeConfig, + ClaudeWebFetchTool(allowed_domains=["a.com"], blocked_domains=["b.com"]), + "[agent loop] ValueError: Use either allowed_domains or blocked_domains, not both.", + ), + "gemini-dynamic-threshold": ( + GeminiAgent, + GeminiConfig, + GeminiGoogleSearchTool(dynamic_threshold=0.5), + "[agent loop] ValueError: dynamic_threshold is not supported by Gemini Google Search.", + ), +} + + +@pytest.mark.parametrize( + ("agent_cls", "config_cls", "hosted", "error"), + INVALID_HOSTED.values(), + ids=INVALID_HOSTED.keys(), +) +async def test_an_invalid_hosted_tool_fails_the_run_before_any_request( + agent_cls: Any, + config_cls: Any, + hosted: Any, + error: str, + models: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(HUD_API_KEY="k") + + run = await run_task( + workspace_env(tmp_path / "ws"), agent_cls(config_cls(hosted_tools=[hosted])) + ) + + assert (run.trace.status, run.trace.error) == ("error", error) + assert models.requests() == [] diff --git a/tests/agents/test_step_spans.py b/tests/agents/test_step_spans.py new file mode 100644 index 000000000..4ae7e7f55 --- /dev/null +++ b/tests/agents/test_step_spans.py @@ -0,0 +1,566 @@ +"""The steps a rollout exports carry each provider's turn in the shared step schema. + +The platform reads ``hud.step.v1`` span payloads; these rows pin what an agent +turn, a tool round-trip and a nested rollout look like there, per provider: +content, reasoning, citations, refusals, usage, model and training samples. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import pytest +from dirty_equals import IsStr +from inline_snapshot import snapshot + +from hud.agents import ClaudeAgent, GeminiAgent, OpenAIAgent, OpenAIChatAgent +from hud.agents.base import Agent +from hud.agents.types import ( + ClaudeConfig, + GeminiConfig, + OpenAIChatConfig, + OpenAIConfig, + SubagentStep, +) +from hud.types import Trace +from tests.agents.support import run_task, workspace_env +from tests.harness import Turn, call, say, steps + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hud.eval import Run + from tests.harness import HudEnv, Models + +TIMESTAMP = IsStr(regex=r"\d{4}-\d{2}-\d{2}T.*") + + +def agent_steps(payloads: list[dict[str, Any]]) -> list[dict[str, Any]]: + """The agent, tool and subagent step payloads, with timestamps checked and dropped.""" + found = [] + for payload in payloads: + if payload["source"] not in {"agent", "tool", "subagent"}: + continue + for key in ("started_at", "ended_at"): + if key in payload: + assert payload.pop(key) == TIMESTAMP + found.append(payload) + return found + + +@dataclass(frozen=True) +class Row: + agent: Callable[[], Agent] + script: list[Turn] + steps: Any + + +CLAUDE_CITATIONS = ( + { + "type": "char_location", + "cited_text": "the sky is blue", + "document_index": 0, + "document_title": "notes", + "start_char_index": 4, + "end_char_index": 19, + }, + { + "type": "web_search_result_location", + "cited_text": "blue skies", + "url": "https://example.com/sky", + "title": "Sky", + "encrypted_index": "x", + }, +) + +ROWS = { + "claude": Row( + agent=lambda: ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")), + script=[ + Turn( + reasoning="look it up", + native=({"type": "text", "text": "Blue.", "citations": list(CLAUDE_CITATIONS)},), + ) + ], + steps=snapshot( + [ + { + "step_id": 3, + "source": "agent", + "messages": [], + "extra": {}, + "content": "Blue.", + "reasoning": "look it up", + "tool_calls": [], + "done": True, + "finish_reason": "end_turn", + "citations": [ + { + "type": "document_citation", + "text": "the sky is blue", + "source": "0", + "title": "notes", + "start_index": 4, + "end_index": 19, + }, + { + "type": "web_search_result_location", + "text": "blue skies", + "source": "https://example.com/sky", + "title": "Sky", + }, + ], + "model": "claude-sonnet-4-6", + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ] + ), + ), + "claude-refusal": Row( + agent=lambda: ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")), + script=[ + Turn( + finish="refusal", + extra={ + "stop_details": { + "type": "refusal", + "category": "cyber", + "explanation": "This request was blocked under the Usage Policy.", + } + }, + ) + ], + steps=snapshot( + [ + { + "step_id": 3, + "source": "agent", + "messages": [], + "extra": {}, + "content": "", + "tool_calls": [], + "done": True, + "finish_reason": "refusal", + "refusal": "This request was blocked under the Usage Policy.", + "citations": [], + "model": "claude-sonnet-4-6", + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ] + ), + ), + "claude-refusal-category": Row( + agent=lambda: ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")), + script=[ + Turn( + finish="refusal", + extra={ + "stop_details": {"type": "refusal", "category": "cyber", "explanation": None} + }, + ) + ], + steps=snapshot( + [ + { + "step_id": 3, + "source": "agent", + "messages": [], + "extra": {}, + "content": "", + "tool_calls": [], + "done": True, + "finish_reason": "refusal", + "refusal": "This request was refused (cyber).", + "citations": [], + "model": "claude-sonnet-4-6", + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ] + ), + ), + "openai": Row( + agent=lambda: OpenAIAgent(OpenAIConfig(model="gpt-5.6")), + script=[ + Turn( + reasoning="look it up", + native=( + { + "type": "message", + "id": "msg_cited", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Blue.", + "annotations": [ + { + "type": "url_citation", + "url": "https://example.com/sky", + "title": "Sky", + "start_index": 0, + "end_index": 5, + }, + { + "type": "file_citation", + "file_id": "file-1", + "filename": "notes.txt", + "index": 0, + }, + ], + } + ], + }, + ), + ) + ], + steps=snapshot( + [ + { + "step_id": 3, + "source": "agent", + "messages": [], + "extra": {}, + "content": "Blue.", + "reasoning": "look it up", + "tool_calls": [], + "done": True, + "citations": [ + { + "type": "url_citation", + "text": "Sky", + "source": "https://example.com/sky", + "title": "Sky", + "start_index": 0, + "end_index": 5, + }, + { + "type": "file_citation", + "text": "notes.txt", + "source": "file-1", + "title": "notes.txt", + }, + ], + "model": "gpt-5.6", + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "cached_tokens": 0}, + } + ] + ), + ), + "gemini": Row( + agent=lambda: GeminiAgent(GeminiConfig(model="gemini-3.1-pro-preview")), + script=[ + Turn( + text="Blue.", + reasoning="look it up", + extra={ + "groundingMetadata": { + "groundingChunks": [ + {"web": {"uri": "https://example.com/sky", "title": "Sky"}}, + {"web": {"uri": "https://example.com/unused", "title": "Unused"}}, + ], + "groundingSupports": [ + { + "segment": {"text": "Blue.", "startIndex": 0, "endIndex": 5}, + "groundingChunkIndices": [0], + } + ], + } + }, + ) + ], + steps=snapshot( + [ + { + "step_id": 3, + "source": "agent", + "messages": [], + "extra": {}, + "content": "Blue.", + "reasoning": "look it up", + "tool_calls": [], + "done": True, + "finish_reason": "STOP", + "citations": [ + { + "type": "grounding", + "text": "Blue.", + "source": "https://example.com/sky", + "title": "Sky", + "start_index": 0, + "end_index": 5, + }, + { + "type": "grounding", + "text": "", + "source": "https://example.com/unused", + "title": "Unused", + }, + ], + "model": "gemini-3.1-pro-preview", + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ] + ), + ), + "claude-tool": Row( + agent=lambda: ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")), + script=[call("bash", command="echo hi"), say("done")], + steps=snapshot( + [ + { + "step_id": 3, + "source": "agent", + "messages": [], + "extra": {}, + "content": "", + "tool_calls": [ + { + "meta": {"citations_enabled": False}, + "name": "bash", + "arguments": {"command": "echo hi"}, + "id": "toolu_1", + } + ], + "done": False, + "finish_reason": "tool_use", + "citations": [], + "model": "claude-sonnet-4-6", + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + }, + { + "step_id": 4, + "source": "tool", + "messages": [], + "extra": {}, + "call": { + "meta": {"citations_enabled": False}, + "name": "bash", + "arguments": {"command": "echo hi"}, + "id": "toolu_1", + }, + "result": { + "content": [ + { + "type": "text", + "text": """\ +$ echo hi +hi + +(exit 0)\ +""", + } + ], + "isError": False, + }, + }, + { + "step_id": 5, + "source": "agent", + "messages": [], + "extra": {}, + "content": "done", + "tool_calls": [], + "done": True, + "finish_reason": "end_turn", + "citations": [], + "model": "claude-sonnet-4-6", + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + }, + ] + ), + ), + "chat-training": Row( + agent=lambda: OpenAIChatAgent( + OpenAIChatConfig( + model="qwen3.6-plus", + completion_kwargs={"extra_body": {"return_token_ids": True}}, + ) + ), + script=[ + Turn( + text="Blue.", + extra={ + "prompt_token_ids": [1, 2], + "token_ids": [3, 4], + "logprobs": { + "content": [ + {"token": "Bl", "logprob": -0.5, "bytes": None, "top_logprobs": []}, + {"token": "ue.", "logprob": -0.25, "bytes": None, "top_logprobs": []}, + ] + }, + }, + ) + ], + steps=snapshot( + [ + { + "step_id": 3, + "source": "agent", + "messages": [], + "extra": {}, + "content": "Blue.", + "tool_calls": [], + "done": True, + "finish_reason": "stop", + "citations": [], + "raw": { + "id": "chatcmpl_1", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": { + "content": [ + { + "token": "Bl", + "bytes": None, + "logprob": -0.5, + "top_logprobs": [], + }, + { + "token": "ue.", + "bytes": None, + "logprob": -0.25, + "top_logprobs": [], + }, + ], + "refusal": None, + }, + "message": { + "content": "Blue.", + "refusal": None, + "role": "assistant", + "annotations": None, + "audio": None, + "function_call": None, + "tool_calls": None, + }, + "prompt_token_ids": [1, 2], + "token_ids": [3, 4], + } + ], + "created": 0, + "model": "qwen3.6-plus", + "object": "chat.completion", + "moderation": None, + "service_tier": None, + "system_fingerprint": None, + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2, + "completion_tokens_details": None, + "prompt_tokens_details": None, + }, + }, + "model": "qwen3.6-plus", + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + "sample": { + "prompt_token_ids": [1, 2], + "output_token_ids": [3, 4], + "output_logprobs": [-0.5, -0.25], + }, + } + ] + ), + ), + "chat": Row( + agent=lambda: OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus")), + script=[Turn(text="Blue.", reasoning="look it up")], + steps=snapshot( + [ + { + "step_id": 3, + "source": "agent", + "messages": [], + "extra": {}, + "content": "Blue.", + "reasoning": "look it up", + "tool_calls": [], + "done": True, + "finish_reason": "stop", + "citations": [], + "raw": { + "id": "chatcmpl_1", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": { + "content": "Blue.", + "refusal": None, + "role": "assistant", + "annotations": None, + "audio": None, + "function_call": None, + "tool_calls": None, + "reasoning_content": "look it up", + }, + } + ], + "created": 0, + "model": "qwen3.6-plus", + "object": "chat.completion", + "moderation": None, + "service_tier": None, + "system_fingerprint": None, + "usage": { + "completion_tokens": 1, + "prompt_tokens": 1, + "total_tokens": 2, + "completion_tokens_details": None, + "prompt_tokens_details": None, + }, + }, + "model": "qwen3.6-plus", + "usage": {"prompt_tokens": 1, "completion_tokens": 1}, + } + ] + ), + ), +} + + +@pytest.mark.parametrize("row", ROWS.values(), ids=ROWS.keys()) +async def test_a_provider_turn_exports_as_an_agent_step( + row: Row, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script(row.script) + + run = await run_task(workspace_env(tmp_path / "ws"), row.agent()) + + assert agent_steps(steps(run.trace_id)) == row.steps + + +class Delegating(Agent): + """Records a nested rollout the way a delegating agent reports one.""" + + async def __call__(self, run: Run) -> None: + run.record(SubagentStep(subagent=Trace(content="inner answer", status="completed"))) + run.trace.content = "outer answer" + + +async def test_a_nested_rollout_exports_as_a_subagent_step(tmp_path: Path) -> None: + run = await run_task(workspace_env(tmp_path / "ws"), Delegating()) + + assert agent_steps(steps(run.trace_id)) == snapshot( + [ + { + "step_id": 3, + "source": "subagent", + "messages": [], + "extra": {}, + "subagent": { + "steps": [], + "status": "completed", + "content": "inner answer", + "extra": {}, + }, + } + ] + ) diff --git a/tests/agents/test_tool_results.py b/tests/agents/test_tool_results.py new file mode 100644 index 000000000..b135db9bb --- /dev/null +++ b/tests/agents/test_tool_results.py @@ -0,0 +1,722 @@ +"""How each provider agent hands a tool's result back to its model. + +The environment serves an MCP server whose tools return each kind of MCP +content: text, nothing, an error, an image, a PDF and structured data. The +scripted model calls every one; the rows pin the result entries the provider +received on its next requests. +""" + +from __future__ import annotations + +import base64 +import io +import json +import random +import struct +import zlib +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import mcp.types as mcp_types +import pytest +from dirty_equals import IsStr +from fastmcp import FastMCP +from fastmcp.exceptions import ToolError +from inline_snapshot import snapshot +from PIL import Image + +from hud.agents import ClaudeAgent, GeminiAgent, OpenAIAgent, OpenAIChatAgent +from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig +from tests.agents.support import image, mcp_server, run_task, wire, workspace_env +from tests.harness import call, say + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hud.agents.base import Agent + from tests.harness import HudEnv, ModelRequest, Models + +PDF = base64.b64encode(b"%PDF-1.4 scripted").decode() + + +def result_tools() -> FastMCP: + server = FastMCP("results") + + @server.tool() + def note() -> str: + """Return a short note.""" + return "the note" + + @server.tool() + def nothing() -> list[mcp_types.ContentBlock]: + """Return no content.""" + return [] + + @server.tool() + def broken() -> str: + """Always fail.""" + raise ToolError("broken tool") + + @server.tool() + def picture() -> list[mcp_types.ContentBlock]: + """Return a picture.""" + data = base64.b64encode(image((20, 10))).decode() + return [mcp_types.ImageContent(type="image", data=data, mimeType="image/png")] + + @server.tool() + def report() -> list[mcp_types.ContentBlock]: + """Return a PDF report.""" + return [ + mcp_types.EmbeddedResource( + type="resource", + resource=mcp_types.BlobResourceContents( + uri="file:///report.pdf", mimeType="application/pdf", blob=PDF + ), + ) + ] + + @server.tool() + def figures() -> dict[str, int]: + """Return structured figures.""" + return {"total": 3} + + @server.tool() + def long_output() -> str: + """Return a long text.""" + return "h" * 500 + "t" * 500 + + @server.tool() + def link() -> list[mcp_types.ContentBlock]: + """Return a resource link.""" + return [ + mcp_types.ResourceLink(type="resource_link", uri="file:///data.csv", name="data.csv") + ] + + return server + + +ALL_TOOLS = ("note", "nothing", "broken", "picture", "report", "figures") + + +def conversation(requests: list[ModelRequest]) -> Any: + """Everything after the prompt: the last request's history, or every Responses input.""" + last = requests[-1] + if last.protocol == "responses": + return [item for request in requests[1:] for item in request.body["input"]] + if last.protocol == "gemini": + return last.body["contents"][1:] + messages = last.body["messages"] + start = 2 if messages[0]["role"] == "system" else 1 + return messages[start:] + + +@dataclass(frozen=True) +class Row: + agent: Callable[[], Agent] + tools: tuple[str, ...] + results: Any + error: str | None = None + + +ROWS = { + "claude": Row( + agent=lambda: ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")), + tools=ALL_TOOLS, + results=snapshot( + [ + { + "role": "assistant", + "content": [{"id": "toolu_1", "input": {}, "name": "note", "type": "tool_use"}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": [{"type": "text", "text": "the note"}], + "is_error": False, + } + ], + }, + { + "role": "assistant", + "content": [ + {"id": "toolu_3", "input": {}, "name": "nothing", "type": "tool_use"} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_3", + "content": [], + "is_error": False, + } + ], + }, + { + "role": "assistant", + "content": [ + {"id": "toolu_5", "input": {}, "name": "broken", "type": "tool_use"} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_5", + "content": [{"type": "text", "text": "Error: broken tool"}], + "is_error": True, + } + ], + }, + { + "role": "assistant", + "content": [ + {"id": "toolu_7", "input": {}, "name": "picture", "type": "tool_use"} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_7", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "", + }, + } + ], + "is_error": False, + } + ], + }, + { + "role": "assistant", + "content": [ + {"id": "toolu_9", "input": {}, "name": "report", "type": "tool_use"} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_9", + "content": [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "JVBERi0xLjQgc2NyaXB0ZWQ=", + }, + } + ], + "is_error": False, + } + ], + }, + { + "role": "assistant", + "content": [ + {"id": "toolu_11", "input": {}, "name": "figures", "type": "tool_use"} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_11", + "content": [{"type": "text", "text": '{"total":3}'}], + "is_error": False, + "cache_control": {"type": "ephemeral"}, + } + ], + }, + ] + ), + ), + "claude-citations": Row( + agent=lambda: ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6", citations_enabled=True)), + tools=("note", "report", "broken"), + results=snapshot( + [ + { + "role": "assistant", + "content": [{"id": "toolu_1", "input": {}, "name": "note", "type": "tool_use"}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": [{"type": "text", "text": "the note"}], + "is_error": False, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "document", + "source": { + "type": "text", + "media_type": "text/plain", + "data": "the note", + }, + "title": "note", + "citations": {"enabled": True}, + } + ], + }, + { + "role": "assistant", + "content": [ + {"id": "toolu_3", "input": {}, "name": "report", "type": "tool_use"} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_3", + "content": [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "JVBERi0xLjQgc2NyaXB0ZWQ=", + }, + } + ], + "is_error": False, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "document", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "JVBERi0xLjQgc2NyaXB0ZWQ=", + }, + "citations": {"enabled": True}, + } + ], + }, + { + "role": "assistant", + "content": [ + {"id": "toolu_5", "input": {}, "name": "broken", "type": "tool_use"} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_5", + "content": [{"type": "text", "text": "Error: broken tool"}], + "is_error": True, + "cache_control": {"type": "ephemeral"}, + } + ], + }, + ] + ), + ), + "claude-resource-link": Row( + agent=lambda: ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")), + tools=("link",), + results=snapshot([]), + error="Unknown content block type: ", + ), + "openai": Row( + agent=lambda: OpenAIAgent(OpenAIConfig(model="gpt-5.6")), + tools=(*ALL_TOOLS, "link"), + results=snapshot( + [ + { + "type": "function_call_output", + "call_id": "call_3", + "output": [ + {"type": "input_text", "text": '{"result": "the note"}'}, + {"type": "input_text", "text": "the note"}, + ], + }, + { + "type": "function_call_output", + "call_id": "call_6", + "output": [{"type": "input_text", "text": ""}], + }, + { + "type": "function_call_output", + "call_id": "call_9", + "output": [ + {"type": "input_text", "text": "[tool_error] true"}, + {"type": "input_text", "text": "broken tool"}, + ], + }, + { + "type": "function_call_output", + "call_id": "call_12", + "output": [{"type": "input_image", "image_url": ""}], + }, + { + "type": "function_call_output", + "call_id": "call_15", + "output": [{"type": "input_file", "file_data": "JVBERi0xLjQgc2NyaXB0ZWQ="}], + }, + { + "type": "function_call_output", + "call_id": "call_18", + "output": [ + {"type": "input_text", "text": '{"total": 3}'}, + {"type": "input_text", "text": '{"total":3}'}, + ], + }, + { + "type": "function_call_output", + "call_id": "call_21", + "output": [{"type": "input_file", "file_url": "file:///data.csv"}], + }, + ] + ), + ), + "gemini": Row( + agent=lambda: GeminiAgent(GeminiConfig(model="gemini-3.1-pro-preview")), + tools=ALL_TOOLS, + results=snapshot( + [ + {"parts": [{"functionCall": {"args": {}, "name": "note"}}], "role": "model"}, + { + "parts": [ + { + "functionResponse": { + "name": "note", + "response": {"success": True, "output": "the note"}, + } + } + ], + "role": "user", + }, + {"parts": [{"functionCall": {"args": {}, "name": "nothing"}}], "role": "model"}, + { + "parts": [ + {"functionResponse": {"name": "nothing", "response": {"success": True}}} + ], + "role": "user", + }, + {"parts": [{"functionCall": {"args": {}, "name": "broken"}}], "role": "model"}, + { + "parts": [ + { + "functionResponse": { + "name": "broken", + "response": {"error": "broken tool"}, + } + } + ], + "role": "user", + }, + {"parts": [{"functionCall": {"args": {}, "name": "picture"}}], "role": "model"}, + { + "parts": [ + { + "functionResponse": { + "parts": [ + { + "inline_data": { + "mime_type": "image/png", + "data": "", + } + } + ], + "name": "picture", + "response": {"success": True}, + } + } + ], + "role": "user", + }, + {"parts": [{"functionCall": {"args": {}, "name": "report"}}], "role": "model"}, + { + "parts": [ + {"functionResponse": {"name": "report", "response": {"success": True}}} + ], + "role": "user", + }, + {"parts": [{"functionCall": {"args": {}, "name": "figures"}}], "role": "model"}, + { + "parts": [ + { + "functionResponse": { + "name": "figures", + "response": {"success": True, "output": '{"total":3}'}, + } + } + ], + "role": "user", + }, + ] + ), + ), + "chat": Row( + agent=lambda: OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus")), + tools=ALL_TOOLS, + results=snapshot( + [ + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "note", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "the note"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_3", + "type": "function", + "function": {"name": "nothing", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_3", "content": "Tool executed successfully"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_5", + "type": "function", + "function": {"name": "broken", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_5", "content": "broken tool"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_7", + "type": "function", + "function": {"name": "picture", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_7", "content": "Tool executed successfully"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Tool returned the following:"}, + {"type": "image_url", "image_url": {"url": ""}}, + ], + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_9", + "type": "function", + "function": {"name": "report", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_9", "content": "Tool executed successfully"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_11", + "type": "function", + "function": {"name": "figures", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_11", "content": '{"total":3}'}, + ] + ), + ), +} + + +@pytest.mark.parametrize("row", ROWS.values(), ids=ROWS.keys()) +async def test_each_kind_of_tool_result_reaches_the_model_in_its_provider_shape( + row: Row, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([*(call(name) for name in row.tools), say("done")]) + + async with mcp_server(result_tools()) as tools: + run = await run_task(workspace_env(tmp_path / "ws", capabilities=(tools,)), row.agent()) + + assert (run.trace.status, run.trace.error) == ( + "completed" if row.error is None else "error", + row.error, + ) + assert wire(conversation(models.requests())) == row.results + + +async def test_gemini_bounds_a_long_tool_output_to_its_head_and_tail( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([call("long_output"), say("done")]) + agent = GeminiAgent(GeminiConfig(model="gemini-3.1-pro-preview", max_tool_result_chars=100)) + + async with mcp_server(result_tools()) as tools: + await run_task(workspace_env(tmp_path / "ws", capabilities=(tools,)), agent) + + (result,) = models.requests()[-1].tool_results + assert json.loads(result) == { + "success": True, + "output": ( + "Output too large: showing the first 20 and last 80 characters, 900 omitted. " + "Re-run scoped to the part you need.\n" + + "h" * 20 + + "\n\n... [900 characters omitted] ...\n\n" + + "t" * 80 + ), + } + + +def noisy_rgba(size: tuple[int, int]) -> bytes: + noise = Image.frombytes("RGBA", size, random.Random(0).randbytes(size[0] * size[1] * 4)) + buffer = io.BytesIO() + noise.save(buffer, format="PNG") + return buffer.getvalue() + + +def png_claiming(size: tuple[int, int]) -> bytes: + """A 1x1 PNG whose header claims ``size``.""" + data = bytearray(image((1, 1))) + data[16:24] = struct.pack(">II", *size) + data[29:33] = struct.pack(">I", zlib.crc32(data[12:29])) + return bytes(data) + + +def rotated_jpeg() -> bytes: + exif = Image.Exif() + exif[274] = 6 # rotate 90 degrees clockwise for display + return image((40, 20), format="JPEG", color="red", exif=exif) + + +def picture(mime: str, size: str) -> list[dict[str, Any]]: + data = f"<{mime} {size}>" + return [{"type": "image", "source": {"type": "base64", "media_type": mime, "data": data}}] + + +def text(value: str | IsStr) -> list[dict[str, Any]]: + return [{"type": "text", "text": value}] + + +CLAUDE_VIEWS = { + "small-png": (image((40, 30)), (False, picture("image/png", "40x30"))), + "wide-png": (image((3000, 2000)), (False, picture("image/png", "1312x875"))), + "tall-png": (image((2000, 4000)), (False, picture("image/png", "758x1516"))), + "exif-jpeg": (rotated_jpeg(), (False, picture("image/png", "20x40"))), + "gif": (image((32, 24), format="GIF"), (False, picture("image/png", "32x24"))), + "webp": (image((32, 24), format="WEBP"), (False, picture("image/png", "32x24"))), + "transparent-png": ( + image((32, 24), mode="RGBA", color=(255, 0, 0, 128)), + (False, picture("image/png", "32x24")), + ), + "noisy-rgba": (noisy_rgba((1072, 1072)), (False, picture("image/jpeg", "1072x1072"))), + "oversized-header": ( + png_claiming((4001, 4000)), + ( + True, + text( + "Cannot view image: Image exceeds the 16,000,000-pixel source limit. " + "Resize or crop it before viewing." + ), + ), + ), + "corrupt-png": ( + b"\x89PNG\r\n\x1a\n" + b"garbage", + (True, text(IsStr(regex=r"Cannot view image: cannot identify image file .*"))), + ), + "binary": ( + b"\x00binary", + (True, text("Error: Cannot view this file: expected UTF-8 text or an image.")), + ), + "invalid-utf8": ( + b"\xff\xfeinvalid", + (True, text("Error: Cannot view this file: expected UTF-8 text or an image.")), + ), + "unicode-text": ("Hello, 世界\n".encode(), (False, text("Hello, 世界\n"))), +} + + +@pytest.mark.filterwarnings("ignore:Image size.*:PIL.Image.DecompressionBombWarning") +@pytest.mark.parametrize(("data", "expected"), CLAUDE_VIEWS.values(), ids=CLAUDE_VIEWS.keys()) +async def test_claude_sees_a_viewed_file_within_its_image_limits( + data: bytes, expected: tuple[bool, Any], models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + root = tmp_path / "ws" + models.script( + [call("str_replace_based_edit_tool", command="view", path=f"{root}/file"), say("done")] + ) + + run = await run_task( + workspace_env(root, files={"file": data}), + ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")), + ) + + assert run.trace.status == "completed" + (_prompt, _call, (result,)) = (m["content"] for m in models.requests()[-1].body["messages"]) + assert (result["is_error"], wire(result["content"])) == expected + + +CHAT_VIEWS = { + "png": (image((40, 30)), ""), + "jpeg": (image((40, 30), format="JPEG"), ""), + "gif": (image((40, 30), format="GIF"), ""), + "webp": (image((40, 30), format="WEBP"), ""), + "large-png": (image((3000, 2000)), ""), +} + + +@pytest.mark.parametrize(("data", "expected"), CHAT_VIEWS.values(), ids=CHAT_VIEWS.keys()) +async def test_chat_models_see_a_read_image_as_it_is_on_disk( + data: bytes, expected: str, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + root = tmp_path / "ws" + models.script([call("read", filePath=f"{root}/file"), say("done")]) + + await run_task( + workspace_env(root, files={"file": data}), + OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus")), + ) + + *_, tool, follow_up = models.requests()[-1].body["messages"] + assert tool["content"] == "Tool executed successfully" + assert wire(follow_up["content"]) == [ + {"type": "text", "text": "Tool returned the following:"}, + {"type": "image_url", "image_url": {"url": expected}}, + ] From 656c9c5d243388fa39fb37b37367520a6e6764fc Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 035/248] fix(agents): make ClaudeToolSearchTool.threshold defer custom tools The threshold field was accepted but never read: Claude always received every tool eagerly, so configuring tool search changed nothing but adding the search tool. Anthropic's tool search finds tools marked defer_loading, the same mechanism OpenAIToolSearchTool already drives, so the threshold now carries the same meaning: once a request holds more custom (schema) tools than the threshold, they are sent with defer_loading. Anthropic- defined tools (bash, editor, computer) and the search tool itself stay loaded, and a search tool the model does not support defers nothing. --- hud/agents/claude/agent.py | 24 ++++++++++++++++- hud/agents/claude/tools/hosted.py | 6 ++++- tests/agents/test_provider_requests.py | 36 ++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/hud/agents/claude/agent.py b/hud/agents/claude/agent.py index ff0b64e5f..6080e1c0f 100644 --- a/hud/agents/claude/agent.py +++ b/hud/agents/claude/agent.py @@ -44,6 +44,7 @@ from .images import image_source from .tools.coding import ClaudeBashTool, ClaudeTextEditorTool from .tools.computer import ClaudeComputerTool +from .tools.hosted import ClaudeToolSearchTool from .tools.mcp_proxy import ClaudeMCPProxyTool if TYPE_CHECKING: @@ -296,7 +297,7 @@ async def get_response( } betas: list[str] | Omit = list(required_betas) if required_betas else Omit() tool_choice = BetaToolChoiceAutoParam(type="auto", disable_parallel_tool_use=True) - tools = cast("list[BetaToolUnionParam]", list(state.params)) + tools = self._defer_for_tool_search(cast("list[BetaToolUnionParam]", list(state.params))) system = system_prompt if system_prompt is not None else Omit() is_bedrock = isinstance(self.anthropic_client, AsyncAnthropicBedrock) @@ -371,6 +372,27 @@ async def get_response( return self.message_to_agent_step(response, citations_enabled=citations_enabled) + def _defer_for_tool_search(self, tools: list[BetaToolUnionParam]) -> list[BetaToolUnionParam]: + """Defer custom tools to tool search once there are more than its threshold.""" + search = next( + ( + hosted + for hosted in self.config.hosted_tools + if isinstance(hosted, ClaudeToolSearchTool) + and hosted.supports_model(self.config.model) + ), + None, + ) + custom = [tool for tool in tools if "type" not in tool or tool["type"] == "custom"] + if search is None or len(custom) <= search.threshold: + return tools + return [ + cast("BetaToolUnionParam", {**tool, "defer_loading": True}) + if "type" not in tool or tool["type"] == "custom" + else tool + for tool in tools + ] + @classmethod def message_to_agent_step( cls, diff --git a/hud/agents/claude/tools/hosted.py b/hud/agents/claude/tools/hosted.py index d7076acb3..4a6e34413 100644 --- a/hud/agents/claude/tools/hosted.py +++ b/hud/agents/claude/tools/hosted.py @@ -81,7 +81,11 @@ def to_params(self) -> BetaWebFetchTool20250910Param: @dataclass(frozen=True, kw_only=True) class ClaudeToolSearchTool(ClaudeHostedTool): - """Claude tool search for large tool sets.""" + """Claude tool search for large tool sets. + + When the request carries more custom tools than ``threshold``, the agent + marks them ``defer_loading`` so the model finds them through search. + """ threshold: int = 10 diff --git a/tests/agents/test_provider_requests.py b/tests/agents/test_provider_requests.py index 39acd7fd1..d449c9915 100644 --- a/tests/agents/test_provider_requests.py +++ b/tests/agents/test_provider_requests.py @@ -693,6 +693,42 @@ class HostedRow: ] ), ), + "claude-tool-search-over-threshold": HostedRow( + agent=lambda: ClaudeAgent( + ClaudeConfig( + model="claude-sonnet-4-6", hosted_tools=[ClaudeToolSearchTool(threshold=1)] + ) + ), + tools=snapshot( + [ + {"type": "bash_20250124", "name": "bash"}, + {"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, + { + "name": "lookup", + "input_schema": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "eager_input_streaming": True, + "defer_loading": True, + }, + { + "name": "store", + "input_schema": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}, "value": {"type": "string"}}, + "required": ["key", "value"], + "type": "object", + }, + "eager_input_streaming": True, + "defer_loading": True, + }, + {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}, + ] + ), + ), "claude-tool-search-under-threshold": HostedRow( agent=lambda: ClaudeAgent( ClaudeConfig( From b91c81d38f7cb5171b975c571c03293a5915d471 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 036/248] refactor(agents): delete the unused OpenAI apply_patch engine hud/agents/openai/tools/apply_patch.py had no importer anywhere in hud, docs or cookbooks; OpenAIAgent offers shell, computer and MCP tools only. Its tests exercised private helpers of dead code. --- hud/agents/openai/tools/apply_patch.py | 327 ------------------------- tests/agents/test_apply_patch.py | 77 ------ 2 files changed, 404 deletions(-) delete mode 100644 hud/agents/openai/tools/apply_patch.py delete mode 100644 tests/agents/test_apply_patch.py diff --git a/hud/agents/openai/tools/apply_patch.py b/hud/agents/openai/tools/apply_patch.py deleted file mode 100644 index 84a7324a4..000000000 --- a/hud/agents/openai/tools/apply_patch.py +++ /dev/null @@ -1,327 +0,0 @@ -"""OpenAI apply_patch parser helpers.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Literal - -if TYPE_CHECKING: - from collections.abc import Callable - - -class DiffError(ValueError): - """Exception raised when diff parsing or application fails.""" - - -ActionType = Literal["add", "delete", "update"] - - -@dataclass -class Chunk: - orig_index: int = -1 # line index of the first line in the original file - del_lines: list[str] = field(default_factory=list[str]) - ins_lines: list[str] = field(default_factory=list[str]) - - -@dataclass -class PatchAction: - type: ActionType - new_file: str | None = None - chunks: list[Chunk] = field(default_factory=list[Chunk]) - move_path: str | None = None - - -class Parser: - """Parser for V4A diff format.""" - - def __init__(self, current_files: dict[str, str], lines: list[str], index: int = 0) -> None: - self.current_files = current_files - self.lines = lines - self.index = index - self.actions: dict[str, PatchAction] = {} - self.fuzz = 0 - - def is_done(self, prefixes: tuple[str, ...] | None = None) -> bool: - if self.index >= len(self.lines): - return True - return prefixes is not None and self.lines[self.index].startswith(prefixes) - - def read_str(self, prefix: str = "") -> str: - if self.index >= len(self.lines): - return "" # At EOF, no match possible - if self.lines[self.index].startswith(prefix): - text = self.lines[self.index][len(prefix) :] - self.index += 1 - return text - return "" - - def parse(self) -> None: - while not self.is_done(("*** End Patch",)): - path = self.read_str("*** Update File: ") - if path: - if path in self.actions: - raise DiffError(f"Update File Error: Duplicate Path: {path}") - move_to = self.read_str("*** Move to: ") - if path not in self.current_files: - raise DiffError(f"Update File Error: Missing File: {path}") - text = self.current_files[path] - action = self.parse_update_file(text) - action.move_path = move_to if move_to else None - self.actions[path] = action - continue - - path = self.read_str("*** Delete File: ") - if path: - if path in self.actions: - raise DiffError(f"Delete File Error: Duplicate Path: {path}") - if path not in self.current_files: - raise DiffError(f"Delete File Error: Missing File: {path}") - self.actions[path] = PatchAction(type="delete") - continue - - path = self.read_str("*** Add File: ") - if path: - if path in self.actions: - raise DiffError(f"Add File Error: Duplicate Path: {path}") - self.actions[path] = self.parse_add_file() - continue - - raise DiffError(f"Unknown Line: {self.lines[self.index]}") - - if self.index >= len(self.lines) or not self.lines[self.index].startswith("*** End Patch"): - raise DiffError("Missing End Patch") - self.index += 1 - - def parse_update_file(self, text: str) -> PatchAction: - action = PatchAction(type="update") - lines = text.split("\n") - index = 0 - - while not self.is_done( - ( - "*** End Patch", - "*** Update File:", - "*** Delete File:", - "*** Add File:", - "*** End of File", - ) - ): - section_anchor = self.read_str("@@ ") - has_section_marker = False - if not section_anchor and self.lines[self.index] == "@@": - has_section_marker = True - self.index += 1 - - if not (section_anchor or has_section_marker or index == 0): - raise DiffError(f"Invalid Line:\n{self.lines[self.index]}") - - if section_anchor.strip(): - found = False - if not any(line == section_anchor for line in lines[:index]): - for i, line in enumerate(lines[index:], index): - if line == section_anchor: - index = i + 1 - found = True - break - - stripped_anchor = section_anchor.strip() - if not found and not any(line.strip() == stripped_anchor for line in lines[:index]): - for i, line in enumerate(lines[index:], index): - if line.strip() == stripped_anchor: - index = i + 1 - self.fuzz += 1 - found = True - break - - next_chunk_context, chunks, end_patch_index, eof = self._peek_next_section() - next_chunk_text = "\n".join(next_chunk_context) - new_index, fuzz = _find_context(lines, next_chunk_context, index, eof) - - if new_index == -1: - if eof: - raise DiffError(f"Invalid EOF Context {index}:\n{next_chunk_text}") - else: - raise DiffError(f"Invalid Context {index}:\n{next_chunk_text}") - - self.fuzz += fuzz - - for chunk in chunks: - chunk.orig_index += new_index - action.chunks.append(chunk) - - index = new_index + len(next_chunk_context) - self.index = end_patch_index - - return action - - def parse_add_file(self) -> PatchAction: - lines: list[str] = [] - while not self.is_done( - ("*** End Patch", "*** Update File:", "*** Delete File:", "*** Add File:") - ): - line = self.read_str() - if not line.startswith("+"): - raise DiffError(f"Invalid Add File Line: {line}") - lines.append(line[1:]) - return PatchAction(type="add", new_file="\n".join(lines)) - - def _peek_next_section(self) -> tuple[list[str], list[Chunk], int, bool]: - old: list[str] = [] - del_lines: list[str] = [] - ins_lines: list[str] = [] - chunks: list[Chunk] = [] - mode = "keep" - orig_index = self.index - index = self.index - - def flush_chunk() -> None: - nonlocal del_lines, ins_lines - if not (ins_lines or del_lines): - return - chunks.append( - Chunk( - orig_index=len(old) - len(del_lines), - del_lines=del_lines, - ins_lines=ins_lines, - ) - ) - del_lines = [] - ins_lines = [] - - while index < len(self.lines): - line = self.lines[index] - if line.startswith( - ( - "@@", - "*** End Patch", - "*** Update File:", - "*** Delete File:", - "*** Add File:", - "*** End of File", - ) - ): - break - if line == "***": - break - elif line.startswith("***"): - raise DiffError(f"Invalid Line: {line}") - - index += 1 - last_mode = mode - - if line == "": - line = " " - - if line[0] == "+": - mode = "add" - elif line[0] == "-": - mode = "delete" - elif line[0] == " ": - mode = "keep" - else: - raise DiffError(f"Invalid Line: {line}") - - line = line[1:] - - if mode == "keep" and last_mode != mode: - flush_chunk() - - if mode == "delete": - del_lines.append(line) - old.append(line) - elif mode == "add": - ins_lines.append(line) - elif mode == "keep": - old.append(line) - - flush_chunk() - - if index < len(self.lines) and self.lines[index] == "*** End of File": - index += 1 - return old, chunks, index, True - - if index == orig_index: - raise DiffError(f"Nothing in this section - {index=} {self.lines[index]}") - - return old, chunks, index, False - - -def _find_context(lines: list[str], context: list[str], start: int, eof: bool) -> tuple[int, int]: - if not context: - return start, 0 - - search_starts = [len(lines) - len(context), start] if eof else [start] - rstripped_context = [line.rstrip() for line in context] - stripped_context = [line.strip() for line in context] - - for attempt, search_start in enumerate(search_starts): - fuzz_offset = 10000 if eof and attempt > 0 else 0 - - for i in range(search_start, len(lines)): - candidate = lines[i : i + len(context)] - if candidate == context: - return i, fuzz_offset - - for i in range(search_start, len(lines)): - candidate = lines[i : i + len(context)] - if [line.rstrip() for line in candidate] == rstripped_context: - return i, fuzz_offset + 1 - - for i in range(search_start, len(lines)): - candidate = lines[i : i + len(context)] - if [line.strip() for line in candidate] == stripped_context: - return i, fuzz_offset + 100 - - return -1, 0 - - -def _text_to_patch(text: str, orig: dict[str, str]) -> tuple[dict[str, PatchAction], int]: - lines = text.strip().split("\n") - if len(lines) < 2 or not lines[0].startswith("*** Begin Patch") or lines[-1] != "*** End Patch": - raise DiffError("Invalid patch text") - - parser = Parser(current_files=orig, lines=lines, index=1) - parser.parse() - return parser.actions, parser.fuzz - - -def _apply_patch( - patch: dict[str, PatchAction], - orig: dict[str, str], - write_fn: Callable[[str, str | None], None], - remove_fn: Callable[[str], None], -) -> None: - for path, action in patch.items(): - match action.type: - case "delete": - remove_fn(path) - case "add": - write_fn(path, action.new_file) - case "update": - orig_lines = orig[path].split("\n") - dest_lines: list[str] = [] - orig_index = 0 - - for chunk in action.chunks: - if chunk.orig_index > len(orig_lines): - raise DiffError( - f"_apply_patch: {path}: chunk.orig_index {chunk.orig_index} " - f"> len(lines) {len(orig_lines)}" - ) - if orig_index > chunk.orig_index: - raise DiffError( - f"_apply_patch: {path}: orig_index {orig_index} " - f"> chunk.orig_index {chunk.orig_index}" - ) - - dest_lines.extend(orig_lines[orig_index : chunk.orig_index]) - dest_lines.extend(chunk.ins_lines) - orig_index = chunk.orig_index + len(chunk.del_lines) - - dest_lines.extend(orig_lines[orig_index:]) - new_content = "\n".join(dest_lines) - if action.move_path: - write_fn(action.move_path, new_content) - remove_fn(path) - else: - write_fn(path, new_content) diff --git a/tests/agents/test_apply_patch.py b/tests/agents/test_apply_patch.py deleted file mode 100644 index c050f9a36..000000000 --- a/tests/agents/test_apply_patch.py +++ /dev/null @@ -1,77 +0,0 @@ -"""The OpenAI V4A apply-patch engine: parse a patch + apply it via callbacks. - -``_text_to_patch`` parses the V4A diff text against the current files; ``_apply_patch`` -applies the parsed actions through ``write``/``remove`` callbacks. Both are pure, so -the file store is just a dict captured by the callbacks. -""" - -from __future__ import annotations - -from typing import Any - -import pytest - -from hud.agents.openai.tools.apply_patch import DiffError, _apply_patch, _text_to_patch - - -def _apply(patch_text: str, orig: dict[str, str]) -> tuple[dict[str, str | None], list[str]]: - actions, _fuzz = _text_to_patch(patch_text, orig) - writes: dict[str, str | None] = {} - removed: list[str] = [] - _apply_patch(actions, orig, lambda p, c: writes.__setitem__(p, c), removed.append) - return writes, removed - - -def test_add_file_writes_new_content() -> None: - patch = "*** Begin Patch\n*** Add File: hello.txt\n+hello\n+world\n*** End Patch" - writes, removed = _apply(patch, {}) - assert writes == {"hello.txt": "hello\nworld"} - assert removed == [] - - -def test_update_file_replaces_a_line_in_context() -> None: - orig = {"f.txt": "line1\nline2\nline3"} - patch = ( - "*** Begin Patch\n*** Update File: f.txt\n@@\n line1\n-line2\n+LINE2\n line3\n*** End Patch" - ) - writes, _ = _apply(patch, orig) - assert writes["f.txt"] == "line1\nLINE2\nline3" - - -def test_delete_file_calls_remove() -> None: - writes, removed = _apply( - "*** Begin Patch\n*** Delete File: f.txt\n*** End Patch", {"f.txt": "gone"} - ) - assert removed == ["f.txt"] - assert writes == {} - - -def test_update_with_move_renames_file() -> None: - orig = {"a.txt": "hi"} - patch = ( - "*** Begin Patch\n*** Update File: a.txt\n*** Move to: b.txt\n@@\n-hi\n+bye\n*** End Patch" - ) - writes, removed = _apply(patch, orig) - assert writes == {"b.txt": "bye"} - assert removed == ["a.txt"] - - -def test_invalid_patch_without_sentinels_raises() -> None: - with pytest.raises(DiffError): - _text_to_patch("just some text", {}) - - -def test_update_missing_file_raises() -> None: - patch = "*** Begin Patch\n*** Update File: ghost.txt\n@@\n-x\n+y\n*** End Patch" - with pytest.raises(DiffError, match="Missing File"): - _text_to_patch(patch, {}) - - -def test_duplicate_update_path_raises() -> None: - orig: dict[str, Any] = {"f.txt": "a"} - patch = ( - "*** Begin Patch\n*** Update File: f.txt\n@@\n-a\n+b\n" - "*** Update File: f.txt\n@@\n-b\n+c\n*** End Patch" - ) - with pytest.raises(DiffError, match="Duplicate Path"): - _text_to_patch(patch, orig) From 48dae2438990a8b4676c5aeab97beb93796b1c32 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 037/248] test(agents): pin the RFB input each computer-use action sends The computer tools were tested through subclasses that replaced every RFBTool primitive, so the pointer, keyboard and raw PointerEvent writes never ran. Claude, OpenAI and Gemini now drive a fake VNC screen through a served environment; each row pins the key and pointer events the screen received and the result the model saw next: modifiers held around clicks and scrolls, button masks, newline and tab typed as Return and Tab, drag interpolation, split mouse down/up, zoom crops, batch errors, acknowledged safety checks, the computer tool advertised per Claude model, and Gemini's screenshot window over the latest three computer turns. --- tests/agents/support.py | 30 ++ tests/agents/test_computer_use.py | 680 ++++++++++++++++++++++++++++++ 2 files changed, 710 insertions(+) create mode 100644 tests/agents/test_computer_use.py diff --git a/tests/agents/support.py b/tests/agents/support.py index f5d37ed8d..da1e0c49e 100644 --- a/tests/agents/support.py +++ b/tests/agents/support.py @@ -32,6 +32,7 @@ from hud.agents.base import Agent from hud.eval import Run + from tests.harness import ModelRequest PROMPT = "Do the task." DATA_URL = re.compile(r"^data:(?P[\w/+.-]+);base64,(?P.*)$", re.DOTALL) @@ -166,5 +167,34 @@ def wire(value: Any, *paths: Path, descriptions: bool = False) -> Any: return value +def tool_results(requests: list[ModelRequest]) -> list[Any]: + """The tool result entries a run handed back to its model, oldest first. + + Responses requests carry only the items new since the previous response, + so their inputs are gathered from every request after the first; the other + protocols resend the whole conversation, so the last request holds them all. + ``cache_control`` markers, which move with the conversation, are dropped. + """ + last = requests[-1] + if last.protocol == "responses": + return [item for request in requests[1:] for item in request.body["input"]] + if last.protocol == "gemini": + return [ + part + for content in last.body["contents"] + if content["role"] == "user" + for part in content["parts"] + if "functionResponse" in part + ] + if last.protocol == "anthropic": + return [ + {key: value for key, value in block.items() if key != "cache_control"} + for message in last.body["messages"][1:] + if message["role"] == "user" + for block in message["content"] + ] + return [message for message in last.body["messages"] if message["role"] == "tool"] + + def json_lines(text: str) -> list[dict[str, Any]]: return [json.loads(line) for line in text.splitlines() if line.strip()] diff --git a/tests/agents/test_computer_use.py b/tests/agents/test_computer_use.py new file mode 100644 index 000000000..7c751fd23 --- /dev/null +++ b/tests/agents/test_computer_use.py @@ -0,0 +1,680 @@ +# ruff: noqa: E501 -- snapshots quote product messages verbatim +"""Computer-use actions reach the environment's screen as RFB input events. + +The environment publishes an RFB capability backed by a fake VNC screen that +records every key and pointer event it receives. Each row scripts one provider +to take an action and pins the events the screen received and the result the +model saw next. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import pytest +from asyncvnc import key_codes +from inline_snapshot import snapshot + +from hud.agents import ClaudeAgent, GeminiAgent, OpenAIAgent +from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIConfig +from hud.capabilities import Capability +from tests.agents.support import run_task, tool_results, wire, workspace_env +from tests.harness import ( + KeyEvent, + Turn, + call, + computer_call, + fake_screen, + say, +) + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hud.agents.base import Agent + from tests.harness import FakeScreen, HudEnv, Models + +KEY_NAMES: dict[int, str] = {} +for _name, _code in key_codes.items(): + KEY_NAMES.setdefault(_code, _name) + + +def events(screen: FakeScreen) -> str: + """The screen's input log in one line. + + ``+Key``/``-Key`` are presses and releases, a bare ``Key`` a press released + straight away, and ``x,y bN`` a pointer event with button mask ``N``. + """ + rendered: list[str] = [] + for event in screen.events: + if isinstance(event, KeyEvent): + name = KEY_NAMES[event.key] + if not event.down and rendered and rendered[-1] == f"+{name}": + rendered[-1] = name + else: + rendered.append(f"{'+' if event.down else '-'}{name}") + else: + rendered.append(f"{event.x},{event.y} b{event.buttons}") + return " ".join(rendered) + + +def seen(entry: dict[str, Any]) -> str: + """One tool result entry as the model sees it, in a line: images as ````.""" + entry = wire(entry) + if entry.get("type") == "tool_result": + parts = [block.get("text") or block["source"]["data"] for block in entry["content"]] + return ("error: " if entry["is_error"] else "") + " ".join(parts) + if entry.get("type") == "computer_call_output": + acknowledged = [check["id"] for check in entry.get("acknowledged_safety_checks", [])] + return " ".join([entry["output"]["image_url"], *(f"ack {id}" for id in acknowledged)]) + if "functionResponse" in entry: + response = entry["functionResponse"] + images = [part["inline_data"]["data"] for part in response.get("parts", [])] + return " ".join([response["name"], json.dumps(response["response"]), *images]) + return "user: " + " ".join(block["text"] for block in entry["content"]) + + +@dataclass(frozen=True) +class Row: + agent: Callable[[], Agent] + script: list[Turn] + observed: Any + + +def claude() -> Agent: + return ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")) + + +def claude_does(*actions: dict[str, Any], observed: Any) -> Row: + turns = [call("computer", **action) for action in actions] + return Row(agent=claude, script=[*turns, say("done")], observed=observed) + + +CLAUDE = { + "left-click-holding-ctrl": claude_does( + {"action": "left_click", "coordinate": [10, 20], "text": "ctrl"}, + observed=snapshot( + { + "events": "10,20 b0 +Control_L 10,20 b1 10,20 b0 -Control_L", + "results": [""], + } + ), + ), + "right-click": claude_does( + {"action": "right_click", "coordinate": [3, 4]}, + observed=snapshot({"events": "3,4 b0 3,4 b4 3,4 b0", "results": [""]}), + ), + "middle-click": claude_does( + {"action": "middle_click", "coordinate": [3, 4]}, + observed=snapshot({"events": "3,4 b0 3,4 b2 3,4 b0", "results": [""]}), + ), + "double-click": claude_does( + {"action": "double_click", "coordinate": [3, 4]}, + observed=snapshot( + {"events": "3,4 b0 3,4 b1 3,4 b0 3,4 b1 3,4 b0", "results": [""]} + ), + ), + "triple-click": claude_does( + {"action": "triple_click", "coordinate": [3, 4]}, + observed=snapshot( + { + "events": "3,4 b0 3,4 b1 3,4 b0 3,4 b1 3,4 b0 3,4 b1 3,4 b0", + "results": [""], + } + ), + ), + "mouse-move": claude_does( + {"action": "mouse_move", "coordinate": [7, 8]}, + observed=snapshot({"events": "7,8 b0", "results": [""]}), + ), + "mouse-down-then-up": claude_does( + {"action": "mouse_move", "coordinate": [5, 6]}, + {"action": "left_mouse_down"}, + {"action": "mouse_move", "coordinate": [30, 20]}, + {"action": "left_mouse_up"}, + observed=snapshot( + { + "events": "5,6 b0 5,6 b1 30,20 b1 30,20 b0", + "results": [ + "", + "", + "", + "", + ], + } + ), + ), + "type-with-newline-and-tab": claude_does( + {"action": "type", "text": "a\nb\tC"}, + observed=snapshot({"events": "a Return b Tab C", "results": [""]}), + ), + "key-sequence-repeated": claude_does( + {"action": "key", "text": "ctrl+a Delete", "repeat": 2}, + observed=snapshot( + { + "events": "+Control_L a -Control_L Delete +Control_L a -Control_L Delete", + "results": [""], + } + ), + ), + "hold-key": claude_does( + {"action": "hold_key", "text": "shift", "duration": 0.01}, + observed=snapshot({"events": "Shift_L", "results": [""]}), + ), + "scroll-down": claude_does( + { + "action": "scroll", + "coordinate": [5, 5], + "scroll_direction": "down", + "scroll_amount": 2, + }, + observed=snapshot( + {"events": "5,5 b0 5,5 b16 5,5 b0 5,5 b16 5,5 b0", "results": [""]} + ), + ), + "scroll-up-holding-shift": claude_does( + { + "action": "scroll", + "coordinate": [5, 5], + "scroll_direction": "up", + "scroll_amount": 1, + "text": "shift", + }, + observed=snapshot( + { + "events": "5,5 b0 +Shift_L 5,5 b8 5,5 b0 -Shift_L", + "results": [""], + } + ), + ), + "scroll-left": claude_does( + { + "action": "scroll", + "coordinate": [5, 5], + "scroll_direction": "left", + "scroll_amount": 3, + }, + observed=snapshot({"events": "5,5 b0", "results": [""]}), + ), + "drag": claude_does( + {"action": "left_click_drag", "start_coordinate": [2, 2], "coordinate": [40, 2]}, + observed=snapshot( + {"events": "2,2 b0 2,2 b1 21,2 b1 40,2 b1 40,2 b0", "results": [""]} + ), + ), + "wait": claude_does( + {"action": "wait", "duration": 0.01}, + observed=snapshot({"events": "", "results": [""]}), + ), + "cursor-position": claude_does( + {"action": "mouse_move", "coordinate": [9, 9]}, + {"action": "cursor_position"}, + observed=snapshot({"events": "9,9 b0", "results": ["", "(9, 9)"]}), + ), + "zoom": claude_does( + {"action": "zoom", "region": [0, 0, 32, 24]}, + observed=snapshot({"events": "", "results": [""]}), + ), + "screenshot": claude_does( + {"action": "screenshot"}, + observed=snapshot({"events": "", "results": [""]}), + ), + "key-without-text": claude_does( + {"action": "key"}, + observed=snapshot( + {"events": "", "results": ["error: Error: `text` (key chord) is required for key"]} + ), + ), + "move-without-coordinate": claude_does( + {"action": "mouse_move"}, + observed=snapshot( + { + "events": "", + "results": [ + "error: Error: computer action 'mouse_move' failed: coordinate.x is required" + ], + } + ), + ), + "unsupported-action": claude_does( + {"action": "frobnicate"}, + observed=snapshot( + {"events": "", "results": ["error: Error: unsupported computer action: 'frobnicate'"]} + ), + ), +} + + +def openai(**config: Any) -> Callable[[], Agent]: + return lambda: OpenAIAgent(OpenAIConfig(model="gpt-5.6", **config)) + + +def openai_does(*actions: dict[str, Any], observed: Any, **call: Any) -> Row: + return Row( + agent=openai(), script=[computer_call(*actions, **call), say("done")], observed=observed + ) + + +OPENAI = { + "click-holding-ctrl": openai_does( + {"type": "click", "x": 3, "y": 4, "button": "left", "keys": ["ctrl"]}, + observed=snapshot( + { + "events": "3,4 b0 +Control_L 3,4 b1 3,4 b0 -Control_L", + "results": [""], + } + ), + ), + "right-and-wheel-click": openai_does( + {"type": "click", "x": 3, "y": 4, "button": "right"}, + {"type": "click", "x": 3, "y": 4, "button": "wheel"}, + observed=snapshot( + { + "events": "3,4 b0 3,4 b4 3,4 b0 3,4 b0 3,4 b2 3,4 b0", + "results": [""], + } + ), + ), + "double-click": openai_does( + {"type": "double_click", "x": 3, "y": 4}, + observed=snapshot( + {"events": "3,4 b0 3,4 b1 3,4 b0 3,4 b1 3,4 b0", "results": [""]} + ), + ), + "scroll": openai_does( + {"type": "scroll", "x": 5, "y": 5, "scroll_x": 4, "scroll_y": -2}, + observed=snapshot( + {"events": "5,5 b0 5,5 b8 5,5 b0 5,5 b8 5,5 b0", "results": [""]} + ), + ), + "type-and-keypress": openai_does( + {"type": "type", "text": "hi\n"}, + {"type": "keypress", "keys": ["CTRL", "c"]}, + observed=snapshot( + {"events": "h i Return +Control_L c -Control_L", "results": [""]} + ), + ), + "move-drag-wait-screenshot": openai_does( + {"type": "move", "x": 1, "y": 1}, + {"type": "drag", "path": [{"x": 2, "y": 2}, {"x": 20, "y": 2}]}, + {"type": "wait", "ms": 10}, + {"type": "screenshot"}, + observed=snapshot( + {"events": "1,1 b0 2,2 b0 2,2 b1 20,2 b1 20,2 b0", "results": [""]} + ), + ), + "response": openai_does( + {"type": "response", "text": "all done"}, + observed=snapshot({"events": "", "results": [""]}), + ), + "invalid-action-stops-the-batch": openai_does( + {"type": "move", "x": 3, "y": 4}, + {"type": "keypress", "keys": "ESC"}, + {"type": "type", "text": "later"}, + observed=snapshot( + { + "events": "3,4 b0", + "results": [ + "", + """\ +user: Computer call call_computer stopped with an error. Remaining actions in this call were not executed. +computer action 'keypress' failed: keys must be a list of non-empty strings\ +""", + ], + } + ), + ), + "unknown-action-type": openai_does( + {"type": "frobnicate"}, + observed=snapshot( + { + "events": "", + "results": [ + "", + """\ +user: Computer call call_computer stopped with an error. Remaining actions in this call were not executed. +computer action 'frobnicate' failed: Invalid action type: frobnicate\ +""", + ], + } + ), + ), + "empty-actions": openai_does( + observed=snapshot( + { + "events": "", + "results": [ + "", + """\ +user: Computer call call_computer stopped with an error. Remaining actions in this call were not executed. +actions list is empty\ +""", + ], + } + ) + ), + "safety-checks-acknowledged": openai_does( + {"type": "screenshot"}, + pending_safety_checks=[{"id": "sc_1", "code": "malicious_instructions", "message": "x"}], + observed=snapshot({"events": "", "results": [" ack sc_1"]}), + ), + "webp-screenshots": Row( + agent=openai(screenshot_encoding={"mime_type": "image/webp", "quality": 42}), + script=[computer_call({"type": "screenshot"}), say("done")], + observed=snapshot({"events": "", "results": [""]}), + ), +} + + +def gemini(**config: Any) -> Callable[[], Agent]: + return lambda: GeminiAgent(GeminiConfig(model="gemini-3.1-pro-preview", **config)) + + +def gemini_does(*turns: Turn, observed: Any) -> Row: + return Row(agent=gemini(), script=[*turns, say("done")], observed=observed) + + +GEMINI = { + "open-web-browser": gemini_does( + call("open_web_browser"), + observed=snapshot( + {"events": "", "results": ['open_web_browser {"success": true} ']} + ), + ), + "scroll-document-down": gemini_does( + call("scroll_document", direction="down"), + observed=snapshot( + { + "events": "0,0 b16 0,0 b0 0,0 b16 0,0 b0 0,0 b16 0,0 b0", + "results": ['scroll_document {"success": true} '], + } + ), + ), + "go-back-and-forward": gemini_does( + call("go_back"), + call("go_forward"), + observed=snapshot( + { + "events": "+Alt_L Left -Alt_L +Alt_L Right -Alt_L", + "results": [ + 'go_back {"success": true} ', + 'go_forward {"success": true} ', + ], + } + ), + ), + "search": gemini_does( + call("search"), + observed=snapshot( + { + "events": "+Control_L l -Control_L h t t p s colon slash slash w w w period g o o g l e period c o m Return", + "results": ['search {"success": true} '], + } + ), + ), + "navigate": gemini_does( + call("navigate", url="https://a.b"), + observed=snapshot( + { + "events": "+Control_L l -Control_L h t t p s colon slash slash a period b Return", + "results": ['navigate {"success": true} '], + } + ), + ), + "key-combination": gemini_does( + call("key_combination", keys="Control+a"), + observed=snapshot( + { + "events": "+Control_L a -Control_L", + "results": ['key_combination {"success": true} '], + } + ), + ), + "key-combination-without-keys": gemini_does( + call("key_combination", keys=3), + observed=snapshot( + { + "events": "", + "results": ['key_combination {"error": "keys must be a \'+\'-separated string"}'], + } + ), + ), +} + +ROWS = { + **{f"claude-{name}": row for name, row in CLAUDE.items()}, + **{f"openai-{name}": row for name, row in OPENAI.items()}, + **{f"gemini-{name}": row for name, row in GEMINI.items()}, +} + + +@pytest.mark.parametrize("row", ROWS.values(), ids=ROWS.keys()) +async def test_a_computer_action_reaches_the_screen_as_rfb_input( + row: Row, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script(row.script) + + async with fake_screen(width=200, height=100) as screen: + screen_capability = Capability.rfb(url=screen.url) + run = await run_task( + workspace_env(tmp_path / "ws", capabilities=(screen_capability,)), row.agent() + ) + + assert run.trace.status == "completed" + observed = { + "events": events(screen), + "results": [seen(entry) for entry in tool_results(models.requests())], + } + assert observed == row.observed + + +CLAUDE_MODELS = { + "claude-sonnet-4-6": snapshot( + { + "beta": "computer-use-2025-11-24", + "tools": [ + {"type": "bash_20250124", "name": "bash"}, + {"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, + { + "type": "computer_20251124", + "name": "computer", + "display_width_px": 200, + "display_height_px": 100, + "display_number": 1, + "enable_zoom": True, + }, + ], + "described_display": None, + } + ), + "claude-sonnet-4-5": snapshot( + { + "beta": "computer-use-2025-01-24", + "tools": [ + {"type": "bash_20250124", "name": "bash"}, + {"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, + { + "type": "computer_20250124", + "name": "computer", + "display_width_px": 200, + "display_height_px": 100, + "display_number": 1, + }, + ], + "described_display": None, + } + ), + "claude-future-model": snapshot( + { + "beta": "computer-use-2025-11-24", + "tools": [ + {"type": "bash_20250124", "name": "bash"}, + {"type": "text_editor_20250728", "name": "str_replace_based_edit_tool"}, + { + "type": "computer_20251124", + "name": "computer", + "display_width_px": 200, + "display_height_px": 100, + "display_number": 1, + "enable_zoom": True, + }, + ], + "described_display": None, + } + ), + "qwen/qwen3.8-max": snapshot( + { + "beta": None, + "tools": [ + { + "name": "bash", + "input_schema": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + }, + }, + { + "name": "str_replace_based_edit_tool", + "input_schema": { + "type": "object", + "properties": { + "command": { + "type": "string", + "enum": ["view", "create", "str_replace", "insert"], + }, + "path": {"type": "string"}, + "file_text": {"type": "string"}, + "old_str": {"type": "string"}, + "new_str": {"type": "string"}, + "insert_line": {"type": "integer"}, + }, + "required": ["command", "path"], + }, + }, + { + "name": "computer", + "input_schema": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "screenshot", + "zoom", + "left_click", + "right_click", + "middle_click", + "double_click", + "triple_click", + "mouse_move", + "left_mouse_down", + "left_mouse_up", + "type", + "key", + "hold_key", + "scroll", + "left_click_drag", + "wait", + "cursor_position", + ], + }, + "coordinate": { + "type": "array", + "items": {"type": "integer"}, + "minItems": 2, + "maxItems": 2, + }, + "start_coordinate": { + "type": "array", + "items": {"type": "integer"}, + "minItems": 2, + "maxItems": 2, + }, + "text": {"type": "string"}, + "duration": {"type": "number"}, + "scroll_direction": { + "type": "string", + "enum": ["up", "down", "left", "right"], + }, + "scroll_amount": {"type": "integer"}, + "repeat": {"type": "integer"}, + "region": { + "type": "array", + "items": {"type": "integer"}, + "minItems": 4, + "maxItems": 4, + }, + }, + "required": ["action"], + }, + }, + ], + "described_display": "200x100", + } + ), +} + + +@pytest.mark.parametrize(("model", "expected"), CLAUDE_MODELS.items(), ids=CLAUDE_MODELS.keys()) +async def test_claude_advertises_the_computer_tool_its_model_supports( + model: str, expected: Any, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + """Claude models get the newest computer-use tool they support with its beta; other + models served over the Messages API get plain schemas describing the display.""" + hud_env.set(HUD_API_KEY="k") + models.script([say("done")]) + + async with fake_screen(width=200, height=100) as screen: + screen_capability = Capability.rfb(url=screen.url) + await run_task( + workspace_env(tmp_path / "ws", capabilities=(screen_capability,)), + ClaudeAgent(ClaudeConfig(model=model)), + ) + + (request,) = models.requests() + computer = next(tool for tool in request.body["tools"] if tool["name"] == "computer") + display = re.search(r"display is (\d+x\d+) pixels", computer.get("description", "")) + advertised = { + "beta": request.headers.get("anthropic-beta"), + "tools": wire(request.body["tools"]), + "described_display": display and display[1], + } + assert advertised == expected + + +async def test_only_the_first_screen_is_driven( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([call("computer", action="left_click", coordinate=[1, 1]), say("done")]) + + async with fake_screen() as first, fake_screen() as second: + screens = ( + Capability.rfb(name="screen-0", url=first.url), + Capability.rfb(name="screen-1", url=second.url), + ) + await run_task(workspace_env(tmp_path / "ws", capabilities=screens), claude()) + + assert (events(first), events(second), second.connections) == ("1,1 b0 1,1 b1 1,1 b0", "", 0) + + +async def test_gemini_keeps_screenshots_for_the_three_latest_computer_turns( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([*(call("open_web_browser") for _ in range(5)), say("done")]) + + async with fake_screen() as screen: + screen_capability = Capability.rfb(url=screen.url) + await run_task( + workspace_env(tmp_path / "ws", capabilities=(screen_capability,)), gemini()() + ) + + responses = [part["functionResponse"] for part in tool_results(models.requests())] + assert ["parts" in response for response in responses] == [False, False, True, True, True] From 50aee75a11b52c873554974cdd6ff2cda3952b73 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 038/248] fix(agents): scale Gemini computer-use coordinates to the screen Gemini's computer-use model places every coordinate on a normalized 1000x1000 grid whatever the screen size; Google's reference converts each axis as int(value / 1000 * dimension). GeminiComputerTool sent the grid values to RFB unchanged, so on any screen other than 1000x1000 clicks, hovers, typing targets, scrolls and drags landed in the wrong place (a click at the centre of a 200x100 screen went to 500,500). Every positional action now denormalizes x against the display width and y against the height. drag_and_drop keeps its 25px edge inset, now per axis in pixel space rather than against the larger display dimension. --- hud/agents/gemini/tools/computer.py | 48 +++++++++++++++------------ tests/agents/test_computer_use.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/hud/agents/gemini/tools/computer.py b/hud/agents/gemini/tools/computer.py index 9532debc8..aff600765 100644 --- a/hud/agents/gemini/tools/computer.py +++ b/hud/agents/gemini/tools/computer.py @@ -19,6 +19,8 @@ logger = logging.getLogger(__name__) GEMINI_DRAG_INSET = 25 +#: Gemini places every coordinate on a 1000x1000 grid regardless of the screen size. +GEMINI_COORDINATE_SCALE = 1000 IS_MAC = platform.system().lower() == "darwin" PREDEFINED_COMPUTER_USE_FUNCTIONS = ( @@ -80,20 +82,20 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: return await self.screenshot() if action == "click_at": - await self.click(args.get("x"), args.get("y")) + await self.click(*self._point(args)) return await self.screenshot() if action == "hover_at": - x, y = args.get("x"), args.get("y") + x, y = self._point(args) if x is not None and y is not None: - await self.move(int(x), int(y)) + await self.move(x, y) return await self.screenshot() if action == "type_text_at": - x, y = args.get("x"), args.get("y") + x, y = self._point(args) if x is not None and y is not None: - await self.move(int(x), int(y)) - await self.click(int(x), int(y)) + await self.move(x, y) + await self.click(x, y) if args.get("clear_before_typing", True): select_all = ["Super_L", "a"] if IS_MAC else ["Control_L", "a"] delete_key = "BackSpace" if IS_MAC else "Delete" @@ -118,14 +120,8 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: sx = magnitude elif direction == "left": sx = -magnitude - x = args.get("x") if action == "scroll_at" else None - y = args.get("y") if action == "scroll_at" else None - await self.scroll( - int(x) if x is not None else None, - int(y) if y is not None else None, - scroll_x=sx, - scroll_y=sy, - ) + x, y = self._point(args) if action == "scroll_at" else (None, None) + await self.scroll(x, y, scroll_x=sx, scroll_y=sy) return await self.screenshot() if action == "wait_5_seconds": @@ -180,21 +176,31 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: return await self.screenshot() if action == "drag_and_drop": - max_coord = max(self.display_width, self.display_height) - def clamp(v: Any) -> int: - if not isinstance(v, int | float): - return 0 - return min(max(int(v), GEMINI_DRAG_INSET), max_coord - GEMINI_DRAG_INSET) + def inset(value: Any, size: int) -> int: + pixel = self._pixel(value if isinstance(value, int | float) else 0, size) + return min(max(pixel, GEMINI_DRAG_INSET), size - GEMINI_DRAG_INSET) + width, height = self.display_width, self.display_height path = [ - (clamp(args.get("x")), clamp(args.get("y"))), - (clamp(args.get("destination_x")), clamp(args.get("destination_y"))), + (inset(args.get("x"), width), inset(args.get("y"), height)), + (inset(args.get("destination_x"), width), inset(args.get("destination_y"), height)), ] await self.drag(path) return await self.screenshot() return tool_err(f"Unknown Gemini computer action: {action}") + @staticmethod + def _pixel(value: float, size: int) -> int: + """Map a coordinate on Gemini's normalized 0-999 grid onto a ``size``-pixel axis.""" + return int(value / GEMINI_COORDINATE_SCALE * size) + + def _point(self, args: dict[str, Any]) -> tuple[int | None, int | None]: + x, y = args.get("x"), args.get("y") + if x is None or y is None: + return None, None + return self._pixel(x, self.display_width), self._pixel(y, self.display_height) + __all__ = ["GEMINI_COMPUTER_SPEC", "PREDEFINED_COMPUTER_USE_FUNCTIONS", "GeminiComputerTool"] diff --git a/tests/agents/test_computer_use.py b/tests/agents/test_computer_use.py index 7c751fd23..d271e2d87 100644 --- a/tests/agents/test_computer_use.py +++ b/tests/agents/test_computer_use.py @@ -385,6 +385,39 @@ def gemini_does(*turns: Turn, observed: Any) -> Row: {"events": "", "results": ['open_web_browser {"success": true} ']} ), ), + "click-at": gemini_does( + call("click_at", x=500, y=500), + observed=snapshot( + { + "events": "100,50 b0 100,50 b1 100,50 b0", + "results": ['click_at {"success": true} '], + } + ), + ), + "hover-at": gemini_does( + call("hover_at", x=250, y=750), + observed=snapshot( + {"events": "50,75 b0", "results": ['hover_at {"success": true} ']} + ), + ), + "type-text-at-clearing-first": gemini_does( + call("type_text_at", x=100, y=200, text="hi", press_enter=True), + observed=snapshot( + { + "events": "20,20 b0 20,20 b0 20,20 b1 20,20 b0 +Control_L a -Control_L Delete h i Return", + "results": ['type_text_at {"success": true} '], + } + ), + ), + "type-text-at-without-clearing": gemini_does( + call("type_text_at", x=100, y=200, text="hi", clear_before_typing=False), + observed=snapshot( + { + "events": "20,20 b0 20,20 b0 20,20 b1 20,20 b0 h i", + "results": ['type_text_at {"success": true} '], + } + ), + ), "scroll-document-down": gemini_does( call("scroll_document", direction="down"), observed=snapshot( @@ -394,6 +427,15 @@ def gemini_does(*turns: Turn, observed: Any) -> Row: } ), ), + "scroll-at-up": gemini_does( + call("scroll_at", x=500, y=500, direction="up", magnitude=2), + observed=snapshot( + { + "events": "100,50 b0 100,50 b8 100,50 b0 100,50 b8 100,50 b0", + "results": ['scroll_at {"success": true} '], + } + ), + ), "go-back-and-forward": gemini_does( call("go_back"), call("go_forward"), @@ -434,6 +476,15 @@ def gemini_does(*turns: Turn, observed: Any) -> Row: } ), ), + "drag-and-drop-near-the-edge": gemini_does( + call("drag_and_drop", x=0, y=500, destination_x=999, destination_y=500), + observed=snapshot( + { + "events": "25,50 b0 25,50 b1 41,50 b1 58,50 b1 75,50 b1 91,50 b1 108,50 b1 125,50 b1 141,50 b1 158,50 b1 175,50 b1 175,50 b0", + "results": ['drag_and_drop {"success": true} '], + } + ), + ), "key-combination-without-keys": gemini_does( call("key_combination", keys=3), observed=snapshot( From 1ffc73b052ee129d45dd1ca2a834928f0e22db5a Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 039/248] fix(agents): send GeminiConfig.excluded_predefined_functions to Gemini GeminiAgent stored the configured exclusions but GeminiComputerTool always built its computer_use tool with an empty list, so the model was still offered every predefined function. ToolAgent now asks _tool_options for the constructor options its config sets on each tool (the screenshot encoding for RFB tools, as before), and GeminiAgent adds the exclusions for its computer tool, which advertises them in the request. --- hud/agents/gemini/agent.py | 9 ++++++++- hud/agents/gemini/tools/computer.py | 18 +++++++++++++++--- hud/agents/tool_agent.py | 18 ++++++++---------- tests/agents/test_computer_use.py | 21 +++++++++++++++++++++ 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/hud/agents/gemini/agent.py b/hud/agents/gemini/agent.py index 649ce27a8..6b6459dac 100644 --- a/hud/agents/gemini/agent.py +++ b/hud/agents/gemini/agent.py @@ -17,6 +17,8 @@ if TYPE_CHECKING: from google import genai + from hud.agents.tools.base import AgentTool + from .settings import gemini_agent_settings from .tools import ( @@ -86,7 +88,6 @@ def __init__(self, config: GeminiConfig | None = None) -> None: self.max_output_tokens = config.max_output_tokens self.thinking_level = config.thinking_level self.include_thoughts = config.include_thoughts - self.excluded_predefined_functions = list(config.excluded_predefined_functions) self.max_tool_result_chars = config.max_tool_result_chars self.max_recent_turn_with_screenshots = ( gemini_agent_settings.MAX_RECENT_TURN_WITH_SCREENSHOTS @@ -94,6 +95,12 @@ def __init__(self, config: GeminiConfig | None = None) -> None: # ─── ToolAgent hooks ────────────────────────────────────────────── + def _tool_options(self, tool_cls: type[AgentTool[Any]]) -> dict[str, Any]: + options = super()._tool_options(tool_cls) + if issubclass(tool_cls, GeminiComputerTool): + options["excluded_predefined_functions"] = self.config.excluded_predefined_functions + return options + async def _initialize_state( self, *, prompt: list[mcp_types.PromptMessage] ) -> RunState[genai_types.Content]: diff --git a/hud/agents/gemini/tools/computer.py b/hud/agents/gemini/tools/computer.py index aff600765..c97811e62 100644 --- a/hud/agents/gemini/tools/computer.py +++ b/hud/agents/gemini/tools/computer.py @@ -14,6 +14,11 @@ from .base import GeminiToolSpec if TYPE_CHECKING: + from collections.abc import Sequence + + from hud.agents.tools.base import AgentToolSpec + from hud.capabilities import RFBClient + from hud.capabilities.rfb import ScreenshotEncoding from hud.types import MCPToolResult logger = logging.getLogger(__name__) @@ -50,9 +55,16 @@ class GeminiComputerTool(RFBTool): name = "computer_use" - def __init__(self, **kwargs: Any) -> None: - super().__init__(**kwargs) - self.excluded_predefined_functions: list[str] = [] + def __init__( + self, + *, + spec: AgentToolSpec, + client: RFBClient, + screenshot_encoding: ScreenshotEncoding, + excluded_predefined_functions: Sequence[str] = (), + ) -> None: + super().__init__(spec=spec, client=client, screenshot_encoding=screenshot_encoding) + self.excluded_predefined_functions = list(excluded_predefined_functions) @classmethod def default_spec(cls, model: str) -> GeminiToolSpec: diff --git a/hud/agents/tool_agent.py b/hud/agents/tool_agent.py index 198776e0a..4ffb6d680 100644 --- a/hud/agents/tool_agent.py +++ b/hud/agents/tool_agent.py @@ -17,7 +17,7 @@ from hud.agents.tools.rfb import RFBTool from hud.agents.tools.ssh import SSHInfrastructureErrorResult, SSHTool from hud.agents.types import AgentStep, ToolStep -from hud.capabilities import MCPClient, RFBClient +from hud.capabilities import MCPClient from hud.capabilities.ssh import SSHConnectionError from hud.types import MCPToolCall, MCPToolResult, Step, StopCondition from hud.utils.time import now_iso @@ -147,15 +147,7 @@ async def _build_tools( tools[tool.provider_name] = tool params.append(tool.to_params()) else: - if issubclass(tool_cls, RFBTool): - assert isinstance(client, RFBClient) - tool = tool_cls( - spec=spec, - client=client, - screenshot_encoding=self.config.screenshot_encoding, - ) - else: - tool = tool_cls(spec=spec, client=client) + tool = tool_cls(spec=spec, client=client, **self._tool_options(tool_cls)) tools[tool.provider_name] = tool params.append(tool.to_params()) @@ -167,6 +159,12 @@ async def _build_tools( return tools, params + def _tool_options(self, tool_cls: type[AgentTool[Any]]) -> dict[str, Any]: + """Constructor options this agent's config sets on a tool beyond its spec and client.""" + if issubclass(tool_cls, RFBTool): + return {"screenshot_encoding": self.config.screenshot_encoding} + return {} + async def _loop( self, run: Run, diff --git a/tests/agents/test_computer_use.py b/tests/agents/test_computer_use.py index d271e2d87..8068e3d45 100644 --- a/tests/agents/test_computer_use.py +++ b/tests/agents/test_computer_use.py @@ -729,3 +729,24 @@ async def test_gemini_keeps_screenshots_for_the_three_latest_computer_turns( responses = [part["functionResponse"] for part in tool_results(models.requests())] assert ["parts" in response for response in responses] == [False, False, True, True, True] + + +async def test_gemini_withholds_the_predefined_functions_it_is_configured_to_exclude( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([say("done")]) + agent = gemini(excluded_predefined_functions=["drag_and_drop", "search"])() + + async with fake_screen() as screen: + screen_capability = Capability.rfb(url=screen.url) + await run_task(workspace_env(tmp_path / "ws", capabilities=(screen_capability,)), agent) + + (request,) = models.requests() + computer = next(tool for tool in request.body["tools"] if "computerUse" in tool) + assert computer == { + "computerUse": { + "environment": "ENVIRONMENT_BROWSER", + "excluded_predefined_functions": ["drag_and_drop", "search"], + } + } From 412b761f5e318d278476910f51ffc5d0ab108935 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 040/248] fix(agents): send Gemini's browser chords for the environment, not the host GeminiComputerTool picked Cmd-based chords (select all, back, forward, address bar, meta) when the agent process ran on macOS, but the keys go to the environment's display over RFB. An agent on a Mac driving a Linux desktop sent Super+L where Ctrl+L was needed. Neither the RFB capability nor the manifest describes the remote OS, and environments run Linux, so the tool always sends the Linux/Windows chords. The type_text_at, go_back/go_forward, search, navigate and key_combination rows in tests/agents/test_computer_use.py pin them; they could only fail on a macOS host. --- hud/agents/gemini/tools/computer.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/hud/agents/gemini/tools/computer.py b/hud/agents/gemini/tools/computer.py index c97811e62..57fe33958 100644 --- a/hud/agents/gemini/tools/computer.py +++ b/hud/agents/gemini/tools/computer.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import platform from typing import TYPE_CHECKING, Any from google.genai import types as genai_types @@ -26,7 +25,6 @@ GEMINI_DRAG_INSET = 25 #: Gemini places every coordinate on a 1000x1000 grid regardless of the screen size. GEMINI_COORDINATE_SCALE = 1000 -IS_MAC = platform.system().lower() == "darwin" PREDEFINED_COMPUTER_USE_FUNCTIONS = ( "open_web_browser", @@ -51,7 +49,12 @@ class GeminiComputerTool(RFBTool): - """Translate Gemini predefined computer functions into RFBTool primitives.""" + """Translate Gemini predefined computer functions into RFBTool primitives. + + Browser chords (select all, back, forward, the address bar) are the Linux + and Windows ones: the keys land on the environment's display, which the + RFB capability does not describe, and environments run Linux. + """ name = "computer_use" @@ -109,10 +112,8 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: await self.move(x, y) await self.click(x, y) if args.get("clear_before_typing", True): - select_all = ["Super_L", "a"] if IS_MAC else ["Control_L", "a"] - delete_key = "BackSpace" if IS_MAC else "Delete" - await self.press_keys(select_all) - await self.press_keys([delete_key]) + await self.press_keys(["Control_L", "a"]) + await self.press_keys(["Delete"]) text = args.get("text") if isinstance(text, str) and text: await self.type_text(text) @@ -141,26 +142,22 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: return await self.screenshot() if action == "go_back": - keys = ["Super_L", "bracketleft"] if IS_MAC else ["Alt_L", "Left"] - await self.press_keys(keys) + await self.press_keys(["Alt_L", "Left"]) return await self.screenshot() if action == "go_forward": - keys = ["Super_L", "bracketright"] if IS_MAC else ["Alt_L", "Right"] - await self.press_keys(keys) + await self.press_keys(["Alt_L", "Right"]) return await self.screenshot() if action == "search": target = args.get("url") or "https://www.google.com" - keys = ["Super_L", "l"] if IS_MAC else ["Control_L", "l"] - await self.press_keys(keys) + await self.press_keys(["Control_L", "l"]) await self.type_text(str(target)) await self.press_keys(["Return"]) return await self.screenshot() if action == "navigate": - keys = ["Super_L", "l"] if IS_MAC else ["Control_L", "l"] - await self.press_keys(keys) + await self.press_keys(["Control_L", "l"]) url = args.get("url") or "" await self.type_text(str(url)) await self.press_keys(["Return"]) @@ -175,7 +172,7 @@ async def _dispatch(self, action: str, args: dict[str, Any]) -> MCPToolResult: "ctrl": "Control_L", "cmd": "Super_L", "command": "Super_L", - "meta": "Super_L" if IS_MAC else "Control_L", + "meta": "Control_L", "alt": "Alt_L", "shift": "Shift_L", "return": "Return", From 5a92be10da96dc889dc8167b0140f23f0bde6070 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 041/248] test(agents): pin how tool-agent rollouts end and recover The loop was tested by calling the private _loop on a hand-built RunState with a stub agent and a fake Run. Rollouts over a served workspace now pin the trace status, stop reason, error and exported steps, and the tool results the model was sent, for natural finishes, the step budget, each provider's token-cap vocabulary, malformed and unknown calls (answered or stopping per stop_on), failing tools and commands, provider faults, Gemini's malformed-call and empty-candidate replies, discarded Responses turns that leave previous_response_id where it was, auto_respond, a workspace whose connection drops (three consecutive failures end the run), the per-tool timeout on workspace tools, and the agent timeout. --- tests/agents/support.py | 40 ++- tests/agents/test_loop.py | 669 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 708 insertions(+), 1 deletion(-) create mode 100644 tests/agents/test_loop.py diff --git a/tests/agents/support.py b/tests/agents/support.py index da1e0c49e..45bbfa7ce 100644 --- a/tests/agents/support.py +++ b/tests/agents/support.py @@ -45,12 +45,14 @@ def workspace_env( passes: Callable[[Path], bool] = lambda _root: True, capabilities: tuple[Capability, ...] = (), name: str = "ws", + settle: float = 0.0, **workspace: Any, ) -> Environment: """An environment serving ``root`` over SSH with one template, ``task``. ``files`` seed the workspace before each run; the task scores 1.0 when - ``passes(root)`` holds after the agent finishes. ``workspace`` goes to + ``passes(root)`` holds ``settle`` seconds after the agent finishes, long + enough for anything the agent left running to show. ``workspace`` goes to :meth:`Environment.workspace` (``env=`` for the shell environment). """ env = Environment(name, capabilities=capabilities) @@ -63,6 +65,7 @@ async def seed() -> None: @env.template() async def task(prompt: str = PROMPT): yield prompt + await asyncio.sleep(settle) yield 1.0 if passes(root) else 0.0 return env @@ -76,6 +79,41 @@ async def run_task(env: Environment, agent: Agent, *, prompt: str = PROMPT, **op ) +class Relay: + """A TCP relay to one address that a test can cut, dropping every connection.""" + + def __init__(self) -> None: + self._server: asyncio.Server | None = None + self._writers: list[asyncio.StreamWriter] = [] + + async def start(self, host: str, port: int) -> int: + """Relay to ``host:port``; return the loopback port the relay listens on.""" + + async def pipe(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + while chunk := await reader.read(65536): + writer.write(chunk) + await writer.drain() + writer.close() + + async def accept(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + upstream_reader, upstream_writer = await asyncio.open_connection(host, port) + self._writers += [writer, upstream_writer] + await asyncio.gather( + pipe(reader, upstream_writer), + pipe(upstream_reader, writer), + return_exceptions=True, + ) + + self._server = await asyncio.start_server(accept, "127.0.0.1", 0) + return self._server.sockets[0].getsockname()[1] + + def cut(self) -> None: + assert self._server is not None + self._server.close() + for writer in self._writers: + writer.transport.abort() + + def write_files(root: Path, files: Mapping[str, bytes | str]) -> None: root.mkdir(parents=True, exist_ok=True) for relative, content in files.items(): diff --git a/tests/agents/test_loop.py b/tests/agents/test_loop.py new file mode 100644 index 000000000..b72117b45 --- /dev/null +++ b/tests/agents/test_loop.py @@ -0,0 +1,669 @@ +# ruff: noqa: E501 -- snapshots quote product messages verbatim +"""How a tool-agent rollout ends, and what the model hears when a call goes wrong. + +Rows script a provider over a served workspace and pin the trace's status, +stop reason and error, the steps it exported, and the tool results the +model was sent: natural finishes, step budgets, token caps, malformed and +unknown calls, failing tools, provider faults, lost workspaces and timeouts. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +import pytest +from fastmcp import FastMCP +from inline_snapshot import snapshot + +from hud.agents import ClaudeAgent, GeminiAgent, OpenAIAgent, OpenAIChatAgent +from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig +from tests.agents.support import Relay, mcp_server, run_task, workspace_env +from tests.harness import Reply, Turn, call, fail, say, shell_call, steps + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hud.agents.base import Agent + from hud.eval import Run + from tests.harness import HudEnv, ModelRequest, Models + + +def chat(**config: Any) -> Callable[[], Agent]: + return lambda: OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus", **config)) + + +def responses(**config: Any) -> Callable[[], Agent]: + return lambda: OpenAIAgent(OpenAIConfig(model="gpt-5.6", **config)) + + +def claude(**config: Any) -> Callable[[], Agent]: + return lambda: ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6", **config)) + + +def gemini(**config: Any) -> Callable[[], Agent]: + return lambda: GeminiAgent(GeminiConfig(model="gemini-3.1-pro-preview", **config)) + + +def result_texts(requests: list[ModelRequest]) -> list[str]: + """Every tool result text the model was sent, oldest first.""" + if requests and requests[-1].protocol == "responses": + return [text for request in requests for text in request.tool_results] + return requests[-1].tool_results if requests else [] + + +def ending(run: Run, requests: list[ModelRequest]) -> dict[str, Any]: + trace = run.trace + return { + "status": trace.status, + "stop_reason": trace.stop_reason, + "truncated": trace.is_truncated, + "error": trace.error, + "steps": [step["source"] for step in steps(run.trace_id)], + "results": result_texts(requests), + } + + +def sequence(*turns: Turn) -> Callable[[ModelRequest], Turn]: + """Answer requests with ``turns`` in arrival order, whatever they carry.""" + remaining = iter(turns) + return lambda _request: next(remaining) + + +@dataclass(frozen=True) +class Row: + agent: Callable[[], Agent] + script: list[Turn] | Callable[[ModelRequest], Turn] + ending: Any + + +MALFORMED = Turn( + native=( + { + "id": "call_cut", + "type": "function", + "function": {"name": "bash", "arguments": '{"command": "ec'}, + }, + ) +) + +ROWS = { + "done": Row( + chat(), + [say("done")], + snapshot( + { + "status": "completed", + "stop_reason": "done", + "truncated": False, + "error": None, + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "step-budget-spent": Row( + chat(max_steps=2), + [call("bash", command="true"), call("bash", command="true")], + snapshot( + { + "status": "completed", + "stop_reason": "max_steps", + "truncated": True, + "error": None, + "steps": ["task", "user", "agent", "tool", "agent", "tool", "task"], + "results": [ + """\ +$ true + +(exit 0)\ +""" + ], + } + ), + ), + "chat-token-cap": Row( + chat(), + [Turn(text="part", truncated=True)], + snapshot( + { + "status": "completed", + "stop_reason": "length", + "truncated": True, + "error": None, + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "responses-token-cap": Row( + responses(), + [Turn(text="part", truncated=True)], + snapshot( + { + "status": "completed", + "stop_reason": "length", + "truncated": True, + "error": None, + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "claude-token-cap": Row( + claude(), + [Turn(text="part", truncated=True)], + snapshot( + { + "status": "completed", + "stop_reason": "length", + "truncated": True, + "error": None, + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "gemini-token-cap": Row( + gemini(), + [Turn(text="part", truncated=True)], + snapshot( + { + "status": "completed", + "stop_reason": "length", + "truncated": True, + "error": None, + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "malformed-call-answered": Row( + chat(), + [MALFORMED, say("recovered")], + snapshot( + { + "status": "completed", + "stop_reason": "done", + "truncated": False, + "error": None, + "steps": ["task", "user", "agent", "tool", "agent", "task"], + "results": [ + "the arguments for this 'bash' call arrived incomplete (cut off mid-generation or invalid JSON), so it was not executed. Received: '{\"command\": \"ec'. Re-issue the call in full." + ], + } + ), + ), + "malformed-call-stops": Row( + chat(stop_on={"malformed_tool_call"}), + [MALFORMED], + snapshot( + { + "status": "completed", + "stop_reason": "malformed_tool_call", + "truncated": True, + "error": None, + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "token-capped-call-stops": Row( + chat(stop_on={"length"}), + [Turn(tool_calls=call("bash", command="ls").tool_calls, truncated=True)], + snapshot( + { + "status": "completed", + "stop_reason": "length", + "truncated": True, + "error": None, + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "unknown-tool": Row( + chat(), + [call("ghost"), say("done")], + snapshot( + { + "status": "completed", + "stop_reason": "done", + "truncated": False, + "error": None, + "steps": ["task", "user", "agent", "tool", "agent", "task"], + "results": ["unknown tool: 'ghost'"], + } + ), + ), + "tool-raises": Row( + chat(), + [call("bash", command=""), say("done")], + snapshot( + { + "status": "completed", + "stop_reason": "done", + "truncated": False, + "error": None, + "steps": ["task", "user", "agent", "tool", "agent", "task"], + "results": ["tool error: command is required"], + } + ), + ), + "failed-command": Row( + chat(), + [call("bash", command="exit 3"), say("done")], + snapshot( + { + "status": "completed", + "stop_reason": "done", + "truncated": False, + "error": None, + "steps": ["task", "user", "agent", "tool", "agent", "task"], + "results": [ + """\ +$ exit 3 + +(exit 3)\ +""" + ], + } + ), + ), + "provider-rejects": Row( + chat(), + [fail(400, "bad request")], + snapshot( + { + "status": "error", + "stop_reason": None, + "truncated": False, + "error": "Error code: 400 - {'error': {'type': 'error', 'message': 'bad request'}}", + "steps": ["task", "user", "system", "task"], + "results": [], + } + ), + ), + "gemini-malformed-function-call": Row( + gemini(), + [Turn(finish="MALFORMED_FUNCTION_CALL")], + snapshot( + { + "status": "error", + "stop_reason": "malformed_tool_call", + "truncated": True, + "error": "Provider returned a malformed function call", + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "gemini-malformed-function-call-auto-respond": Row( + gemini(auto_respond=True), + [Turn(finish="MALFORMED_FUNCTION_CALL")], + snapshot( + { + "status": "error", + "stop_reason": "malformed_tool_call", + "truncated": True, + "error": "Provider returned a malformed function call", + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "gemini-no-candidates": Row( + gemini(), + [Turn(fail=Reply(json={"candidates": []}))], + snapshot( + { + "status": "error", + "stop_reason": None, + "truncated": False, + "error": "Gemini returned no candidates for model gemini-3.1-pro-preview", + "steps": ["task", "user", "system", "task"], + "results": [], + } + ), + ), + "responses-empty-shell-call-resampled": Row( + responses(), + sequence(shell_call(), say("done")), + snapshot( + { + "status": "completed", + "stop_reason": "done", + "truncated": False, + "error": None, + "steps": ["task", "user", "agent", "task"], + "results": [], + } + ), + ), + "responses-empty-shell-call-on-last-step": Row( + responses(max_steps=1), + sequence(shell_call()), + snapshot( + { + "status": "error", + "stop_reason": None, + "truncated": False, + "error": "Model emitted shell_call(s) with an empty commands array (call_shell); continuing from response resp_1 would fail with OpenAI 500s, so it was not committed to the run state.", + "steps": ["task", "user", "system", "task"], + "results": [], + } + ), + ), +} + + +@pytest.mark.parametrize("row", ROWS.values(), ids=ROWS.keys()) +async def test_a_rollout_ends_for_the_reason_its_last_turn_gives( + row: Row, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + if callable(row.script): + models.respond(row.script) + else: + models.script(row.script) + + run = await run_task(workspace_env(tmp_path / "ws"), row.agent()) + + assert ending(run, models.requests()) == row.ending + + +async def test_a_malformed_call_replays_as_empty_arguments( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + """History stays parseable for the provider after a call that never parsed.""" + hud_env.set(HUD_API_KEY="k") + models.script([MALFORMED, say("recovered")]) + + await run_task(workspace_env(tmp_path / "ws"), chat()()) + + replayed = models.requests()[-1].body["messages"][1]["tool_calls"] + assert replayed == [ + {"id": "call_cut", "type": "function", "function": {"name": "bash", "arguments": "{}"}} + ] + + +async def test_a_discarded_responses_turn_does_not_advance_the_response_chain( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(sequence(shell_call("true"), shell_call(), say("done"))) + + run = await run_task(workspace_env(tmp_path / "ws"), responses()()) + + chain = [request.body.get("previous_response_id") for request in models.requests()] + assert (run.trace.content, chain) == ("done", [None, "resp_1", "resp_1"]) + + +def responder(*decisions: Turn) -> Callable[[ModelRequest], Turn]: + """Chat turns for the agent; the auto-responder's Responses requests get ``decisions``.""" + agent_turns = [say("Shall I go on with the rest?"), say("finished")] + answers = iter(decisions) + + def answer(request: ModelRequest) -> Turn: + if request.protocol == "responses": + return next(answers) + return agent_turns[request.turn] + + return answer + + +AUTO_RESPOND = { + "continue-then-stop": ( + responder(say("CONTINUE"), say("STOP")), + snapshot( + { + "content": "finished", + "steps": ["task", "user", "agent", "user", "agent", "task"], + "last_user_message": [{"type": "text", "text": "CONTINUE"}], + } + ), + ), + "stop": ( + responder(say("STOP")), + snapshot( + { + "content": "Shall I go on with the rest?", + "steps": ["task", "user", "agent", "task"], + "last_user_message": [{"type": "text", "text": "Do the task."}], + } + ), + ), + "responder-fails": ( + responder(fail(400)), + snapshot( + { + "content": "Shall I go on with the rest?", + "steps": ["task", "user", "agent", "task"], + "last_user_message": [{"type": "text", "text": "Do the task."}], + } + ), + ), +} + + +@pytest.mark.parametrize(("policy", "expected"), AUTO_RESPOND.values(), ids=AUTO_RESPOND.keys()) +async def test_auto_respond_answers_a_question_or_lets_the_rollout_end( + policy: Callable[[ModelRequest], Turn], + expected: Any, + models: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(policy) + + run = await run_task(workspace_env(tmp_path / "ws"), chat(auto_respond=True)()) + + agent_requests = [request for request in models.requests() if request.protocol == "chat"] + observed = { + "content": run.trace.content, + "steps": [step["source"] for step in steps(run.trace_id)], + "last_user_message": agent_requests[-1].body["messages"][-1]["content"], + } + assert observed == expected + + +def cutter(relay: Relay) -> FastMCP: + server = FastMCP("cutter") + + @server.tool() + def cut_workspace() -> str: + """Drop the connection to the workspace.""" + relay.cut() + return "cut" + + return server + + +LOST = { + "three-lost-calls-in-a-row": ( + ["bash", "bash", "bash"], + snapshot( + { + "status": "error", + "stop_reason": None, + "truncated": False, + "error": "SSH tool failure limit reached after 3 consecutive errors", + "steps": [ + "task", + "user", + "agent", + "tool", + "agent", + "tool", + "agent", + "tool", + "agent", + "tool", + "system", + "task", + ], + "results": [ + "cut", + "tool error: SSH reconnect failed after 3 attempts", + "tool error: SSH reconnect failed after 3 attempts", + ], + } + ), + ), + "another-result-resets-the-count": ( + ["bash", "ghost", "bash", "bash"], + snapshot( + { + "status": "completed", + "stop_reason": "done", + "truncated": False, + "error": None, + "steps": [ + "task", + "user", + "agent", + "tool", + "agent", + "tool", + "agent", + "tool", + "agent", + "tool", + "agent", + "tool", + "agent", + "task", + ], + "results": [ + "cut", + "tool error: SSH reconnect failed after 3 attempts", + "unknown tool: 'ghost'", + "tool error: SSH reconnect failed after 3 attempts", + "tool error: SSH reconnect failed after 3 attempts", + ], + } + ), + ), +} + + +@pytest.mark.parametrize(("tools", "expected"), LOST.values(), ids=LOST.keys()) +async def test_a_lost_workspace_ends_the_rollout_after_three_consecutive_failures( + tools: list[str], expected: Any, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + root = tmp_path / "ws" + script = [call("cut_workspace"), *(call(name, command="true") for name in tools)] + models.script([*script, say("done")]) + relay = Relay() + env = workspace_env(root) + + @env.initialize + async def reach_the_workspace_through_the_relay() -> None: + shell = env.capability("shell") + address = urlsplit(shell.url) + assert address.hostname is not None and address.port is not None + port = await relay.start(address.hostname, address.port) + env.add_capability(replace(shell, url=f"ssh://127.0.0.1:{port}")) + + async with mcp_server(cutter(relay)) as tools: + env.add_capability(tools) + run = await run_task(env, chat(max_steps=10)()) + + assert ending(run, models.requests()) == expected + + +async def test_a_slow_shell_command_times_out_and_the_rollout_goes_on( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + root = tmp_path / "ws" + models.script( + [ + call("bash", command="touch started; sleep 1.5; touch late"), + call("bash", command="sleep 0.1; touch quick"), + say("done"), + ] + ) + env = workspace_env( + root, + settle=1.5, + passes=lambda root: all( + [(root / "started").exists(), (root / "quick").exists(), not (root / "late").exists()] + ), + ) + + run = await run_task(env, claude(tool_timeout_seconds=0.5)()) + + assert run.reward == 1.0 + assert ending(run, models.requests())["results"] == [ + "Error: bash timed out after 0.5s; retry with a shorter command", + "$ sleep 0.1; touch quick\n\n(exit 0)", + ] + + +async def test_an_editor_blocked_on_its_file_times_out( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + root = tmp_path / "ws" + models.script( + [ + call("bash", command="mkfifo pipe"), + call( + "str_replace_based_edit_tool", + command="str_replace", + path=f"{root}/pipe", + old_str="a", + new_str="b", + ), + say("done"), + ] + ) + + run = await run_task(workspace_env(root), claude(tool_timeout_seconds=0.5)()) + + assert run.trace.status == "completed" + assert ending(run, models.requests())["results"][-1] == ( + "Error: str_replace_based_edit_tool timed out after 0.5s; retry with a shorter command" + ) + + +def slow_tools() -> FastMCP: + server = FastMCP("slow") + + @server.tool() + async def slow() -> str: + """Answer after a second.""" + await asyncio.sleep(1.0) + return "slow answer" + + return server + + +async def test_the_tool_timeout_covers_workspace_tools_only( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([call("slow"), say("done")]) + + async with mcp_server(slow_tools()) as tools: + env = workspace_env(tmp_path / "ws", capabilities=(tools,)) + await run_task(env, claude(tool_timeout_seconds=0.5)()) + + assert models.requests()[-1].tool_results == ["slow answer"] + + +async def test_the_agent_timeout_stops_a_rollout_waiting_on_its_provider( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([Turn(fail=Reply(status=500, delay=2.0))]) + + run = await run_task(workspace_env(tmp_path / "ws"), chat(timeout_seconds=0.5)()) + + assert (run.trace.status, run.trace.stop_reason) == ("error", "timeout") From d7f1c4352bd15edae40770234ac4917cb63f6264 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 042/248] test(agents): pin each provider's native workspace tools on a real workspace The shell, file and editor tools were tested against an in-memory SSH emulator that parsed command spellings. Rollouts now run them over a served workspace and pin what the model is sent back and the files left behind: OpenAI shell command order, whole-command timeouts and per-command output bounds, chat bash workdir/timeout, read paging and directory listings, edit and write rules, Gemini shell/replace/read/search, and the Claude bash and text editor, including each error a model can hit. --- tests/agents/support.py | 24 ++ tests/agents/test_computer_use.py | 21 +- tests/agents/test_workspace_tools.py | 511 +++++++++++++++++++++++++++ 3 files changed, 537 insertions(+), 19 deletions(-) create mode 100644 tests/agents/test_workspace_tools.py diff --git a/tests/agents/support.py b/tests/agents/support.py index 45bbfa7ce..3338719e8 100644 --- a/tests/agents/support.py +++ b/tests/agents/support.py @@ -234,5 +234,29 @@ def tool_results(requests: list[ModelRequest]) -> list[Any]: return [message for message in last.body["messages"] if message["role"] == "tool"] +def result_line(entry: dict[str, Any], *paths: Path) -> str: + """One tool result entry from :func:`tool_results` as the model sees it, in a line. + + Images read ````; an Anthropic error result starts ``error:``. + """ + entry = wire(entry, *paths) + if entry.get("role") == "tool": + return entry["content"] + if entry.get("type") == "tool_result": + parts = [block.get("text") or block["source"]["data"] for block in entry["content"]] + return ("error: " if entry["is_error"] else "") + " ".join(parts) + if entry.get("type") == "computer_call_output": + acknowledged = [check["id"] for check in entry.get("acknowledged_safety_checks", [])] + return " ".join([entry["output"]["image_url"], *(f"ack {id}" for id in acknowledged)]) + if entry.get("type") == "shell_call_output": + limit = entry.get("max_output_length") + return json.dumps(entry["output"]) + ("" if limit is None else f" limit={limit}") + if "functionResponse" in entry: + response = entry["functionResponse"] + images = [part["inline_data"]["data"] for part in response.get("parts", [])] + return " ".join([response["name"], json.dumps(response["response"]), *images]) + return "user: " + " ".join(block["text"] for block in entry["content"]) + + def json_lines(text: str) -> list[dict[str, Any]]: return [json.loads(line) for line in text.splitlines() if line.strip()] diff --git a/tests/agents/test_computer_use.py b/tests/agents/test_computer_use.py index 8068e3d45..0394dba0c 100644 --- a/tests/agents/test_computer_use.py +++ b/tests/agents/test_computer_use.py @@ -9,7 +9,6 @@ from __future__ import annotations -import json import re from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -21,7 +20,7 @@ from hud.agents import ClaudeAgent, GeminiAgent, OpenAIAgent from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIConfig from hud.capabilities import Capability -from tests.agents.support import run_task, tool_results, wire, workspace_env +from tests.agents.support import result_line, run_task, tool_results, wire, workspace_env from tests.harness import ( KeyEvent, Turn, @@ -62,22 +61,6 @@ def events(screen: FakeScreen) -> str: return " ".join(rendered) -def seen(entry: dict[str, Any]) -> str: - """One tool result entry as the model sees it, in a line: images as ````.""" - entry = wire(entry) - if entry.get("type") == "tool_result": - parts = [block.get("text") or block["source"]["data"] for block in entry["content"]] - return ("error: " if entry["is_error"] else "") + " ".join(parts) - if entry.get("type") == "computer_call_output": - acknowledged = [check["id"] for check in entry.get("acknowledged_safety_checks", [])] - return " ".join([entry["output"]["image_url"], *(f"ack {id}" for id in acknowledged)]) - if "functionResponse" in entry: - response = entry["functionResponse"] - images = [part["inline_data"]["data"] for part in response.get("parts", [])] - return " ".join([response["name"], json.dumps(response["response"]), *images]) - return "user: " + " ".join(block["text"] for block in entry["content"]) - - @dataclass(frozen=True) class Row: agent: Callable[[], Agent] @@ -519,7 +502,7 @@ async def test_a_computer_action_reaches_the_screen_as_rfb_input( assert run.trace.status == "completed" observed = { "events": events(screen), - "results": [seen(entry) for entry in tool_results(models.requests())], + "results": [result_line(entry) for entry in tool_results(models.requests())], } assert observed == row.observed diff --git a/tests/agents/test_workspace_tools.py b/tests/agents/test_workspace_tools.py new file mode 100644 index 000000000..c1ec210e8 --- /dev/null +++ b/tests/agents/test_workspace_tools.py @@ -0,0 +1,511 @@ +# ruff: noqa: E501 -- snapshots quote product messages verbatim +"""Each provider's native workspace tools act on the files a real workspace serves. + +Rows script one or more tool calls against a workspace seeded with files and +pin what the model was sent back and what the workspace holds afterwards. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Any + +import pytest +from inline_snapshot import snapshot + +from hud.agents import ClaudeAgent, GeminiAgent, OpenAIAgent, OpenAIChatAgent +from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig +from tests.agents.support import result_line, run_task, tool_results, workspace_env +from tests.harness import call, say, shell_call + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hud.agents.base import Agent + from tests.harness import HudEnv, Models, Turn + + +def files(root: Path) -> dict[str, str]: + """Every regular file under ``root``, by relative path.""" + return { + str(path.relative_to(root)): path.read_text(errors="replace") + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +@dataclass(frozen=True) +class Row: + agent: Callable[[], Agent] + script: Callable[[Path], list[Turn]] + observed: Any + seed: dict[str, str] | None = None + + +def claude() -> Agent: + return ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")) + + +def openai() -> Agent: + return OpenAIAgent(OpenAIConfig(model="gpt-5.6")) + + +def gemini() -> Agent: + return GeminiAgent(GeminiConfig(model="gemini-3.1-pro-preview")) + + +def chat() -> Agent: + return OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus")) + + +def editor(**arguments: Any) -> Turn: + return call("str_replace_based_edit_tool", **arguments) + + +NOTES = {"notes.txt": "one\ntwo\nthree\n", "dup.txt": "a a a"} + +ROWS = { + # OpenAI Responses shell + "openai-shell-commands-run-in-order": Row( + openai, + lambda _: [shell_call("printf A > a.txt", "cat a.txt"), say("done")], + snapshot( + { + "results": [ + '[{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}, {"stdout": "A", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}] limit=10485760' + ], + "files": {"a.txt": "A"}, + } + ), + ), + "openai-shell-timeout-covers-the-whole-command": Row( + openai, + lambda _: [shell_call("echo ready; sleep 5; echo late", timeout_ms=500), say("done")], + snapshot( + { + "results": [ + '[{"stdout": "ready\\n", "stderr": "", "outcome": {"type": "exit", "exit_code": 124}}] limit=10485760' + ], + "files": {}, + } + ), + ), + "openai-shell-bounds-each-command-on-its-own": Row( + openai, + lambda _: [ + shell_call( + "printf 'head-%050d' 0; printf '%050d-tail' 0 >&2", + "printf 'head-%050d' 0", + max_output_length=30, + ), + say("done"), + ], + snapshot( + { + "results": [ + '[{"stdout": "head-00000[truncated]", "stderr": "0000-tail", "outcome": {"type": "exit", "exit_code": 0}}, {"stdout": "head-00000[truncated]000000000", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}] limit=30' + ], + "files": {}, + } + ), + ), + "openai-shell-limit-smaller-than-the-marker": Row( + openai, + lambda _: [shell_call("seq 100", max_output_length=1), say("done")], + snapshot( + { + "results": [ + '[{"stdout": "[", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}] limit=1' + ], + "files": {}, + } + ), + ), + "openai-shell-caps-the-limit": Row( + openai, + lambda _: [shell_call("true", max_output_length=20 * 1024 * 1024), say("done")], + snapshot( + { + "results": [ + '[{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}] limit=10485760' + ], + "files": {}, + } + ), + ), + "openai-shell-rejects-a-zero-limit": Row( + openai, + lambda _: [shell_call("touch ran", max_output_length=0), say("done")], + snapshot( + { + "results": [ + '[{"stdout": "", "stderr": "max_output_length must be a positive integer", "outcome": {"type": "exit", "exit_code": 1}}]' + ], + "files": {}, + } + ), + ), + "openai-shell-rejects-commands-that-are-not-a-list": Row( + openai, + lambda _: [ + replace( + shell_call(), + native=( + { + "type": "shell_call", + "id": "sh_bad", + "call_id": "call_bad", + "action": {"commands": [1, 2]}, + "status": "completed", + }, + ), + ), + say("done"), + ], + snapshot( + { + "results": [ + '[{"stdout": "", "stderr": "commands must be a list of strings", "outcome": {"type": "exit", "exit_code": 1}}] limit=10485760' + ], + "files": {}, + } + ), + ), + # OpenAI-compatible chat tools + "chat-bash-workdir-and-timeout": Row( + chat, + lambda root: [ + call("bash", command="mkdir -p 'my dir'"), + call( + "bash", + command="pwd; sleep 5; echo late", + workdir=f"{root}/my dir", + timeout=500, + ), + say("done"), + ], + snapshot( + { + "results": [ + """\ +$ mkdir -p 'my dir' + +(exit 0)\ +""", + """\ +$ cd '/my dir' && timeout 1s bash -lc 'pwd; sleep 5; echo late' +/my dir + +(exit 124)\ +""", + ], + "files": {}, + } + ), + ), + "chat-write-creates-parent-directories": Row( + chat, + lambda root: [call("write", filePath=f"{root}/deep/er/x.txt", content="x"), say("done")], + snapshot( + {"results": ["wrote 1 bytes to /deep/er/x.txt"], "files": {"deep/er/x.txt": "x"}} + ), + ), + "chat-read-pages-text": Row( + chat, + lambda root: [ + call("read", filePath=f"{root}/notes.txt", offset=0, limit=1), + call("read", filePath=f"{root}/notes.txt", offset=2), + call("read", filePath=f"{root}/notes.txt", offset=9), + say("done"), + ], + snapshot( + { + "results": [ + """\ +/notes.txt +file + +1: one + +(Showing lines 1-1 of 3. Use offset=2 to continue.) +\ +""", + """\ +/notes.txt +file + +2: two +3: three + +(End of file - total 3 lines) +\ +""", + "Offset 9 is out of range for this file (3 lines)", + ], + "files": { + "dup.txt": "a a a", + "notes.txt": """\ +one +two +three +""", + }, + } + ), + seed=NOTES, + ), + "chat-read-lists-a-directory": Row( + chat, + lambda root: [ + call("bash", command="mkdir -p pkg/sub && touch pkg/mod.py"), + call("read", filePath=f"{root}/pkg"), + say("done"), + ], + snapshot( + { + "results": [ + """\ +$ mkdir -p pkg/sub && touch pkg/mod.py + +(exit 0)\ +""", + """\ +/pkg +directory + +mod.py +sub + +(2 entries) +\ +""", + ], + "files": {"pkg/mod.py": ""}, + } + ), + ), + "chat-edit": Row( + chat, + lambda root: [ + call("edit", filePath=f"{root}/notes.txt", oldString="two", newString="2"), + call("edit", filePath=f"{root}/dup.txt", oldString="a", newString="b"), + call("edit", filePath=f"{root}/dup.txt", oldString="a", newString="b", replaceAll=True), + call("edit", filePath=f"{root}/notes.txt", oldString="one", newString="one"), + call("edit", filePath=f"{root}/notes.txt", oldString="", newString="new"), + call("edit", filePath=f"{root}/fresh/new.txt", oldString="", newString="new"), + call("edit", filePath=f"{root}/notes.txt", oldString="zzz", newString="y"), + say("done"), + ], + snapshot( + { + "results": [ + "wrote 12 bytes to /notes.txt", + "oldString matches 3 times in /dup.txt; set replaceAll to true", + "wrote 5 bytes to /dup.txt", + "No changes to apply: oldString and newString are identical.", + "oldString cannot be empty when editing an existing file. Provide exact text to replace, or use write for full-file replacement.", + "wrote 3 bytes to /fresh/new.txt", + "oldString not found in /notes.txt", + ], + "files": { + "dup.txt": "b b b", + "fresh/new.txt": "new", + "notes.txt": """\ +one +2 +three +""", + }, + } + ), + seed=NOTES, + ), + "chat-grep-and-glob": Row( + chat, + lambda root: [ + call("grep", pattern="tw.", path=str(root), include="*.txt"), + call("glob", pattern="*.txt", path=str(root)), + say("done"), + ], + snapshot( + { + "results": [ + """\ +$ grep -rn tw. --include='*.txt' +/notes.txt:1:two + +(exit 0)\ +""", + """\ +$ find -name '*.txt' +/notes.txt + +(exit 0)\ +""", + ], + "files": {"notes.txt": "two\n"}, + } + ), + seed={"notes.txt": "two\n"}, + ), + # Gemini tools + "gemini-shell-in-a-quoted-directory": Row( + gemini, + lambda root: [ + call("run_shell_command", command="mkdir -p 'my dir'"), + call("run_shell_command", command="pwd", dir_path=f"{root}/my dir"), + say("done"), + ], + snapshot( + { + "results": [ + 'run_shell_command {"success": true, "output": "$ mkdir -p \'my dir\'\\n\\n(exit 0)"}', + 'run_shell_command {"success": true, "output": "$ cd \'/my dir\' && pwd\\n/my dir\\n\\n(exit 0)"}', + ], + "files": {}, + } + ), + ), + "gemini-shell-without-a-command": Row( + gemini, + lambda _: [call("run_shell_command", command=""), say("done")], + snapshot( + { + "results": ['run_shell_command {"error": "tool error: command is required"}'], + "files": {}, + } + ), + ), + "gemini-replace-and-read-lines": Row( + gemini, + lambda root: [ + call("replace", file_path=f"{root}/notes.txt", old_string="two", new_string="2"), + call("replace", file_path=f"{root}/notes.txt", old_string="zzz", new_string="y"), + call("replace", file_path=f"{root}/made.txt", old_string="", new_string="made"), + call("read_file", file_path=f"{root}/notes.txt", start_line=2, end_line=3), + say("done"), + ], + snapshot( + { + "results": [ + 'replace {"success": true, "output": "wrote 12 bytes to /notes.txt"}', + 'replace {"error": "old_string not found in /notes.txt"}', + 'replace {"success": true, "output": "wrote 4 bytes to /made.txt"}', + 'read_file {"success": true, "output": "2\\nthree\\n"}', + ], + "files": { + "dup.txt": "a a a", + "made.txt": "made", + "notes.txt": """\ +one +2 +three +""", + }, + } + ), + seed=NOTES, + ), + "gemini-search-and-glob": Row( + gemini, + lambda root: [ + call("grep_search", pattern="tw.", dir_path=str(root), include_pattern="*.txt"), + call("glob", pattern="*.txt", dir_path=str(root)), + say("done"), + ], + snapshot( + { + "results": [ + 'grep_search {"success": true, "output": "$ grep -rn tw. --include=\'*.txt\'\\n/notes.txt:1:two\\n\\n(exit 0)"}', + 'glob {"success": true, "output": "$ find -name \'*.txt\'\\n/notes.txt\\n\\n(exit 0)"}', + ], + "files": {"notes.txt": "two\n"}, + } + ), + seed={"notes.txt": "two\n"}, + ), + # Claude tools + "claude-bash-restart-and-missing-command": Row( + claude, + lambda _: [call("bash", restart=True), call("bash"), say("done")], + snapshot( + { + "results": [ + "restart is unnecessary; each command runs in a fresh shell session", + "error: Error: command is required unless restart is true", + ], + "files": {}, + } + ), + ), + "claude-editor": Row( + claude, + lambda root: [ + editor(command="create", path=f"{root}/made.txt", file_text="made\n"), + editor(command="str_replace", path=f"{root}/notes.txt", old_str="two", new_str="2"), + editor(command="str_replace", path=f"{root}/dup.txt", old_str="a", new_str="b"), + editor(command="str_replace", path=f"{root}/notes.txt", old_str="zzz", new_str="y"), + editor(command="insert", path=f"{root}/notes.txt", insert_line=1, new_str="1.5"), + editor(command="insert", path=f"{root}/notes.txt", insert_line=99, new_str="x"), + editor(command="view", path=f"{root}/missing.txt"), + editor(command="delete", path=f"{root}/notes.txt"), + say("done"), + ], + snapshot( + { + "results": [ + "wrote 5 bytes to /made.txt", + "wrote 12 bytes to /notes.txt", + "error: Error: old_str matches 3 times in /dup.txt; must be unique", + "error: Error: old_str not found in /notes.txt", + "wrote 16 bytes to /notes.txt", + "error: Error: insert_line 99 out of range (file has 4 lines)", + "error: Error: cat: /missing.txt: No such file or directory", + "error: Error: unknown editor command: 'delete'", + ], + "files": { + "dup.txt": "a a a", + "made.txt": "made\n", + "notes.txt": """\ +one +1.5 +2 +three +""", + }, + } + ), + seed=NOTES, + ), +} + + +@pytest.mark.parametrize("row", ROWS.values(), ids=ROWS.keys()) +async def test_a_native_tool_acts_on_the_served_workspace( + row: Row, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + root = tmp_path / "ws" + models.script(row.script(root)) + + run = await run_task(workspace_env(root, files=row.seed), row.agent()) + + assert run.trace.status == "completed" + observed = { + "results": [result_line(entry, root) for entry in tool_results(models.requests())], + "files": files(root), + } + assert observed == row.observed + + +async def test_a_path_outside_the_workspace_root_is_used_as_given( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + outside = tmp_path / "elsewhere" / "probe.txt" + models.script([call("write", filePath=str(outside), content="done"), say("done")]) + + await run_task(workspace_env(tmp_path / "ws"), chat()) + + assert outside.read_text() == "done" From d7ed49c2cdb2cc51a7759b3f330e120664111aaa Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 043/248] test(agents): pin how ClaudeAgent retries a failing Messages stream Retries were tested by patching asyncio.sleep and the jitter source inside the agent module. The scripted Anthropic stream now reports inline errors, drops mid-message and streams tool input that is not JSON; rows pin which faults are retried and how often, what the third request says after two invalid tool payloads, and how an exhausted retry ends the run. A Bedrock inference profile row runs where the bedrock extra is installed. --- tests/agents/test_claude_transport.py | 329 ++++++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 tests/agents/test_claude_transport.py diff --git a/tests/agents/test_claude_transport.py b/tests/agents/test_claude_transport.py new file mode 100644 index 000000000..0d01ab3cf --- /dev/null +++ b/tests/agents/test_claude_transport.py @@ -0,0 +1,329 @@ +# ruff: noqa: E501 -- snapshots quote product messages verbatim +"""How ClaudeAgent recovers from what the Messages stream throws at it. + +Rows script the Anthropic stream behind the fake gateway to fail transiently, +fail for good, cut out, or emit tool input that is not JSON, and pin how many +requests the agent made, what it sent on a retry, and how the run ended. A +Bedrock inference profile goes through Bedrock's own invoke endpoint. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest +from inline_snapshot import snapshot + +from hud.agents import ClaudeAgent +from hud.agents.types import ClaudeConfig +from tests.agents.support import run_task, wire, workspace_env +from tests.harness import Turn, interrupted, say, steps, stream_error + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from tests.harness import FakeServices, HudEnv, ModelRequest, Models + +NO_WAIT = {"retry-after-ms": "0"} + + +def sequence(*turns: Turn) -> Callable[[ModelRequest], Turn]: + remaining = iter(turns) + return lambda _request: next(remaining) + + +def bad_json(fragment: str) -> Turn: + """A tool call whose streamed input is not JSON.""" + return Turn( + native=({"type": "tool_use", "id": "toolu_bad", "name": "bash", "input": fragment},) + ) + + +STREAMS = { + "transient-error-then-answer": ( + [stream_error("upstream_error", headers=NO_WAIT), say("recovered")], + snapshot( + { + "status": "completed", + "content": "recovered", + "error": None, + "requests": 2, + "last_messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Do the task.", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + "steps": ["task", "user", "agent", "task"], + } + ), + ), + "two-transient-errors-then-answer": ( + [ + stream_error("timeout_error", headers=NO_WAIT), + stream_error("overloaded_error", headers=NO_WAIT), + say("recovered"), + ], + snapshot( + { + "status": "completed", + "content": "recovered", + "error": None, + "requests": 3, + "last_messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Do the task.", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + "steps": ["task", "user", "agent", "task"], + } + ), + ), + "transient-errors-exhaust-the-retries": ( + [stream_error("gateway_timeout", headers=NO_WAIT)] * 3, + snapshot( + { + "status": "error", + "content": None, + "error": "{'type': 'error', 'error': {'type': 'gateway_timeout', 'message': 'scripted gateway_timeout'}}", + "requests": 3, + "last_messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Do the task.", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + "steps": ["task", "user", "system", "task"], + } + ), + ), + "a-request-error-is-not-retried": ( + [stream_error("invalid_request_error", headers=NO_WAIT)], + snapshot( + { + "status": "error", + "content": None, + "error": "{'type': 'error', 'error': {'type': 'invalid_request_error', 'message': 'scripted invalid_request_error'}}", + "requests": 1, + "last_messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Do the task.", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + "steps": ["task", "user", "system", "task"], + } + ), + ), + "an-interrupted-stream-is-retried": ( + [interrupted(), say("recovered")], + snapshot( + { + "status": "completed", + "content": "recovered", + "error": None, + "requests": 2, + "last_messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Do the task.", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + "steps": ["task", "user", "agent", "task"], + } + ), + ), + "invalid-tool-json-is-retried-silently": ( + [bad_json('{"command": ls}'), say("recovered")], + snapshot( + { + "status": "completed", + "content": "recovered", + "error": None, + "requests": 2, + "last_messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Do the task.", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + "steps": ["task", "user", "agent", "task"], + } + ), + ), + "invalid-tool-json-twice-is-pointed-out": ( + [bad_json('{"command": ls}'), bad_json('{"command": ls}'), say("recovered")], + snapshot( + { + "status": "completed", + "content": "recovered", + "error": None, + "requests": 3, + "last_messages": [ + {"role": "user", "content": [{"type": "text", "text": "Do the task."}]}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": """\ +Your previous tool-call arguments were invalid JSON. Retry the same tool call with valid JSON arguments. +Malformed payload (wrapped): {"INVALID_JSON": "{\\"command\\": ls}"}\ +""", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + ], + "steps": ["task", "user", "agent", "task"], + } + ), + ), + "invalid-tool-json-three-times-fails-the-run": ( + [bad_json('{"command": ls}')] * 3, + snapshot( + { + "status": "error", + "content": None, + "error": 'Unable to parse tool parameter JSON from model. Please retry your request or adjust your prompt. Error: expected value at line 1 column 13. JSON: {"command": ls}', + "requests": 3, + "last_messages": [ + {"role": "user", "content": [{"type": "text", "text": "Do the task."}]}, + { + "role": "user", + "content": [ + { + "type": "text", + "text": """\ +Your previous tool-call arguments were invalid JSON. Retry the same tool call with valid JSON arguments. +Malformed payload (wrapped): {"INVALID_JSON": "{\\"command\\": ls}"}\ +""", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + ], + "steps": ["task", "user", "system", "task"], + } + ), + ), +} + + +@pytest.mark.parametrize(("turns", "expected"), STREAMS.values(), ids=STREAMS.keys()) +async def test_claude_retries_what_the_stream_lets_it_retry( + turns: list[Turn], expected: Any, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(sequence(*turns)) + + run = await run_task( + workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")) + ) + + requests = models.requests() + observed = { + "status": run.trace.status, + "content": run.trace.content, + "error": run.trace.error, + "requests": len(requests), + "last_messages": wire(requests[-1].body["messages"]), + "steps": [step["source"] for step in steps(run.trace_id)], + } + assert observed == expected + + +async def test_a_bedrock_inference_profile_is_invoked_on_bedrock_without_streaming( + services: FakeServices, hud_env: HudEnv, tmp_path: Path +) -> None: + pytest.importorskip("botocore", reason="needs the bedrock extra") + profile = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/claude" + hud_env.set( + AWS_ACCESS_KEY_ID="AKIA", + AWS_SECRET_ACCESS_KEY="secret", + AWS_REGION="us-east-1", + ANTHROPIC_BEDROCK_BASE_URL=services.url("gateway"), + ) + services.route( + "gateway", + "POST", + "/model/{model:path}", + json={ + "id": "msg_bedrock", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [{"type": "text", "text": "from bedrock"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + run = await run_task(workspace_env(tmp_path / "ws"), ClaudeAgent(ClaudeConfig(model=profile))) + + (request,) = services.requests("gateway", "POST") + assert run.trace.content == "from bedrock" + assert (request.path, request.headers["authorization"].split(" ")[0]) == ( + "/model/arn:aws:bedrock:us-east-1:123456789012:inference-profile%2Fclaude/invoke", + "AWS4-HMAC-SHA256", + ) + body = request.json + assert (body["anthropic_version"], "model" in body, "stream" in body) == ( + "bedrock-2023-05-31", + False, + False, + ) + assert body["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Do the task.", "cache_control": {"type": "ephemeral"}} + ], + } + ] + + +def test_a_bedrock_inference_profile_cannot_use_the_gateway() -> None: + profile = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/claude" + + with pytest.raises(ValueError, match="is a Bedrock inference profile; it cannot use the HUD"): + ClaudeAgent(ClaudeConfig(model=profile, gateway=True)) From 7c502592c61b77050a28a07519841439d63a9dd9 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 044/248] test(agents): pin Chat Completions training options and citation requests The Chat Completions training path (return_token_ids turning on logprobs and the KV-cache continuation, multimodal prompts resetting it, the checkpoint in extra_body, reserved completion kwargs dropped) and the citation request parameters for OpenAI and Gemini had no test that read the request a provider received. --- tests/agents/test_provider_requests.py | 144 ++++++++++++++++++++++++- 1 file changed, 142 insertions(+), 2 deletions(-) diff --git a/tests/agents/test_provider_requests.py b/tests/agents/test_provider_requests.py index d449c9915..83c3c4cf0 100644 --- a/tests/agents/test_provider_requests.py +++ b/tests/agents/test_provider_requests.py @@ -26,14 +26,14 @@ from hud.agents.openai.tools import OpenAICodeInterpreterTool, OpenAIToolSearchTool from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig from tests.agents.support import image, mcp_server, run_task, wire, workspace_env -from tests.harness import call, say, shell_call, steps +from tests.harness import Turn, call, say, shell_call, steps if TYPE_CHECKING: from collections.abc import Callable from pathlib import Path from hud.agents.base import Agent - from tests.harness import HudEnv, Models, Turn + from tests.harness import HudEnv, ModelRequest, Models @dataclass(frozen=True) @@ -1053,3 +1053,143 @@ async def test_an_invalid_hosted_tool_fails_the_run_before_any_request( assert (run.trace.status, run.trace.error) == ("error", error) assert models.requests() == [] + + +def request_options(request: ModelRequest) -> dict[str, Any]: + """A Chat Completions request without its conversation and tool list.""" + return {key: value for key, value in request.body.items() if key not in {"messages", "tools"}} + + +TRAINING = { + "token-ids-continue-the-kv-cache": ( + OpenAIChatConfig( + model="qwen3.6-plus", + completion_kwargs={"extra_body": {"return_token_ids": True}, "temperature": 0.2}, + ), + [ + Turn( + tool_calls=call("bash", command="true").tool_calls, + extra={"prompt_token_ids": [1, 2], "token_ids": [3]}, + ), + Turn(text="done", extra={"prompt_token_ids": [1, 2, 3, 4], "token_ids": [5]}), + ], + snapshot( + [ + { + "model": "qwen3.6-plus", + "logprobs": True, + "stream": False, + "temperature": 0.2, + "return_token_ids": True, + }, + { + "model": "qwen3.6-plus", + "logprobs": True, + "stream": False, + "temperature": 0.2, + "return_token_ids": True, + "prompt_token_ids": [1, 2, 3], + "continuation_from": 2, + }, + ] + ), + ), + "multimodal-prompt-chunks-reset-the-continuation": ( + OpenAIChatConfig( + model="qwen3.6-plus", completion_kwargs={"extra_body": {"return_token_ids": True}} + ), + [ + Turn( + tool_calls=call("bash", command="true").tool_calls, + extra={"prompt_chunks": [{"type": "text", "text": "hi"}], "token_ids": [3]}, + ), + say("done"), + ], + snapshot( + [ + { + "model": "qwen3.6-plus", + "logprobs": True, + "stream": False, + "return_token_ids": True, + }, + { + "model": "qwen3.6-plus", + "logprobs": True, + "stream": False, + "return_token_ids": True, + }, + ] + ), + ), + "checkpoint-routes-in-the-body": ( + OpenAIChatConfig(model="qwen3.6-plus", checkpoint="ckpt-7"), + [say("done")], + snapshot([{"model": "qwen3.6-plus", "stream": False, "checkpoint": "ckpt-7"}]), + ), + "reserved-completion-kwargs-are-dropped": ( + OpenAIChatConfig( + model="qwen3.6-plus", + completion_kwargs={ + "model": "other", + "stream": True, + "tools": [], + "messages": [], + "max_tokens": 50, + }, + ), + [say("done")], + snapshot([{"model": "qwen3.6-plus", "max_tokens": 50, "stream": False}]), + ), +} + + +@pytest.mark.parametrize(("config", "turns", "expected"), TRAINING.values(), ids=TRAINING.keys()) +async def test_chat_completion_options_reach_every_request( + config: OpenAIChatConfig, + turns: list[Turn], + expected: Any, + models: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script(turns) + + await run_task(workspace_env(tmp_path / "ws"), OpenAIChatAgent(config)) + + assert [request_options(request) for request in models.requests()] == expected + + +CITATIONS = { + "openai": ( + lambda: OpenAIAgent(OpenAIConfig(model="gpt-5.6", citations_enabled=True)), + lambda request: request.body.get("include"), + snapshot(["web_search_call.action.sources"]), + ), + "gemini": ( + lambda: GeminiAgent(GeminiConfig(model="gemini-3.1-pro-preview", citations_enabled=True)), + lambda request: [ + tool for tool in request.body["tools"] if "functionDeclarations" not in tool + ], + snapshot([{"googleSearch": {}}]), + ), +} + + +@pytest.mark.parametrize(("agent", "part", "expected"), CITATIONS.values(), ids=CITATIONS.keys()) +async def test_citations_ask_the_provider_for_sources( + agent: Callable[[], Agent], + part: Callable[[ModelRequest], Any], + expected: Any, + models: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([say("done")]) + + await run_task(workspace_env(tmp_path / "ws"), agent()) + + (request,) = models.requests() + assert part(request) == expected From 227f4085bbacfd8ee0a450ff38b2ec4bc2051dcd Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 045/248] test(agents): pin which endpoint and credential each agent's requests use Routing was tested by patching hud.utils.gateway and hud.agents.settings. Agents now run one turn against the fake gateway with HUD and provider keys set the way a user sets them; rows pin the credential the provider received, whether the trace id rode along, and the model requested, for create_agent shortcuts and catalog names (served paginated), direct construction with and without provider keys, explicit chat credentials, and a dumped gateway agent loaded back. Construction errors, dump/load round trips and the missing-SDK hint (in a subprocess with anthropic blocked) are rows too. --- tests/agents/test_gateway_routing.py | 359 +++++++++++++++++++++++++++ 1 file changed, 359 insertions(+) create mode 100644 tests/agents/test_gateway_routing.py diff --git a/tests/agents/test_gateway_routing.py b/tests/agents/test_gateway_routing.py new file mode 100644 index 000000000..6dd433d66 --- /dev/null +++ b/tests/agents/test_gateway_routing.py @@ -0,0 +1,359 @@ +"""Which endpoint and credential an agent's model requests use. + +Agents built with ``create_agent`` route through the HUD gateway; agents built +directly use a provider's own key when one is configured and the gateway +otherwise. Rows build an agent the way a user does, run one turn, and pin the +credential the provider received, whether the run's trace id rode along, and +the model it was asked for. +""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +from typing import TYPE_CHECKING, Any + +import pytest +from inline_snapshot import snapshot + +from hud.agents import ( + ClaudeAgent, + ClaudeCLIAgent, + CodexCLIAgent, + GeminiAgent, + OpenAIAgent, + OpenAIChatAgent, + create_agent, +) +from hud.agents.base import Agent +from hud.agents.types import ( + AgentConfig, + ClaudeCLIConfig, + ClaudeConfig, + CodexCLIConfig, + OpenAIChatConfig, + OpenAIConfig, +) +from hud.utils.exceptions import HudAuthenticationError +from tests.agents.support import run_task, workspace_env +from tests.harness import Reply, say + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hud.eval import Run + from tests.harness import FakeServices, HudEnv, Models, Request + +CATALOG = [ + { + "id": "ft:custom-123", + "model_name": "gpt-5.5", + "sdk_agent_type": "openai_compatible", + "provider": {"name": "openai"}, + }, + { + "id": "deepseek/deepseek-v4-flash", + "model_name": "deepseek/deepseek-v4-flash", + "sdk_agent_type": "openai_compatible", + }, + {"id": "MiniMax-M3", "model_name": "MiniMax-M3", "sdk_agent_type": "openai_compatible"}, + {"id": "claude-sonnet-4-6", "name": "Claude Sonnet 4.6", "sdk_agent_type": "claude"}, +] + + +def serve_catalog(services: FakeServices) -> None: + """The gateway model catalog, served two entries a page.""" + + def page(request: Request) -> Reply: + offset = int(request.query.get("offset", ["0"])[0]) + return Reply(json={"items": CATALOG[offset : offset + 2], "total": len(CATALOG)}) + + services.route("api", "GET", "/v2/models", handler=page) + + +ROUTES = { + "create-agent-shortcut": ( + lambda _url: create_agent("openai"), + snapshot( + { + "protocol": "responses", + "model": "gpt-5.6", + "credential": "Bearer hud-key", + "trace_id": True, + } + ), + ), + "create-agent-ignores-a-provider-key": ( + lambda _url: create_agent("openai_compatible"), + snapshot( + { + "protocol": "chat", + "model": "gpt-5.4-mini", + "credential": "Bearer hud-key", + "trace_id": True, + } + ), + ), + "create-agent-claude": ( + lambda _url: create_agent("claude"), + snapshot( + { + "protocol": "anthropic", + "model": "claude-sonnet-4-6", + "credential": "hud-key", + "trace_id": True, + } + ), + ), + "create-agent-gemini": ( + lambda _url: create_agent("gemini"), + snapshot( + { + "protocol": "gemini", + "model": "gemini-3.1-pro-preview", + "credential": "hud-key", + "trace_id": True, + } + ), + ), + "create-agent-catalog-id": ( + lambda _url: create_agent("ft:custom-123"), + snapshot( + { + "protocol": "chat", + "model": "gpt-5.5", + "credential": "Bearer hud-key", + "trace_id": True, + } + ), + ), + "create-agent-catalog-slug-without-provider": ( + lambda _url: create_agent("deepseek-v4-flash"), + snapshot( + { + "protocol": "chat", + "model": "deepseek/deepseek-v4-flash", + "credential": "Bearer hud-key", + "trace_id": True, + } + ), + ), + "create-agent-catalog-name-any-case": ( + lambda _url: create_agent("minimax-m3"), + snapshot( + { + "protocol": "chat", + "model": "MiniMax-M3", + "credential": "Bearer hud-key", + "trace_id": True, + } + ), + ), + "reloaded-gateway-agent": ( + lambda _url: OpenAIAgent.load(create_agent("openai").dump()), + snapshot( + { + "protocol": "responses", + "model": "gpt-5.6", + "credential": "Bearer hud-key", + "trace_id": True, + } + ), + ), + "direct-agent-with-a-provider-key": ( + lambda _url: OpenAIAgent(OpenAIConfig()), + snapshot( + { + "protocol": "responses", + "model": "gpt-5.6", + "credential": "Bearer openai-key", + "trace_id": False, + } + ), + ), + "direct-claude-with-a-provider-key": ( + lambda _url: ClaudeAgent(ClaudeConfig()), + snapshot( + { + "protocol": "anthropic", + "model": "claude-sonnet-4-6", + "credential": "anthropic-key", + "trace_id": False, + } + ), + ), + "direct-gemini-without-a-provider-key": ( + lambda _url: GeminiAgent(), + snapshot( + { + "protocol": "gemini", + "model": "gemini-3.1-pro-preview", + "credential": "hud-key", + "trace_id": True, + } + ), + ), +} + + +@pytest.mark.parametrize(("build", "expected"), ROUTES.values(), ids=ROUTES.keys()) +async def test_an_agent_reaches_its_model_with_the_credential_its_routing_chose( + build: Callable[[str], Agent], + expected: Any, + services: FakeServices, + models: Models, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set( + HUD_API_KEY="hud-key", + OPENAI_API_KEY="openai-key", + ANTHROPIC_API_KEY="anthropic-key", + OPENAI_BASE_URL=models.url, + ANTHROPIC_BASE_URL=models.url, + ) + serve_catalog(services) + models.script([say("done")]) + + run = await run_task(workspace_env(tmp_path / "ws"), build(models.url)) + + (request,) = models.requests() + received = { + "protocol": request.protocol, + "model": request.model, + "credential": request.headers.get("authorization") + or request.headers.get("x-api-key") + or request.headers.get("x-goog-api-key"), + "trace_id": request.headers.get("trace-id") == run.trace_id, + } + assert received == expected + + +async def test_explicit_chat_credentials_reach_the_given_endpoint( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + models.script([say("done")]) + agent = OpenAIChatAgent(OpenAIChatConfig(api_key="explicit", base_url=models.url)) + + await run_task(workspace_env(tmp_path / "ws"), agent) + + (request,) = models.requests() + assert (request.headers["authorization"], "trace-id" in request.headers) == ( + "Bearer explicit", + False, + ) + + +def test_create_agent_needs_a_hud_api_key() -> None: + with pytest.raises(HudAuthenticationError, match="HUD_API_KEY is required"): + create_agent("openai") + + +@pytest.mark.parametrize("hud_api_key", ["hud-key", None]) +@pytest.mark.parametrize( + "credentials", + [{"api_key": "provider-key"}, {"base_url": "https://provider.example"}, {"api_key": None}], +) +def test_create_agent_refuses_provider_credentials( + credentials: dict[str, Any], hud_api_key: str | None, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY=hud_api_key) + + with pytest.raises(ValueError, match="instantiate the provider agent directly"): + create_agent("openai_compatible", **credentials) + + +def test_create_agent_names_close_matches_for_an_unknown_model( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="hud-key") + serve_catalog(services) + + with pytest.raises(ValueError, match=r"'minimax-m4' not found .* Did you mean: MiniMax-M3"): + create_agent("minimax-m4") + + +def test_a_chat_agent_refuses_a_foreign_key_for_the_hud_gateway( + services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(HUD_API_KEY="hud-key") + + with pytest.raises(ValueError, match="api_key is not allowed with HUD Gateway"): + OpenAIChatAgent(OpenAIChatConfig(api_key="other", base_url=services.url("gateway"))) + + +AGENTS_WITHOUT_CREDENTIALS = { + "chat": (lambda: OpenAIChatAgent(), ValueError, "No API key found"), + "claude": (lambda: ClaudeAgent(), HudAuthenticationError, "No API key for anthropic"), + "openai": (lambda: OpenAIAgent(), HudAuthenticationError, "No API key for openai"), + "gemini": (lambda: GeminiAgent(), HudAuthenticationError, "No API key for gemini"), +} + + +@pytest.mark.parametrize( + ("build", "error", "message"), + AGENTS_WITHOUT_CREDENTIALS.values(), + ids=AGENTS_WITHOUT_CREDENTIALS.keys(), +) +def test_an_agent_without_any_credential_fails_at_construction( + build: Callable[[], Agent], error: type[Exception], message: str +) -> None: + with pytest.raises(error, match=message): + build() + + +def test_an_agent_with_a_custom_model_client_cannot_be_dumped() -> None: + agent = OpenAIAgent(OpenAIConfig(model_client=object())) + + with pytest.raises(ValueError, match="a custom model_client cannot be serialized"): + agent.dump() + + +class Answering(Agent): + async def __call__(self, run: Run) -> None: + run.trace.content = "done" + + +DUMPED = { + "claude-cli": ClaudeCLIAgent(ClaudeCLIConfig(reasoning_effort="max", max_steps=3)), + "codex-cli": CodexCLIAgent(CodexCLIConfig(sandbox="read-only")), + "custom": Answering(AgentConfig(model="custom", timeout_seconds=30)), +} + + +@pytest.mark.parametrize("agent", DUMPED.values(), ids=DUMPED.keys()) +def test_a_dumped_agent_loads_back_to_the_same_configuration(agent: Agent) -> None: + loaded = type(agent).load(agent.dump()) + + assert (type(loaded), loaded.dump()) == (type(agent), agent.dump()) + + +def test_a_provider_agent_without_its_sdk_points_at_the_agents_extra(tmp_path: Path) -> None: + script = textwrap.dedent( + """ + import sys + + class Blocker: + def find_spec(self, name, path=None, target=None): + if name == "anthropic" or name.startswith("anthropic."): + raise ModuleNotFoundError(f"No module named {name!r}", name=name) + + sys.meta_path.insert(0, Blocker()) + import hud.agents + + try: + hud.agents.ClaudeAgent + except ImportError as exc: + print(exc) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=True, cwd=tmp_path + ) + + assert result.stdout.strip() == ( + "ClaudeAgent requires the agents extra. Install with: pip install 'hud[agents]'" + ) From 6450fc1659b5598ab7ff2bee2b202ef2bf55c9b1 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 046/248] fix(agents): request a catalog model by its id, not the name typed create_agent resolves a model by id, display name or slug, then asked the gateway for entry.model_name, falling back to whatever the caller typed. For a catalog entry without a model_name, create_agent("Claude Sonnet 4.6") sent the display name as the model and the gateway could not route it. The fallback is now the entry's id, the name the catalog knows it by. --- hud/agents/__init__.py | 2 +- tests/agents/test_gateway_routing.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 5f4d821ae..90077770d 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -51,7 +51,7 @@ def create_agent(model: str, **kwargs: Any) -> GatewayAgent: else: entry = resolve_gateway_model(model) agent_type = AgentType(entry.sdk_agent_type) - kwargs.setdefault("model", entry.model_name or model) + kwargs.setdefault("model", entry.model_name or entry.id) kwargs["gateway"] = True return cast("GatewayAgent", agent_type.cls.load(kwargs)) diff --git a/tests/agents/test_gateway_routing.py b/tests/agents/test_gateway_routing.py index 6dd433d66..ea8c6da35 100644 --- a/tests/agents/test_gateway_routing.py +++ b/tests/agents/test_gateway_routing.py @@ -151,6 +151,17 @@ def page(request: Request) -> Reply: } ), ), + "create-agent-catalog-display-name": ( + lambda _url: create_agent("Claude Sonnet 4.6"), + snapshot( + { + "protocol": "anthropic", + "model": "claude-sonnet-4-6", + "credential": "hud-key", + "trace_id": True, + } + ), + ), "reloaded-gateway-agent": ( lambda _url: OpenAIAgent.load(create_agent("openai").dump()), snapshot( From 42ecea4534cb053aef0a6444bf18596f95711ac3 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 047/248] test(agents): pin how MCP server tools are offered to and reach each provider MCP tool qualification was tested through _build_tools with stub MCP clients, and the strict-schema conversion through its private helper. Environments now serve real MCP servers over streamable HTTP and SSE; rows pin the tool definitions OpenAI (strict schemas), Claude and Chat Completions receive, that each qualified call reaches the server that owns it carrying the run's trace id, that one server keeps names unqualified, that long or unusual server names give valid names, and that a clash after qualification fails the run before any request. --- tests/agents/test_mcp_tools.py | 375 +++++++++++++++++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 tests/agents/test_mcp_tools.py diff --git a/tests/agents/test_mcp_tools.py b/tests/agents/test_mcp_tools.py new file mode 100644 index 000000000..ec6ab80c7 --- /dev/null +++ b/tests/agents/test_mcp_tools.py @@ -0,0 +1,375 @@ +"""Tools from the environment's MCP servers, as each provider agent offers and calls them. + +The environment serves MCP servers over HTTP. Rows pin the tool list each +provider received (names qualified by server once there are several, within +64 characters, OpenAI schemas in strict form), that every call reaches the +server that owns the tool with the run's trace id, and that a name clash +fails the run. +""" + +from __future__ import annotations + +from contextlib import AsyncExitStack +from typing import TYPE_CHECKING, Any, Literal + +import pytest +from dirty_equals import IsStr +from fastmcp import FastMCP +from inline_snapshot import snapshot +from pydantic import BaseModel, Field + +from hud.agents import ClaudeAgent, OpenAIAgent, OpenAIChatAgent +from hud.agents.types import ClaudeConfig, OpenAIChatConfig, OpenAIConfig +from hud.capabilities.mcp import get_mcp_trace_id +from tests.agents.support import mcp_server, run_task, wire, workspace_env +from tests.harness import call, say + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hud.agents.base import Agent + from hud.capabilities import Capability + from tests.harness import HudEnv, Models + + +class Address(BaseModel): + street: str = Field(title="Street", min_length=1) + + +def database() -> FastMCP: + server = FastMCP("database") + + @server.tool() + def lookup(key: str) -> str: + """Look a key up in the database.""" + return f"database:{key}" + + @server.tool() + def trace() -> str: + """Report the trace this call belongs to.""" + return str(get_mcp_trace_id()) + + @server.tool() + def register( + name: str = Field(title="Name", min_length=2), address: Address | None = None + ) -> str: + """Register a person.""" + return f"registered {name}" + + return server + + +def search() -> FastMCP: + server = FastMCP("search") + + @server.tool() + def lookup(key: str) -> str: + """Look a key up in the search index.""" + return f"search:{key}" + + return server + + +def named(*tools: str) -> Callable[[], FastMCP]: + def build() -> FastMCP: + server = FastMCP("named") + for tool in tools: + server.tool(name=tool, description=f"Run {tool}.")(lambda: tool) + return server + + return build + + +async def serve( + stack: AsyncExitStack, + servers: dict[str, Callable[[], FastMCP]], + transport: Literal["streamable-http", "sse"] = "streamable-http", +) -> tuple[Capability, ...]: + return tuple( + [ + await stack.enter_async_context(mcp_server(build(), name=name, transport=transport)) + for name, build in servers.items() + ] + ) + + +def tool_name(tool: dict[str, Any]) -> str: + return tool.get("name") or tool.get("function", {}).get("name", "") + + +AGENTS: dict[str, Callable[[], Agent]] = { + "openai": lambda: OpenAIAgent(OpenAIConfig(model="gpt-5.6")), + "claude": lambda: ClaudeAgent(ClaudeConfig(model="claude-sonnet-4-6")), + "chat": lambda: OpenAIChatAgent(OpenAIChatConfig(model="qwen3.6-plus")), +} + +OFFERED = { + "openai": snapshot( + [ + { + "type": "function", + "name": "database__lookup", + "description": "Look a key up in the database.", + "parameters": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "strict": True, + }, + { + "type": "function", + "name": "database__trace", + "description": "Report the trace this call belongs to.", + "parameters": { + "additionalProperties": False, + "properties": {}, + "type": "object", + "required": [], + }, + "strict": True, + }, + { + "type": "function", + "name": "database__register", + "description": "Register a person.", + "parameters": { + "additionalProperties": False, + "properties": { + "name": {"type": "string"}, + "address": { + "anyOf": [ + { + "properties": {"street": {"type": "string"}}, + "required": ["street"], + "type": "object", + "additionalProperties": False, + }, + {"type": "null"}, + ] + }, + }, + "required": ["name", "address"], + "type": "object", + }, + "strict": True, + }, + { + "type": "function", + "name": "search__lookup", + "description": "Look a key up in the search index.", + "parameters": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "strict": True, + }, + ] + ), + "claude": snapshot( + [ + { + "name": "database__lookup", + "description": "Look a key up in the database.", + "input_schema": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "eager_input_streaming": True, + }, + { + "name": "database__trace", + "description": "Report the trace this call belongs to.", + "input_schema": {"additionalProperties": False, "properties": {}, "type": "object"}, + "eager_input_streaming": True, + }, + { + "name": "database__register", + "description": "Register a person.", + "input_schema": { + "additionalProperties": False, + "properties": { + "name": {"minLength": 2, "type": "string"}, + "address": { + "anyOf": [ + { + "properties": {"street": {"minLength": 1, "type": "string"}}, + "required": ["street"], + "type": "object", + }, + {"type": "null"}, + ], + "default": None, + }, + }, + "required": ["name"], + "type": "object", + }, + "eager_input_streaming": True, + }, + { + "name": "search__lookup", + "description": "Look a key up in the search index.", + "input_schema": { + "additionalProperties": False, + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + "eager_input_streaming": True, + }, + ] + ), + "chat": snapshot( + [ + { + "type": "function", + "function": { + "name": "database__lookup", + "description": "Look a key up in the database.", + "parameters": { + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + }, + }, + { + "type": "function", + "function": { + "name": "database__trace", + "description": "Report the trace this call belongs to.", + "parameters": {"properties": {}, "type": "object"}, + }, + }, + { + "type": "function", + "function": { + "name": "database__register", + "description": "Register a person.", + "parameters": { + "properties": { + "name": {"type": "string"}, + "address": { + "properties": {"street": {"type": "string"}}, + "required": ["street"], + "type": "object", + "default": None, + }, + }, + "required": ["name"], + "type": "object", + }, + }, + }, + { + "type": "function", + "function": { + "name": "search__lookup", + "description": "Look a key up in the search index.", + "parameters": { + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "type": "object", + }, + }, + }, + ] + ), +} + + +@pytest.mark.parametrize("provider", AGENTS.keys()) +async def test_tools_from_two_servers_are_offered_qualified_and_reach_their_server( + provider: str, models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script( + [ + call("database__lookup", key="k1"), + call("search__lookup", key="k2"), + call("database__trace"), + say("done"), + ] + ) + + async with AsyncExitStack() as stack: + tools = await serve(stack, {"database": database, "search": search}) + run = await run_task(workspace_env(tmp_path / "ws", capabilities=tools), AGENTS[provider]()) + + requests = models.requests() + first = requests[0] + mcp_tools = [ + tool for tool in wire(first.body["tools"], descriptions=True) if "__" in tool_name(tool) + ] + assert mcp_tools == OFFERED[provider] + answers = [text for request in requests for text in request.tool_results] + if provider != "openai": + answers = requests[-1].tool_results + assert answers == [ + IsStr(regex=f".*{expected}") for expected in ("database:k1", "search:k2", run.trace_id) + ] + + +async def test_a_single_server_offers_its_tools_unqualified( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([call("lookup", key="k1"), say("done")]) + + async with AsyncExitStack() as stack: + tools = await serve(stack, {"search": search}, transport="sse") + await run_task(workspace_env(tmp_path / "ws", capabilities=tools), AGENTS["chat"]()) + + first, last = models.requests() + assert ("lookup" in first.tools, last.tool_results) == (True, ["search:k1"]) + + +async def test_long_and_unusual_server_names_still_give_valid_tool_names( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + models.script([say("done")]) + long_name = "lookup_" * 10 + + async with AsyncExitStack() as stack: + tools = await serve( + stack, + { + "company.tools": named(long_name + "a"), + "company-tools": named(long_name + "b"), + }, + ) + await run_task(workspace_env(tmp_path / "ws", capabilities=tools), AGENTS["openai"]()) + + (request,) = models.requests() + names = [name for name in request.tools if name != "shell"] + assert names == snapshot( + [ + "company_tools__lookup_lookup_lookup_lookup_lookup_looku_525f784f", + "company-tools__lookup_lookup_lookup_lookup_lookup_looku_442a1793", + ] + ) + assert all(len(name) <= 64 for name in names) + + +async def test_tool_names_that_clash_after_qualification_fail_the_run( + models: Models, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="k") + + async with AsyncExitStack() as stack: + tools = await serve(stack, {"a": named("b__c"), "a__b": named("c")}) + run = await run_task(workspace_env(tmp_path / "ws", capabilities=tools), AGENTS["openai"]()) + + assert (run.trace.status, run.trace.error, models.requests()) == ( + "error", + "[agent loop] ValueError: MCP tool name collision after qualification: 'a__b__c'", + [], + ) From 1afd5f310adfde52f668b10beaff09927e56bbfd Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 048/248] test(agents): run the Claude and Codex CLI agents against stub CLIs The CLI agents were tested with a SimpleNamespace run, a scripted SSH connection that regex-parsed commands, and resolve_executable patched out. They now run real rollouts over a served workspace with stub claude and codex executables first on the session PATH. Each stub records its argv, the variables the agent set, stdin and the .hud_* files beside it, then prints a canned JSONL stream and exits as told. Rows pin the invocation per routing (provider key, gateway, forced gateway, ambient Codex login) and option, the steps and trace extras the streams become, how each exit ends the run, MCP config files and their removal, the computer-use bridge driving a fake screen end to end, a missing CLI, concurrent rollouts on one agent, and an agent timeout stopping the CLI process. --- tests/agents/cli_stub.py | 179 +++++ tests/agents/test_cli_agents.py | 1104 +++++++++++++++++++++++++++++++ 2 files changed, 1283 insertions(+) create mode 100644 tests/agents/cli_stub.py create mode 100644 tests/agents/test_cli_agents.py diff --git a/tests/agents/cli_stub.py b/tests/agents/cli_stub.py new file mode 100644 index 000000000..09ad50601 --- /dev/null +++ b/tests/agents/cli_stub.py @@ -0,0 +1,179 @@ +"""Stub ``claude`` and ``codex`` executables for the CLI agent scenarios. + +A stub is a Python script placed on the workspace's ``PATH``. Each invocation +records what the agent handed it (argv, the environment variables the agent +set, stdin, its working directory and the ``.hud_*`` files there) to a capture +file, optionally drives a computer-use MCP server named in its MCP config, +then prints a canned JSONL stream, writes stderr and exits as told. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +#: Variables the CLI agents set for the process they launch. +AGENT_VARIABLES = ("ANTHROPIC_", "CLAUDE_", "CODEX_", "DISABLE_", "HUD_", "IS_SANDBOX") + +STUB = """#!{python} +import json, os, subprocess, sys, time + +behavior = json.load(open({behavior!r})) +record = {{ + "argv": sys.argv[1:], + "env": {{ + key: value + for key, value in os.environ.items() + if key.startswith({prefixes!r}) and value + }}, + "stdin": sys.stdin.read(), + "cwd": os.getcwd(), + "files": {{ + name: open(name).read() for name in sorted(os.listdir(".")) if name.startswith(".hud_") + }}, + "pid": os.getpid(), +}} + + +def drive_computer(server, actions): + relay = subprocess.Popen( + [server["command"], *server["args"]], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + ) + + def ask(message): + relay.stdin.write(json.dumps(message) + "\\n") + relay.stdin.flush() + while True: + reply = json.loads(relay.stdout.readline()) + if reply.get("id") == message["id"]: + return reply + + ask({{"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {{ + "protocolVersion": "2025-06-18", "capabilities": {{}}, + "clientInfo": {{"name": "stub", "version": "0"}}}}}}) + initialized = {{"jsonrpc": "2.0", "method": "notifications/initialized"}} + relay.stdin.write(json.dumps(initialized) + "\\n") + results = [] + for index, arguments in enumerate(actions, start=1): + reply = ask({{"jsonrpc": "2.0", "id": index, "method": "tools/call", + "params": {{"name": "computer", "arguments": arguments}}}}) + result = reply["result"] + results.append({{ + "isError": result.get("isError", False), + "content": [block.get("text") or block["type"] for block in result["content"]], + }}) + # Like the real CLI, end the stdio server rather than wait for it to drain. + relay.terminate() + relay.wait() + return results + + +if behavior.get("computer"): + servers = json.load(open(".hud_mcp_config.json"))["mcpServers"] + record["computer"] = drive_computer(servers["computer-use"], behavior["computer"]) + +with open(os.path.join({captures!r}, str(os.getpid()) + ".json"), "w") as capture: + json.dump(record, capture) + +sys.stdout.write(behavior["stdout"]) +sys.stdout.flush() +sys.stderr.write(behavior.get("stderr", "")) +sys.stderr.flush() +if behavior.get("hang"): + time.sleep(60) +sys.exit(behavior.get("exit", 0)) +""" + + +@dataclass +class Stub: + """A stub CLI installed in ``bin``; ``captures()`` reads what each invocation recorded.""" + + root: Path + name: str + stdout: str = "" + stderr: str = "" + exit: int = 0 + hang: bool = False + computer: list[dict[str, Any]] = field(default_factory=list[dict[str, Any]]) + + @property + def bin(self) -> Path: + return self.root / "bin" + + @property + def captures_dir(self) -> Path: + return self.root / "captures" + + def install(self) -> None: + self.bin.mkdir(parents=True, exist_ok=True) + self.captures_dir.mkdir(parents=True, exist_ok=True) + behavior = self.root / "behavior.json" + behavior.write_text( + json.dumps( + { + "stdout": self.stdout, + "stderr": self.stderr, + "exit": self.exit, + "hang": self.hang, + "computer": self.computer, + } + ) + ) + script = self.bin / self.name + script.write_text( + STUB.format( + python=sys.executable, + behavior=str(behavior), + prefixes=AGENT_VARIABLES, + captures=str(self.captures_dir), + ) + ) + script.chmod(0o755) + + def captures(self) -> list[dict[str, Any]]: + return [json.loads(path.read_text()) for path in sorted(self.captures_dir.glob("*.json"))] + + def shell_env(self, *, path: str | None = None) -> dict[str, str]: + """The workspace shell environment, with every agent variable this process holds + blanked so a capture shows only what the agent set. + + Workspace sessions are login shells, and system profiles prepend their own + directories (where a real ``claude`` may live), so the session's home gets a + profile that puts ``path`` (default: the stub, then the system) on ``PATH`` last. + """ + home = self.root / "home" + home.mkdir(parents=True, exist_ok=True) + search = path or f"{self.bin}:/usr/local/bin:/usr/bin:/bin" + (home / ".bash_profile").write_text(f"export PATH={search}\n") + blanked = {key: "" for key in os.environ if key.startswith(AGENT_VARIABLES)} + return {**blanked, "HOME": str(home)} + + +def process_exited(pid: int, *, within: float = 5.0) -> bool: + """Whether ``pid`` has exited (gone or a zombie), polling for up to ``within`` seconds.""" + status = Path(f"/proc/{pid}/status") + deadline = time.monotonic() + within + while time.monotonic() < deadline: + try: + state = next( + line for line in status.read_text().splitlines() if line.startswith("State:") + ) + except FileNotFoundError: + return True + if "Z" in state.split()[1]: + return True + time.sleep(0.05) + return False + + +def jsonl(*events: dict[str, Any]) -> str: + return "".join(json.dumps(event) + "\n" for event in events) diff --git a/tests/agents/test_cli_agents.py b/tests/agents/test_cli_agents.py new file mode 100644 index 000000000..c9c722916 --- /dev/null +++ b/tests/agents/test_cli_agents.py @@ -0,0 +1,1104 @@ +# ruff: noqa: E501 -- snapshots quote product messages verbatim +"""The Claude and Codex CLI agents run their CLI in the workspace over SSH. + +A stub ``claude`` or ``codex`` on the workspace's PATH records what the agent +handed it (argv, the variables it set, stdin, the ``.hud_*`` files beside it), +prints a canned JSONL stream and exits as told. Rows pin that invocation, the +steps and trace the stream turned into, and how each exit ends the run. +""" + +from __future__ import annotations + +import asyncio +import uuid +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import anyio +import pytest +from dirty_equals import IsStr +from inline_snapshot import snapshot + +from hud.agents import ClaudeCLIAgent, CodexCLIAgent +from hud.agents.types import ClaudeCLIConfig, CodexCLIConfig +from hud.capabilities import Capability +from tests.agents.cli_stub import Stub, jsonl, process_exited +from tests.agents.support import run_task, workspace_env +from tests.harness import fake_screen, steps + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from hud import Environment + from hud.agents.base import Agent + from hud.eval import Run + from tests.harness import HudEnv + +TIMESTAMP = IsStr(regex=r"\d{4}-\d{2}-\d{2}T.*") + +# ─── canned Claude CLI streams ────────────────────────────────────────── + + +def assistant(*content: dict[str, Any], stop: str = "end_turn") -> dict[str, Any]: + return { + "type": "assistant", + "message": { + "id": "msg", + "type": "message", + "role": "assistant", + "model": "claude-test", + "content": list(content), + "stop_reason": stop, + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 2}, + }, + } + + +def tool_result(tool_use_id: str, *content: Any, is_error: bool = False) -> dict[str, Any]: + return { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": list(content), + "is_error": is_error, + } + ] + }, + } + + +def result(text: str = "done", *, is_error: bool = False) -> dict[str, Any]: + return { + "type": "result", + "subtype": "success", + "is_error": is_error, + "result": text, + "session_id": "session-1", + "duration_ms": 5, + "num_turns": 2, + "total_cost_usd": 0.01, + } + + +WRITE = assistant( + {"type": "text", "text": "editing"}, + {"type": "tool_use", "id": "tool-1", "name": "Write", "input": {"file_path": "a.txt"}}, + stop="tool_use", +) +WROTE = tool_result( + "tool-1", + {"type": "text", "text": "wrote a.txt"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aW1n"}}, +) +DONE = assistant({"type": "text", "text": "done"}) +CLAUDE_STREAM = jsonl(WRITE, WROTE, DONE, result()) + + +def claude_stub(root: Path, **behavior: Any) -> Stub: + return Stub(root / "stub", "claude", **{"stdout": CLAUDE_STREAM, **behavior}) + + +def codex_stub(root: Path, **behavior: Any) -> Stub: + return Stub(root / "stub", "codex", **{"stdout": CODEX_STREAM, **behavior}) + + +def cli_env(root: Path, stub: Stub, **options: Any) -> Environment: + stub.install() + return workspace_env(root / "ws", env=stub.shell_env(), **options) + + +def step_payloads(run: Run) -> list[dict[str, Any]]: + """The agent and tool steps the run exported, timestamps checked and dropped.""" + found = [] + for payload in steps(run.trace_id): + if payload["source"] in {"task", "user"}: + continue + for key in ("started_at", "ended_at"): + if key in payload: + assert payload.pop(key) == TIMESTAMP + found.append(payload) + return found + + +def invocation(capture: dict[str, Any], run: Run) -> dict[str, Any]: + """The arguments (joined) and variables the agent launched its CLI with, the run's trace id as + ```` and a temporary ``CODEX_HOME`` as ````.""" + trace = str(run.trace_id) + traces = (trace, uuid.UUID(trace).hex) + + def scrub(value: str) -> str: + for form in traces: + value = value.replace(form, "") + return value + + env = {key: scrub(value) for key, value in capture["env"].items()} + if "CODEX_HOME" in env: + assert env["CODEX_HOME"] == IsStr(regex=r".*/hud-codex\.\w+") + env["CODEX_HOME"] = "" + return {"argv": " ".join(scrub(arg) for arg in capture["argv"]), "env": env} + + +def outcome(run: Run) -> dict[str, Any]: + return { + "status": run.trace.status, + "content": run.trace.content, + "error": run.trace.error, + "extra": run.trace.extra, + } + + +# ─── Claude CLI ───────────────────────────────────────────────────────── + + +async def test_the_claude_cli_streams_its_turns_into_steps(hud_env: HudEnv, tmp_path: Path) -> None: + hud_env.set(ANTHROPIC_API_KEY="anthropic-key") + stub = claude_stub(tmp_path) + + run = await run_task(cli_env(tmp_path, stub), ClaudeCLIAgent(), prompt="build it") + + (capture,) = stub.captures() + assert outcome(run) == snapshot( + { + "status": "completed", + "content": "done", + "error": None, + "extra": { + "subtype": "success", + "session_id": "session-1", + "duration_ms": 5, + "num_turns": 2, + "total_cost_usd": 0.01, + }, + } + ) + assert step_payloads(run) == snapshot( + [ + { + "step_id": 3, + "source": "agent", + "messages": [], + "extra": {}, + "content": "editing", + "tool_calls": [ + { + "meta": {"citations_enabled": False}, + "name": "Write", + "arguments": {"file_path": "a.txt"}, + "id": "tool-1", + } + ], + "done": False, + "finish_reason": "tool_use", + "citations": [], + "model": "claude-test", + "usage": {"prompt_tokens": 3, "completion_tokens": 2}, + }, + { + "step_id": 4, + "source": "tool", + "messages": [], + "extra": {}, + "call": { + "meta": {"citations_enabled": False}, + "name": "Write", + "arguments": {"file_path": "a.txt"}, + "id": "tool-1", + }, + "result": { + "content": [ + {"type": "text", "text": "wrote a.txt"}, + {"type": "image", "data": "aW1n", "mimeType": "image/png"}, + ], + "isError": False, + "call_id": "tool-1", + }, + }, + { + "step_id": 5, + "source": "agent", + "messages": [], + "extra": {}, + "content": "done", + "tool_calls": [], + "done": True, + "finish_reason": "end_turn", + "citations": [], + "model": "claude-test", + "usage": {"prompt_tokens": 3, "completion_tokens": 2}, + }, + ] + ) + assert (capture["stdin"], capture["files"]) == snapshot( + ( + '{"type": "user", "message": {"role": "user", "content": [{"type": "text", "text": "build it"}]}}\n', + {}, + ) + ) + + +CLAUDE_INVOCATIONS = { + "provider-key": ( + {"ANTHROPIC_API_KEY": "anthropic-key"}, + ClaudeCLIConfig(), + snapshot( + { + "argv": "--verbose --input-format=stream-json --output-format=stream-json --print --permission-mode=bypassPermissions --allowedTools Read --allowedTools Write --allowedTools Edit --allowedTools Bash --allowedTools Glob --allowedTools Grep --allowedTools WebSearch --allowedTools WebFetch", + "env": { + "IS_SANDBOX": "1", + "ANTHROPIC_MODEL": "claude-sonnet-5", + "ANTHROPIC_API_KEY": "anthropic-key", + "DISABLE_AUTOUPDATER": "1", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + "ANTHROPIC_SMALL_FAST_MODEL": "claude-sonnet-5", + }, + } + ), + ), + "gateway-without-a-provider-key": ( + {"HUD_API_KEY": "hud-key"}, + ClaudeCLIConfig(), + snapshot( + { + "argv": "--verbose --input-format=stream-json --output-format=stream-json --print --permission-mode=bypassPermissions --allowedTools Read --allowedTools Write --allowedTools Edit --allowedTools Bash --allowedTools Glob --allowedTools Grep --allowedTools WebSearch --allowedTools WebFetch", + "env": { + "ANTHROPIC_CUSTOM_HEADERS": "Trace-Id: ", + "ANTHROPIC_DEFAULT_HAIKU_MODEL": "claude-sonnet-5", + "IS_SANDBOX": "1", + "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS": "1", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + "CLAUDE_CODE_SUBAGENT_MODEL": "claude-sonnet-5", + "ANTHROPIC_BASE_URL": "http://127.0.0.1:9/gateway", + "DISABLE_AUTOUPDATER": "1", + "ANTHROPIC_DEFAULT_OPUS_MODEL": "claude-sonnet-5", + "ANTHROPIC_DEFAULT_SONNET_MODEL": "claude-sonnet-5", + "ANTHROPIC_MODEL": "claude-sonnet-5", + "ANTHROPIC_SMALL_FAST_MODEL": "claude-sonnet-5", + "ANTHROPIC_API_KEY": "hud-key", + }, + } + ), + ), + "gateway-forced-over-a-provider-key": ( + {"HUD_API_KEY": "hud-key", "ANTHROPIC_API_KEY": "anthropic-key"}, + ClaudeCLIConfig(gateway=True), + snapshot( + { + "argv": "--verbose --input-format=stream-json --output-format=stream-json --print --permission-mode=bypassPermissions --allowedTools Read --allowedTools Write --allowedTools Edit --allowedTools Bash --allowedTools Glob --allowedTools Grep --allowedTools WebSearch --allowedTools WebFetch", + "env": { + "ANTHROPIC_CUSTOM_HEADERS": "Trace-Id: ", + "ANTHROPIC_DEFAULT_HAIKU_MODEL": "claude-sonnet-5", + "IS_SANDBOX": "1", + "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS": "1", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + "CLAUDE_CODE_SUBAGENT_MODEL": "claude-sonnet-5", + "ANTHROPIC_BASE_URL": "http://127.0.0.1:9/gateway", + "DISABLE_AUTOUPDATER": "1", + "ANTHROPIC_DEFAULT_OPUS_MODEL": "claude-sonnet-5", + "ANTHROPIC_DEFAULT_SONNET_MODEL": "claude-sonnet-5", + "ANTHROPIC_MODEL": "claude-sonnet-5", + "ANTHROPIC_SMALL_FAST_MODEL": "claude-sonnet-5", + "ANTHROPIC_API_KEY": "hud-key", + }, + } + ), + ), + "configured-options": ( + {"ANTHROPIC_API_KEY": "anthropic-key"}, + ClaudeCLIConfig( + model="claude-opus-4-8", + reasoning_effort="max", + max_steps=3, + system_prompt="Be brief.", + allowed_tools=["Read", "Bash"], + permission_mode="acceptEdits", + ), + snapshot( + { + "argv": "--verbose --input-format=stream-json --output-format=stream-json --print --permission-mode=acceptEdits --max-turns=3 --effort max --system-prompt Be brief. --allowedTools Read --allowedTools Bash", + "env": { + "IS_SANDBOX": "1", + "ANTHROPIC_MODEL": "claude-opus-4-8", + "ANTHROPIC_API_KEY": "anthropic-key", + "DISABLE_AUTOUPDATER": "1", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + "ANTHROPIC_SMALL_FAST_MODEL": "claude-opus-4-8", + }, + } + ), + ), +} + + +@pytest.mark.parametrize( + ("variables", "config", "expected"), + CLAUDE_INVOCATIONS.values(), + ids=CLAUDE_INVOCATIONS.keys(), +) +async def test_the_claude_cli_is_launched_with_the_routing_and_options_configured( + variables: dict[str, str], + config: ClaudeCLIConfig, + expected: Any, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(**variables) + stub = claude_stub(tmp_path) + + run = await run_task(cli_env(tmp_path, stub), ClaudeCLIAgent(config)) + + (capture,) = stub.captures() + assert invocation(capture, run) == expected + assert run.trace.status == "completed" + + +CLAUDE_EXITS = { + "nonzero-exit-without-output": ( + {"stdout": "", "stderr": "boom", "exit": 1}, + snapshot( + { + "status": "error", + "content": None, + "error": "[agent loop] RuntimeError: boom", + "extra": {"returncode": 1, "stderr": "boom"}, + } + ), + ), + "nonzero-exit-after-a-result-with-stderr": ( + {"stderr": "transport failed", "exit": 1}, + snapshot( + { + "status": "error", + "content": "done", + "error": "[agent loop] RuntimeError: transport failed", + "extra": { + "subtype": "success", + "session_id": "session-1", + "duration_ms": 5, + "num_turns": 2, + "total_cost_usd": 0.01, + "returncode": 1, + "stderr": "transport failed", + }, + } + ), + ), + "nonzero-exit-after-a-result-without-stderr": ( + {"exit": 1}, + snapshot( + { + "status": "completed", + "content": "done", + "error": None, + "extra": { + "subtype": "success", + "session_id": "session-1", + "duration_ms": 5, + "num_turns": 2, + "total_cost_usd": 0.01, + "returncode": 1, + }, + } + ), + ), + "zero-exit-without-a-result": ( + {"stdout": jsonl(WRITE, WROTE, DONE)}, + snapshot( + { + "status": "error", + "content": "done", + "error": "[agent loop] RuntimeError: claude CLI exited without a result event", + "extra": {}, + } + ), + ), + "result-reports-an-error": ( + {"stdout": jsonl(DONE, result("rate limited upstream", is_error=True))}, + snapshot( + { + "status": "error", + "content": "rate limited upstream", + "error": "[agent loop] RuntimeError: rate limited upstream", + "extra": { + "subtype": "success", + "session_id": "session-1", + "duration_ms": 5, + "num_turns": 2, + "total_cost_usd": 0.01, + }, + } + ), + ), + "tool-call-left-without-a-result": ( + {"stdout": jsonl(WRITE, result())}, + snapshot( + { + "status": "error", + "content": "done", + "error": "[agent loop] RuntimeError: claude CLI exited without results for tool calls: tool-1", + "extra": { + "subtype": "success", + "session_id": "session-1", + "duration_ms": 5, + "num_turns": 2, + "total_cost_usd": 0.01, + }, + } + ), + ), +} + + +@pytest.mark.parametrize(("behavior", "expected"), CLAUDE_EXITS.values(), ids=CLAUDE_EXITS.keys()) +async def test_how_the_claude_cli_exits_decides_how_the_run_ends( + behavior: dict[str, Any], expected: Any, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(ANTHROPIC_API_KEY="anthropic-key") + stub = claude_stub(tmp_path, **behavior) + + run = await run_task(cli_env(tmp_path, stub), ClaudeCLIAgent()) + + assert outcome(run) == expected + + +async def test_claude_cli_tool_results_keep_documents_and_name_unknown_blocks( + hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(ANTHROPIC_API_KEY="anthropic-key") + read = assistant( + {"type": "tool_use", "id": "tool-1", "name": "Read", "input": {}}, stop="tool_use" + ) + pdf = { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0="}, + } + stub = claude_stub( + tmp_path, + stdout=jsonl(read, tool_result("tool-1", pdf, {"type": "novelty"}), DONE, result()), + ) + + run = await run_task(cli_env(tmp_path, stub), ClaudeCLIAgent()) + + tool = next(payload for payload in step_payloads(run) if payload["source"] == "tool") + assert (run.trace.status, tool["result"]["content"]) == snapshot( + ( + "completed", + [ + { + "type": "resource", + "resource": { + "uri": "document://tool-1/0", + "mimeType": "application/pdf", + "blob": "JVBERi0=", + }, + }, + {"type": "text", "text": "[unsupported novelty block]"}, + ], + ) + ) + + +async def test_mcp_servers_reach_the_claude_cli_in_a_config_removed_after_the_run( + hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(ANTHROPIC_API_KEY="anthropic-key") + stub = claude_stub(tmp_path) + servers = ( + Capability.mcp(name="database", url="http://db.internal:8000/mcp", auth_token="secret"), + Capability.mcp(name="search", url="http://search.internal:8000/sse", transport="sse"), + ) + + await run_task(cli_env(tmp_path, stub, capabilities=servers), ClaudeCLIAgent()) + + (capture,) = stub.captures() + assert capture["argv"][-2:] == ["--mcp-config", ".hud_mcp_config.json"] + assert capture["files"] == snapshot( + { + ".hud_mcp_config.json": """\ +{ + "mcpServers": { + "database": { + "type": "http", + "url": "http://db.internal:8000/mcp", + "headers": { + "Authorization": "Bearer secret" + } + }, + "search": { + "type": "sse", + "url": "http://search.internal:8000/sse" + } + } +}\ +""" + } + ) + assert sorted(path.name for path in (tmp_path / "ws").iterdir()) == [] + + +async def test_the_claude_cli_drives_the_screen_through_the_computer_bridge( + hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(ANTHROPIC_API_KEY="anthropic-key") + stub = claude_stub( + tmp_path, + computer=[ + {"action": "left_click", "coordinate": [3, 4]}, + {"action": "frobnicate"}, + ], + ) + + async with fake_screen() as screen: + env = cli_env(tmp_path, stub, capabilities=(Capability.rfb(url=screen.url),)) + run = await run_task(env, ClaudeCLIAgent()) + + (capture,) = stub.captures() + assert run.trace.status == "completed" + assert capture["computer"] == snapshot( + [ + {"isError": False, "content": ["image"]}, + {"isError": True, "content": ["unsupported computer action: 'frobnicate'"]}, + ] + ) + assert [(event.x, event.y, event.buttons) for event in screen.pointer_events()] == [ + (3, 4, 0), + (3, 4, 1), + (3, 4, 0), + ] + + +async def test_an_mcp_server_named_like_the_computer_bridge_is_refused( + hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(ANTHROPIC_API_KEY="anthropic-key") + stub = claude_stub(tmp_path) + + async with fake_screen() as screen: + clash = ( + Capability.rfb(url=screen.url), + Capability.mcp(name="computer-use", url="http://tools.internal:8000/mcp"), + ) + run = await run_task(cli_env(tmp_path, stub, capabilities=clash), ClaudeCLIAgent()) + + assert (run.trace.error, stub.captures()) == snapshot( + ("[agent loop] RuntimeError: duplicate MCP server name 'computer-use'", []) + ) + + +async def test_the_claude_cli_needs_a_credential_to_route_through_the_gateway( + tmp_path: Path, +) -> None: + stub = claude_stub(tmp_path) + + run = await run_task(cli_env(tmp_path, stub), ClaudeCLIAgent(ClaudeCLIConfig(gateway=True))) + + assert (run.trace.error, stub.captures()) == snapshot( + ("[agent loop] ValueError: HUD_API_KEY is required for HUD gateway routing", []) + ) + + +# ─── both CLIs ────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Cli: + stub: Callable[..., Stub] + agent: Callable[[], Agent] + variables: dict[str, str] + + +CLIS = { + "claude": Cli(claude_stub, ClaudeCLIAgent, {"ANTHROPIC_API_KEY": "anthropic-key"}), + "codex": Cli( + lambda root, **kw: codex_stub(root, **kw), CodexCLIAgent, {"OPENAI_API_KEY": "openai-key"} + ), +} + + +@pytest.mark.parametrize("cli", CLIS.values(), ids=CLIS.keys()) +async def test_a_missing_cli_names_the_runtime_platform( + cli: Cli, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(**cli.variables) + stub = cli.stub(tmp_path) + stub.install() + env = workspace_env(tmp_path / "ws", env=stub.shell_env(path="/usr/bin:/bin")) + + run = await run_task(env, cli.agent()) + + assert run.trace.error == IsStr( + regex=r".* is unavailable for runtime platform linux-\w+; install it in the " + r"environment or provide a managed runtime bundle" + ) + + +@pytest.mark.parametrize("cli", CLIS.values(), ids=CLIS.keys()) +async def test_one_cli_agent_drives_concurrent_rollouts( + cli: Cli, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(**cli.variables) + stub = cli.stub(tmp_path) + env = cli_env(tmp_path, stub) + agent = cli.agent() + + runs = await asyncio.gather( + run_task(env, agent, prompt="first"), run_task(env, agent, prompt="second") + ) + + stdins = sorted(capture["stdin"] for capture in stub.captures()) + assert [run.trace.status for run in runs] == ["completed", "completed"] + assert [("first" in stdin, "second" in stdin) for stdin in stdins] == [ + (True, False), + (False, True), + ] + + +@pytest.mark.parametrize("cli", CLIS.values(), ids=CLIS.keys()) +async def test_an_agent_timeout_stops_the_cli(cli: Cli, hud_env: HudEnv, tmp_path: Path) -> None: + hud_env.set(**cli.variables) + first_line = (CLAUDE_STREAM if cli.agent is ClaudeCLIAgent else CODEX_STREAM).splitlines()[0] + stub = cli.stub(tmp_path, stdout=first_line + "\n", hang=True) + agent = type(cli.agent()).load({**cli.agent().dump(), "timeout_seconds": 2.0}) + + run = await run_task(cli_env(tmp_path, stub), agent) + + (capture,) = stub.captures() + assert (run.trace.status, run.trace.stop_reason) == ("error", "timeout") + assert process_exited(capture["pid"]) + + +# ─── Codex CLI ────────────────────────────────────────────────────────── + + +def item(kind: str, item_id: str, **fields: Any) -> dict[str, Any]: + return {"type": "item.completed", "item": {"id": item_id, "type": kind, **fields}} + + +CODEX_ITEMS = [ + {"type": "thread.started", "thread_id": "thread-1"}, + {"type": "turn.started"}, + { + "type": "item.started", + "item": { + "id": "cmd-1", + "type": "command_execution", + "command": "pytest -q", + "aggregated_output": "", + "exit_code": None, + "status": "in_progress", + }, + }, + item( + "command_execution", + "cmd-1", + command="pytest -q", + aggregated_output="1 passed\n", + exit_code=0, + status="completed", + ), + item( + "command_execution", + "cmd-2", + command="false", + aggregated_output="", + exit_code=1, + status="completed", + ), + item( + "file_change", + "patch-1", + changes=[{"path": "calc.py", "kind": "update"}], + status="completed", + ), + item( + "mcp_tool_call", + "mcp-1", + server="db", + tool="query", + arguments={"sql": "select 42"}, + result={"content": [{"type": "text", "text": "42"}], "structured_content": {"answer": 42}}, + error=None, + status="completed", + ), + item( + "mcp_tool_call", + "mcp-2", + server="db", + tool="query", + arguments={"sql": "drop"}, + result=None, + error={"message": "denied"}, + status="failed", + ), + item("web_search", "search-1", query="HUD evals", action={"type": "search"}), + item("todo_list", "todo-1", items=[{"text": "ship", "completed": False}]), + item("reasoning", "reason-1", text="The test now passes."), + item("agent_message", "message-1", text="Implemented and verified."), +] +CODEX_DONE = { + "type": "turn.completed", + "usage": {"input_tokens": 20, "cached_input_tokens": 5, "output_tokens": 8}, +} +CODEX_STREAM = jsonl(*CODEX_ITEMS, CODEX_DONE) + + +async def test_codex_events_become_steps(hud_env: HudEnv, tmp_path: Path) -> None: + hud_env.set(OPENAI_API_KEY="openai-key") + stub = codex_stub(tmp_path) + + run = await run_task(cli_env(tmp_path, stub), CodexCLIAgent(), prompt="Fix the failing test") + + (capture,) = stub.captures() + assert capture["stdin"] == "Fix the failing test" + assert not await anyio.Path(capture["env"]["CODEX_HOME"]).exists() + assert outcome(run) == snapshot( + { + "status": "completed", + "content": "Implemented and verified.", + "error": None, + "extra": { + "codex_thread_id": "thread-1", + "usage": {"input_tokens": 20, "cached_input_tokens": 5, "output_tokens": 8}, + }, + } + ) + assert step_payloads(run) == snapshot( + [ + { + "step_id": 3, + "source": "tool", + "messages": [], + "extra": { + "codex_item": { + "id": "cmd-1", + "type": "command_execution", + "command": "pytest -q", + "aggregated_output": "1 passed\n", + "exit_code": 0, + "status": "completed", + } + }, + "call": {"name": "shell", "arguments": {"command": "pytest -q"}, "id": "cmd-1"}, + "result": { + "content": [{"type": "text", "text": "1 passed\n"}], + "isError": False, + "call_id": "cmd-1", + }, + }, + { + "step_id": 4, + "source": "tool", + "messages": [], + "extra": { + "codex_item": { + "id": "cmd-2", + "type": "command_execution", + "command": "false", + "aggregated_output": "", + "exit_code": 1, + "status": "completed", + } + }, + "call": {"name": "shell", "arguments": {"command": "false"}, "id": "cmd-2"}, + "result": { + "content": [{"type": "text", "text": ""}], + "isError": True, + "call_id": "cmd-2", + }, + }, + { + "step_id": 5, + "source": "tool", + "messages": [], + "extra": { + "codex_item": { + "id": "patch-1", + "type": "file_change", + "changes": [{"path": "calc.py", "kind": "update"}], + "status": "completed", + } + }, + "call": { + "name": "apply_patch", + "arguments": {"changes": [{"path": "calc.py", "kind": "update"}]}, + "id": "patch-1", + }, + "result": { + "content": [{"type": "text", "text": "update: calc.py"}], + "isError": False, + "call_id": "patch-1", + }, + }, + { + "step_id": 6, + "source": "tool", + "messages": [], + "extra": { + "codex_item": { + "id": "mcp-1", + "type": "mcp_tool_call", + "server": "db", + "tool": "query", + "arguments": {"sql": "select 42"}, + "result": { + "content": [{"type": "text", "text": "42"}], + "structured_content": {"answer": 42}, + }, + "error": None, + "status": "completed", + } + }, + "call": { + "name": "query", + "arguments": {"sql": "select 42"}, + "id": "mcp-1", + "provider_name": "db.query", + }, + "result": { + "content": [{"type": "text", "text": "42"}], + "structuredContent": {"answer": 42}, + "isError": False, + "call_id": "mcp-1", + }, + }, + { + "step_id": 7, + "source": "tool", + "messages": [], + "extra": { + "codex_item": { + "id": "mcp-2", + "type": "mcp_tool_call", + "server": "db", + "tool": "query", + "arguments": {"sql": "drop"}, + "result": None, + "error": {"message": "denied"}, + "status": "failed", + } + }, + "call": { + "name": "query", + "arguments": {"sql": "drop"}, + "id": "mcp-2", + "provider_name": "db.query", + }, + "result": { + "content": [{"type": "text", "text": "denied"}], + "isError": True, + "call_id": "mcp-2", + }, + }, + { + "step_id": 8, + "source": "tool", + "messages": [], + "extra": { + "codex_item": { + "id": "search-1", + "type": "web_search", + "query": "HUD evals", + "action": {"type": "search"}, + } + }, + "call": { + "name": "web_search", + "arguments": {"query": "HUD evals", "action": {"type": "search"}}, + "id": "search-1", + }, + "result": { + "content": [{"type": "text", "text": '{"type":"search"}'}], + "isError": False, + "call_id": "search-1", + }, + }, + { + "step_id": 9, + "source": "agent", + "messages": [], + "extra": { + "codex_item": { + "id": "todo-1", + "type": "todo_list", + "items": [{"text": "ship", "completed": False}], + } + }, + }, + { + "step_id": 10, + "source": "agent", + "messages": [], + "extra": {}, + "reasoning": "The test now passes.", + "tool_calls": [], + "done": False, + "citations": [], + "raw": {"id": "reason-1", "type": "reasoning", "text": "The test now passes."}, + "model": "gpt-5.6-sol", + }, + { + "step_id": 11, + "source": "agent", + "messages": [], + "extra": {}, + "content": "Implemented and verified.", + "tool_calls": [], + "done": False, + "citations": [], + "raw": { + "id": "message-1", + "type": "agent_message", + "text": "Implemented and verified.", + }, + "model": "gpt-5.6-sol", + }, + ] + ) + + +CODEX_INVOCATIONS = { + "provider-key": ( + {"OPENAI_API_KEY": "openai-key"}, + CodexCLIConfig(), + snapshot( + { + "argv": "exec --json --ephemeral --skip-git-repo-check --color never --sandbox workspace-write --model gpt-5.6-sol -", + "env": {"CODEX_API_KEY": "openai-key", "CODEX_HOME": ""}, + } + ), + ), + "gateway-without-a-provider-key": ( + {"HUD_API_KEY": "hud-key"}, + CodexCLIConfig(), + snapshot( + { + "argv": 'exec --json --ephemeral --skip-git-repo-check --color never --sandbox workspace-write --model gpt-5.6-sol -c model_provider="hud" -c model_providers.hud.name="HUD" -c model_providers.hud.base_url="http://127.0.0.1:9/gateway" -c model_providers.hud.env_key="HUD_API_KEY" -c model_providers.hud.wire_api="responses" -c model_providers.hud.http_headers={"Trace-Id"=""} -', + "env": {"HUD_API_KEY": "hud-key", "CODEX_HOME": ""}, + } + ), + ), + "gateway-forced-over-a-provider-key": ( + {"HUD_API_KEY": "hud-key", "OPENAI_API_KEY": "openai-key"}, + CodexCLIConfig(gateway=True), + snapshot( + { + "argv": 'exec --json --ephemeral --skip-git-repo-check --color never --sandbox workspace-write --model gpt-5.6-sol -c model_provider="hud" -c model_providers.hud.name="HUD" -c model_providers.hud.base_url="http://127.0.0.1:9/gateway" -c model_providers.hud.env_key="HUD_API_KEY" -c model_providers.hud.wire_api="responses" -c model_providers.hud.http_headers={"Trace-Id"=""} -', + "env": {"HUD_API_KEY": "hud-key", "CODEX_HOME": ""}, + } + ), + ), + "ambient-login": ( + {}, + CodexCLIConfig(), + snapshot( + { + "argv": "exec --json --ephemeral --skip-git-repo-check --color never --sandbox workspace-write --model gpt-5.6-sol -", + "env": {}, + } + ), + ), + "configured-options": ( + {"OPENAI_API_KEY": "openai-key"}, + CodexCLIConfig(model="gpt-5.6", sandbox="read-only", reasoning_effort="max"), + snapshot( + { + "argv": 'exec --json --ephemeral --skip-git-repo-check --color never --sandbox read-only --model gpt-5.6 -c model_reasoning_effort="max" -', + "env": {"CODEX_API_KEY": "openai-key", "CODEX_HOME": ""}, + } + ), + ), +} + + +@pytest.mark.parametrize( + ("variables", "config", "expected"), + CODEX_INVOCATIONS.values(), + ids=CODEX_INVOCATIONS.keys(), +) +async def test_codex_is_launched_with_the_routing_and_options_configured( + variables: dict[str, str], + config: CodexCLIConfig, + expected: Any, + hud_env: HudEnv, + tmp_path: Path, +) -> None: + hud_env.set(**variables) + stub = codex_stub(tmp_path) + + run = await run_task(cli_env(tmp_path, stub), CodexCLIAgent(config)) + + (capture,) = stub.captures() + assert invocation(capture, run) == expected + + +CODEX_EXITS = { + "turn-failed": ( + { + "stdout": jsonl( + {"type": "turn.started"}, + {"type": "turn.failed", "error": {"message": "model unavailable"}}, + ) + }, + snapshot( + { + "status": "error", + "content": None, + "error": "[agent loop] RuntimeError: model unavailable", + "extra": {}, + } + ), + ), + "nonzero-exit": ( + {"stdout": "", "stderr": "authentication failed", "exit": 1}, + snapshot( + { + "status": "error", + "content": None, + "error": "[agent loop] RuntimeError: authentication failed", + "extra": {"returncode": 1, "stderr": "authentication failed"}, + } + ), + ), + "nonzero-exit-with-a-structured-error": ( + { + "stdout": jsonl({"type": "error", "message": "gateway rejected streaming"}), + "stderr": "noisy", + "exit": 1, + }, + snapshot( + { + "status": "error", + "content": None, + "error": "[agent loop] RuntimeError: gateway rejected streaming", + "extra": {"returncode": 1}, + } + ), + ), + "zero-exit-without-turn-completed": ( + {"stdout": jsonl(*CODEX_ITEMS)}, + snapshot( + { + "status": "error", + "content": "Implemented and verified.", + "error": "[agent loop] RuntimeError: codex CLI exited without a turn.completed event", + "extra": {"codex_thread_id": "thread-1"}, + } + ), + ), +} + + +@pytest.mark.parametrize(("behavior", "expected"), CODEX_EXITS.values(), ids=CODEX_EXITS.keys()) +async def test_how_codex_exits_decides_how_the_run_ends( + behavior: dict[str, Any], expected: Any, hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(OPENAI_API_KEY="openai-key") + stub = codex_stub(tmp_path, **behavior) + + run = await run_task(cli_env(tmp_path, stub), CodexCLIAgent()) + + assert outcome(run) == expected From 01f6cd484f36ddba4d7b05d5186da51eb0870f2c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 049/248] test(agents): sandbox-lane rows for CLI agents on a process-bound connection A process-bound inference connection needs an isolated workspace, so these rows run in the e2e sandbox lane: the Claude CLI gets the connection's client URL and the placeholder credential (never the scoped token, the HUD key or a trace header), and Codex refuses a launcher script before it starts. --- tests/agents/test_cli_agents.py | 61 ++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/agents/test_cli_agents.py b/tests/agents/test_cli_agents.py index c9c722916..f48392406 100644 --- a/tests/agents/test_cli_agents.py +++ b/tests/agents/test_cli_agents.py @@ -21,7 +21,8 @@ from hud.agents import ClaudeCLIAgent, CodexCLIAgent from hud.agents.types import ClaudeCLIConfig, CodexCLIConfig -from hud.capabilities import Capability +from hud.capabilities import Capability, Connection +from hud.environment.egress import ANY_HOST from tests.agents.cli_stub import Stub, jsonl, process_exited from tests.agents.support import run_task, workspace_env from tests.harness import fake_screen, steps @@ -1102,3 +1103,61 @@ async def test_how_codex_exits_decides_how_the_run_ends( run = await run_task(cli_env(tmp_path, stub), CodexCLIAgent()) assert outcome(run) == expected + + +# ─── process-bound inference connections (sandbox lane) ───────────────── + +CONNECTION = Connection( + name="inference", + capability="ssh", + url="https://inference.hud.so/v1", + headers={"Authorization": "Bearer scoped-runtime-token"}, +) + + +def isolated_env(tmp_path: Path, stub: Stub) -> Environment: + stub.install() + return workspace_env( + tmp_path / "ws", + env=stub.shell_env(), + allowed_hosts={ANY_HOST}, + require_isolation=True, + track_files=False, + ) + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_the_claude_cli_reaches_a_bound_connection_without_its_credential( + hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="hud-key") + stub = claude_stub(tmp_path) + + run = await run_task(isolated_env(tmp_path, stub), ClaudeCLIAgent(), connections=(CONNECTION,)) + + (capture,) = stub.captures() + env = capture["env"] + assert run.trace.status == "completed" + assert (env["ANTHROPIC_BASE_URL"], env["ANTHROPIC_AUTH_TOKEN"]) == ( + CONNECTION.client_url, + "hud-process-bound", + ) + assert not {"ANTHROPIC_API_KEY", "ANTHROPIC_CUSTOM_HEADERS"} & env.keys() + assert "scoped-runtime-token" not in str(capture) and "hud-key" not in str(capture) + + +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_codex_refuses_a_launcher_script_for_a_bound_connection( + hud_env: HudEnv, tmp_path: Path +) -> None: + hud_env.set(HUD_API_KEY="hud-key") + stub = codex_stub(tmp_path) + + run = await run_task(isolated_env(tmp_path, stub), CodexCLIAgent(), connections=(CONNECTION,)) + + assert (run.trace.error, stub.captures()) == ( + IsStr(regex=r".*/codex is a launcher script, but a process-bound inference connection .*"), + [], + ) From 07c17d0e14567128b175f558157e2fb0534e5238 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 050/248] test(agents): pin the LeRobot datasets RobotAgent(save=True) records DatasetWriter was tested by clearing its private class registries and faking importlib inside the module. Real robot rollouts against a stub sim now record through a stand-in lerobot package on the import path that writes each dataset as JSON; rows pin the features and frames (observation, executed action) per tick, one dataset shared by concurrent same-contract rollouts and separate ones per rate or robot, the frames of a failed rollout kept, nested camera names, clashing feature names, the Hub push only with HF_REPO (and the on-disk dataset kept when it fails), and the single warning when lerobot is missing. --- tests/agents/fake_lerobot/lerobot/__init__.py | 1 + .../fake_lerobot/lerobot/datasets/__init__.py | 0 .../lerobot/datasets/lerobot_dataset.py | 79 ++++ tests/agents/test_robot_recording.py | 359 ++++++++++++++++++ 4 files changed, 439 insertions(+) create mode 100644 tests/agents/fake_lerobot/lerobot/__init__.py create mode 100644 tests/agents/fake_lerobot/lerobot/datasets/__init__.py create mode 100644 tests/agents/fake_lerobot/lerobot/datasets/lerobot_dataset.py create mode 100644 tests/agents/test_robot_recording.py diff --git a/tests/agents/fake_lerobot/lerobot/__init__.py b/tests/agents/fake_lerobot/lerobot/__init__.py new file mode 100644 index 000000000..3b4bc6639 --- /dev/null +++ b/tests/agents/fake_lerobot/lerobot/__init__.py @@ -0,0 +1 @@ +"""A stand-in for the ``lerobot`` package that records datasets as JSON.""" diff --git a/tests/agents/fake_lerobot/lerobot/datasets/__init__.py b/tests/agents/fake_lerobot/lerobot/datasets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/agents/fake_lerobot/lerobot/datasets/lerobot_dataset.py b/tests/agents/fake_lerobot/lerobot/datasets/lerobot_dataset.py new file mode 100644 index 000000000..35b9bfa2a --- /dev/null +++ b/tests/agents/fake_lerobot/lerobot/datasets/lerobot_dataset.py @@ -0,0 +1,79 @@ +"""``LeRobotDataset`` as the recording code uses it, written to ``/dataset.json``. + +Frames are stored as lists; image arrays as ``""``. Setting +``FAKE_LEROBOT_PUSH_FAILS`` makes ``push_to_hub`` raise. +""" + +from __future__ import annotations + +import json +import os +from typing import TYPE_CHECKING, Any + +import numpy as np + +if TYPE_CHECKING: + from pathlib import Path + + +def _value(value: Any) -> Any: + array = np.asarray(value) + if array.dtype == np.uint8 and array.ndim == 3: + return "".format("x".join(str(size) for size in array.shape)) + return array.tolist() if isinstance(value, np.ndarray) else value + + +class LeRobotDataset: + def __init__(self, root: Path, record: dict[str, Any]) -> None: + self.root = root + self.record = record + self.pending: list[dict[str, Any]] = [] + self._write() + + @classmethod + def create( + cls, + *, + repo_id: str, + fps: int, + features: dict[str, Any], + root: Path, + robot_type: str | None, + use_videos: bool, + ) -> LeRobotDataset: + root.mkdir(parents=True) + record = { + "repo_id": repo_id, + "fps": fps, + "robot_type": robot_type, + "use_videos": use_videos, + "features": { + key: {**feature, "shape": list(feature["shape"])} + for key, feature in features.items() + }, + "episodes": [], + "finalized": False, + "pushed": None, + } + return cls(root, record) + + def add_frame(self, frame: dict[str, Any]) -> None: + self.pending.append({key: _value(value) for key, value in frame.items()}) + + def save_episode(self) -> None: + self.record["episodes"].append(self.pending) + self.pending = [] + self._write() + + def finalize(self) -> None: + self.record["finalized"] = True + self._write() + + def push_to_hub(self, *, private: bool) -> None: + if os.environ.get("FAKE_LEROBOT_PUSH_FAILS"): + raise ConnectionError("hub unreachable") + self.record["pushed"] = {"private": private} + self._write() + + def _write(self) -> None: + (self.root / "dataset.json").write_text(json.dumps(self.record)) diff --git a/tests/agents/test_robot_recording.py b/tests/agents/test_robot_recording.py new file mode 100644 index 000000000..671c851ce --- /dev/null +++ b/tests/agents/test_robot_recording.py @@ -0,0 +1,359 @@ +# ruff: noqa: E501 -- snapshots quote product messages verbatim +"""``RobotAgent(save=True)`` records each rollout as a LeRobot episode. + +A stub sim serves the ``robot`` capability and a stub policy acts on it; the +``lerobot`` package is a stand-in on the import path that writes each dataset +it is asked to create as JSON under ``RECORD_DIR``. Rows pin the datasets on +disk: which rollouts share one, the features taken from the contract, the +frames (observation, executed action) per tick, failed rollouts kept, and the +Hub push. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import sys +from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime +from contextlib import asynccontextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest +from inline_snapshot import snapshot + +from hud.agents.robot import DatasetWriter, Model, RobotAgent +from hud.environment import Environment +from hud.environment.robot import RobotBridge, RobotEndpoint +from hud.eval import LocalRuntime, Task, rollout + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterator + + from numpy.typing import NDArray + + from hud.eval import Run + from tests.harness import HudEnv + +FAKE_LEROBOT = Path(__file__).parent / "fake_lerobot" +TICKS = 3 + + +def contract(*, rate: int = 5, robot: str = "arm", **features: dict[str, Any]) -> dict[str, Any]: + return { + "control_rate": rate, + "robot_type": robot, + "features": features + or { + "observation/image": {"role": "observation", "type": "rgb"}, + "observation/state": {"role": "observation", "names": ["tick", "half"]}, + "action": {"role": "action", "names": ["a0", "a1"]}, + }, + } + + +class Sim(RobotBridge): + """One camera and a 2-D state ``[tick, 0.5]``; the episode ends after ``TICKS``.""" + + def __init__(self, robot_contract: dict[str, Any]) -> None: + super().__init__() + self.num_envs = 1 + self.contract = robot_contract + self.tick = 0 + + def reset(self, **task_args: Any) -> str: + self.tick = 0 + return "pick it up" + + def step(self, action: NDArray[Any]) -> None: + self.tick += 1 + + def get_observation(self) -> tuple[dict[str, NDArray[Any]], NDArray[Any]]: + data: dict[str, NDArray[Any]] = {} + for name, feature in self.contract["features"].items(): + if feature["role"] != "observation": + continue + if feature.get("type") == "rgb": + data[name] = np.full((1, 2, 2, 3), self.tick, dtype=np.uint8) + else: + data[name] = np.array([[self.tick, 0.5]], dtype=np.float32) + return data, np.array([self.tick >= TICKS]) + + def result_slots(self) -> list[dict[str, Any]]: + return [{"score": 1.0, "success": True, "total_reward": 1.0}] + + +class Echo(Model): + """Acts ``[tick, 1]``; raises on the tick named by ``fail_at``.""" + + def __init__(self, fail_at: int | None = None) -> None: + self.fail_at = fail_at + + def infer(self, batch: Any) -> Any: + state = next(value for value in batch["data"].values() if np.asarray(value).ndim == 1) + tick = float(np.asarray(state).reshape(-1)[0]) + if tick == self.fail_at: + raise RuntimeError("policy crashed") + return np.array([[[tick, 1.0]]], dtype=np.float32) + + +class Recorder(RobotAgent): + save = True + max_steps = TICKS + 2 + log_every = 0 + + def __init__(self, model: Model | None = None) -> None: + super().__init__() + self.model = model or Echo() + self.adapter = None + + +@asynccontextmanager +async def sim_env(name: str, robot_contract: dict[str, Any]) -> AsyncIterator[Environment]: + sim = Sim(robot_contract) + await sim.start() + control = await sim.serve_control() + env = Environment(name) + endpoint = RobotEndpoint.remote("127.0.0.1", control.sockets[0].getsockname()[1]).attach(env) + + @env.initialize + async def up() -> None: + await endpoint.start() + for capability in await endpoint.capabilities(): + env.add_capability(capability) + + @env.shutdown + async def down() -> None: + await endpoint.stop() + + @env.template() + async def episode() -> AsyncGenerator[Any, Any]: + started = await endpoint.reset() + yield {"prompt": started["prompt"]} + yield await endpoint.result() + + try: + await env.start() + yield env + finally: + await env.stop() + control.close() + await sim.stop() + + +async def record(env: Environment, agent: RobotAgent | None = None) -> Run: + return await rollout( + Task(env=env.name, id="episode"), agent or Recorder(), runtime=LocalRuntime(env) + ) + + +def datasets(record_dir: Path) -> list[dict[str, Any]]: + """Every dataset written under ``record_dir``, oldest first, without its timestamped name.""" + found = [json.loads(path.read_text()) for path in sorted(record_dir.glob("*/dataset.json"))] + for dataset in found: + dataset.pop("repo_id") + return found + + +@pytest.fixture +def recording(hud_env: HudEnv, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: + """``RECORD_DIR`` for this test with the stand-in ``lerobot`` importable. Open + datasets are finalized and the stand-in unloaded afterwards, so neither + outlives the test.""" + monkeypatch.syspath_prepend(str(FAKE_LEROBOT)) + record_dir = tmp_path / "records" + hud_env.set(RECORD_DIR=str(record_dir)) + yield record_dir + DatasetWriter.finalize() + for name in [name for name in sys.modules if name.split(".")[0] == "lerobot"]: + del sys.modules[name] + + +async def test_a_recorded_rollout_saves_one_episode_of_observed_and_executed_frames( + recording: Path, +) -> None: + async with sim_env("arm", contract()) as env: + run = await record(env) + DatasetWriter.finalize() + + assert run.reward == 1.0 + assert datasets(recording) == snapshot( + [ + { + "fps": 5, + "robot_type": "arm", + "use_videos": True, + "features": { + "observation.images.observation_image": { + "dtype": "video", + "shape": [2, 2, 3], + "names": ["height", "width", "channel"], + }, + "observation.state": { + "dtype": "float32", + "shape": [2], + "names": ["tick", "half"], + }, + "action": {"dtype": "float32", "shape": [2], "names": ["a0", "a1"]}, + }, + "episodes": [ + [ + { + "observation.images.observation_image": "", + "observation.state": [0.0, 0.5], + "action": [0.0, 1.0], + "task": "pick it up", + }, + { + "observation.images.observation_image": "", + "observation.state": [1.0, 0.5], + "action": [1.0, 1.0], + "task": "pick it up", + }, + { + "observation.images.observation_image": "", + "observation.state": [2.0, 0.5], + "action": [2.0, 1.0], + "task": "pick it up", + }, + ] + ], + "finalized": True, + "pushed": None, + } + ] + ) + + +async def test_concurrent_rollouts_of_one_contract_share_a_dataset(recording: Path) -> None: + async with sim_env("left", contract()) as left, sim_env("right", contract()) as right: + await asyncio.gather(record(left), record(right)) + DatasetWriter.finalize() + + (dataset,) = datasets(recording) + assert [len(episode) for episode in dataset["episodes"]] == [TICKS, TICKS] + + +async def test_a_different_rate_or_robot_gets_its_own_dataset(recording: Path) -> None: + async with ( + sim_env("five", contract(rate=5)) as five, + sim_env("ten", contract(rate=10)) as ten, + sim_env("other", contract(robot="other")) as other, + ): + await asyncio.gather(record(five), record(ten), record(other)) + DatasetWriter.finalize() + + assert sorted((d["fps"], d["robot_type"], len(d["episodes"])) for d in datasets(recording)) == [ + (5, "arm", 1), + (5, "other", 1), + (10, "arm", 1), + ] + + +async def test_a_failed_rollout_keeps_the_frames_it_recorded(recording: Path) -> None: + async with sim_env("arm", contract()) as env: + run = await record(env, Recorder(Echo(fail_at=2))) + DatasetWriter.finalize() + + (dataset,) = datasets(recording) + assert run.trace.status == "error" + assert [frame["action"] for frame in dataset["episodes"][0]] == [[0.0, 1.0], [1.0, 1.0]] + + +async def test_nested_camera_names_stay_distinct_features(recording: Path) -> None: + stereo = contract( + **{ + "left/image": {"role": "observation", "type": "rgb"}, + "right/image": {"role": "observation", "type": "rgb"}, + "state": {"role": "observation", "names": ["tick", "half"]}, + "action": {"role": "action", "names": ["a0", "a1"]}, + } + ) + async with sim_env("stereo", stereo) as env: + await record(env) + DatasetWriter.finalize() + + (dataset,) = datasets(recording) + assert sorted(dataset["features"]) == [ + "action", + "observation.images.left_image", + "observation.images.right_image", + "observation.state", + ] + + +async def test_features_that_flatten_to_one_key_fail_the_rollout(recording: Path) -> None: + clash = contract( + **{ + "cam/rgb": {"role": "observation", "type": "rgb"}, + "cam_rgb": {"role": "observation", "type": "rgb"}, + "state": {"role": "observation", "names": ["tick", "half"]}, + "action": {"role": "action", "names": ["a0", "a1"]}, + } + ) + async with sim_env("clash", clash) as env: + run = await record(env) + + assert run.trace.error == snapshot( + "[agent loop] ValueError: contract features 'cam/rgb' and 'cam_rgb' both map to LeRobot key 'observation.images.cam_rgb'" + ) + assert datasets(recording) == [] + + +PUSHES = { + "no-repo": ({}, snapshot((True, None, []))), + "repo": ({"HF_REPO": "lab"}, snapshot((True, {"private": False}, []))), + "private-repo": ( + {"HF_REPO": "lab", "HF_PRIVATE": "1"}, + snapshot((True, {"private": True}, [])), + ), + "push-fails": ( + {"HF_REPO": "lab", "FAKE_LEROBOT_PUSH_FAILS": "1"}, + snapshot( + ( + True, + None, + [ + "[agent] WARNING: HF push failed: ConnectionError('hub unreachable') (dataset still on disk)" + ], + ) + ), + ), +} + + +@pytest.mark.parametrize(("variables", "expected"), PUSHES.values(), ids=PUSHES.keys()) +async def test_finalizing_pushes_to_the_hub_only_with_a_repo_and_keeps_the_dataset( + variables: dict[str, str], + expected: Any, + recording: Path, + hud_env: HudEnv, + capsys: pytest.CaptureFixture[str], +) -> None: + hud_env.set(**variables) + async with sim_env("arm", contract()) as env: + await record(env) + DatasetWriter.finalize() + + (dataset,) = datasets(recording) + printed = [line for line in capsys.readouterr().out.splitlines() if "HF push" in line] + assert (dataset["finalized"], dataset["pushed"], printed) == expected + + +async def test_without_lerobot_recording_warns_and_the_rollout_still_runs( + hud_env: HudEnv, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + hud_env.set(RECORD_DIR=str(tmp_path / "records")) + + with caplog.at_level(logging.WARNING, logger="hud.agents.robot.dataset"): + async with sim_env("arm", contract()) as env: + run = await record(env) + + assert run.reward == 1.0 + assert [record.message for record in caplog.records if "lerobot" in record.message] == [ + "save=True but lerobot is not installed; streaming telemetry only " + "(pip install 'lerobot[dataset]')" + ] + assert not (tmp_path / "records").exists() From 9005fa6f372d12d4c42e285db8462c4e370f2a31 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 051/248] test(agents): delete the agent tests the scenarios replace The provider, loop, computer-use, native-tool, CLI, gateway, trace and dataset-writer tests reached through get_response on hand-built run states, private formatters and loops, recording subclasses that replaced the RFB primitives, an in-memory SSH emulator, patched hud internals and stand-in Run objects. Their contracts now live in the scenario families under tests/agents; rows that only pinned private structure, restated their own mocks, or checked config defaults are dropped. --- tests/agents/claude/__init__.py | 0 tests/agents/claude/tools/__init__.py | 0 tests/agents/claude/tools/test_computer.py | 227 ----- tests/agents/cli_fakes.py | 111 --- tests/agents/gemini/__init__.py | 0 tests/agents/gemini/tools/__init__.py | 0 tests/agents/gemini/tools/test_computer.py | 124 --- tests/agents/openai/__init__.py | 0 tests/agents/openai/tools/__init__.py | 0 tests/agents/openai/tools/test_computer.py | 174 ---- .../agents/openai/tools/test_strict_schema.py | 74 -- tests/agents/test_base.py | 242 ----- tests/agents/test_claude_agent.py | 463 --------- tests/agents/test_claude_cli_agent.py | 921 ------------------ tests/agents/test_codex_cli_agent.py | 392 -------- tests/agents/test_gemini_agent.py | 279 ------ tests/agents/test_openai_agent.py | 340 ------- tests/agents/test_openai_compatible_agent.py | 113 --- tests/agents/test_provider_native_tools.py | 700 ------------- tests/agents/test_robot_dataset_writer.py | 162 --- tests/agents/test_tool_agent.py | 826 ---------------- tests/agents/test_trace.py | 134 --- 22 files changed, 5282 deletions(-) delete mode 100644 tests/agents/claude/__init__.py delete mode 100644 tests/agents/claude/tools/__init__.py delete mode 100644 tests/agents/claude/tools/test_computer.py delete mode 100644 tests/agents/cli_fakes.py delete mode 100644 tests/agents/gemini/__init__.py delete mode 100644 tests/agents/gemini/tools/__init__.py delete mode 100644 tests/agents/gemini/tools/test_computer.py delete mode 100644 tests/agents/openai/__init__.py delete mode 100644 tests/agents/openai/tools/__init__.py delete mode 100644 tests/agents/openai/tools/test_computer.py delete mode 100644 tests/agents/openai/tools/test_strict_schema.py delete mode 100644 tests/agents/test_base.py delete mode 100644 tests/agents/test_claude_agent.py delete mode 100644 tests/agents/test_claude_cli_agent.py delete mode 100644 tests/agents/test_codex_cli_agent.py delete mode 100644 tests/agents/test_gemini_agent.py delete mode 100644 tests/agents/test_openai_agent.py delete mode 100644 tests/agents/test_openai_compatible_agent.py delete mode 100644 tests/agents/test_provider_native_tools.py delete mode 100644 tests/agents/test_robot_dataset_writer.py delete mode 100644 tests/agents/test_tool_agent.py delete mode 100644 tests/agents/test_trace.py diff --git a/tests/agents/claude/__init__.py b/tests/agents/claude/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/agents/claude/tools/__init__.py b/tests/agents/claude/tools/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/agents/claude/tools/test_computer.py b/tests/agents/claude/tools/test_computer.py deleted file mode 100644 index 2d0f22bdd..000000000 --- a/tests/agents/claude/tools/test_computer.py +++ /dev/null @@ -1,227 +0,0 @@ -"""``ClaudeComputerTool`` — key translation, per-model spec gating, and the -computer-use action dispatch (translation to RFB primitives), without a live VNC. -""" - -from __future__ import annotations - -from io import BytesIO -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, patch - -import mcp.types as mcp_types -from PIL import Image - -from hud.agents.claude.tools.computer import ( - CLAUDE_COMPUTER_SPECS, - GENERIC_COMPUTER_SPEC, - ClaudeComputerTool, - _crop_png, - _hold_keys, - _split_keys, - _translate_key, -) - -if TYPE_CHECKING: - from collections.abc import Iterable -from hud.agents.tools.base import result_text, tool_ok -from hud.capabilities.rfb import WebPScreenshotEncoding - - -class RecordingComputer(ClaudeComputerTool): - """Bypasses RFBTool init; records the primitive calls dispatch makes.""" - - client: Any - - def __init__(self) -> None: - self.calls: list[tuple[Any, ...]] = [] - self.screenshot_encoding = WebPScreenshotEncoding(quality=42) - self.client = SimpleNamespace(width=200, height=100) - - async def screenshot(self) -> Any: - self.calls.append(("screenshot",)) - return tool_ok("shot") - - async def click( - self, - x: int | None = None, - y: int | None = None, - *, - button: Any = "left", - hold_keys: Iterable[str] | None = None, - count: int = 1, - interval_ms: int = 0, - ) -> None: - kw: dict[str, Any] = {} - if button != "left": - kw["button"] = button - if hold_keys is not None: - kw["hold_keys"] = hold_keys - if count != 1: - kw["count"] = count - if interval_ms: - kw["interval_ms"] = interval_ms - self.calls.append(("click", x, y, kw)) - - async def move(self, x: Any, y: Any) -> None: - self.calls.append(("move", x, y)) - - async def mouse_down(self, button: Any = "left") -> None: - self.calls.append(("down", button)) - - async def mouse_up(self, button: Any = "left") -> None: - self.calls.append(("up", button)) - - async def type_text(self, text: Any) -> None: - self.calls.append(("type", text)) - - async def press_keys(self, keys: Any, **kw: Any) -> None: - self.calls.append(("keys", tuple(keys), kw)) - - async def hold_key(self, key: Any, **kw: Any) -> None: - self.calls.append(("hold", key, kw)) - - async def scroll( - self, - x: int | None = None, - y: int | None = None, - *, - scroll_x: int = 0, - scroll_y: int = 0, - hold_keys: Iterable[str] | None = None, - ) -> None: - kw = {"scroll_x": scroll_x, "scroll_y": scroll_y, "hold_keys": hold_keys} - self.calls.append(("scroll", x, y, kw)) - - async def drag(self, path: Any, **kw: Any) -> None: - self.calls.append(("drag", tuple(path), kw)) - - -# ─── key translation helpers ────────────────────────────────────────── - - -def test_translate_key_maps_anthropic_to_x11() -> None: - assert _translate_key("Return") == "Return" - assert _translate_key("ctrl") == "Control_L" - assert _translate_key("ctrl+c") == "Control_L+c" - - -def test_split_and_hold_keys() -> None: - assert _split_keys("ctrl+c") == ["Control_L", "c"] - assert _split_keys(None) == [] - assert _split_keys("") == [] - assert _hold_keys(None) is None - assert _hold_keys("alt") == ["Alt_L"] - - -# ─── spec gating + params ───────────────────────────────────────────── - - -def test_default_spec_per_model() -> None: - spec_45 = ClaudeComputerTool.default_spec("claude-sonnet-4-5-20250101") - assert spec_45 is not None - assert spec_45.api_type == "computer_20250124" - # Unknown Claude model falls back to the latest spec. - spec_unknown = ClaudeComputerTool.default_spec("claude-totally-unknown") - assert spec_unknown is not None - assert spec_unknown.api_type == "computer_20251124" - # Non-Anthropic models get the generic input_schema spec. - assert ClaudeComputerTool.default_spec("qwen/qwen3.8-max") is GENERIC_COMPUTER_SPEC - - -def test_to_params_reflects_spec_version() -> None: - tool = RecordingComputer() - tool.spec = CLAUDE_COMPUTER_SPECS[0] - assert tool.to_params().get("type") == "computer_20251124" - tool.spec = CLAUDE_COMPUTER_SPECS[1] - assert tool.to_params().get("type") == "computer_20250124" - - -def test_to_params_generic_spec_uses_input_schema() -> None: - tool = RecordingComputer() - tool.spec = GENERIC_COMPUTER_SPEC - params: Any = tool.to_params() - assert "type" not in params - assert params["name"] == "computer" - assert "200x100" in params["description"] - assert params["input_schema"]["required"] == ["action"] - assert "left_click" in params["input_schema"]["properties"]["action"]["enum"] - - -# ─── action dispatch ────────────────────────────────────────────────── - - -async def test_left_click_then_screenshot() -> None: - tool = RecordingComputer() - await tool.execute({"action": "left_click", "coordinate": [10, 20], "text": "ctrl"}) - assert tool.calls[0] == ("click", 10, 20, {"hold_keys": ["Control_L"]}) - assert tool.calls[-1] == ("screenshot",) - - -async def test_type_action() -> None: - tool = RecordingComputer() - await tool.execute({"action": "type", "text": "hello"}) - assert ("type", "hello") in tool.calls - - -async def test_key_action_translates_chord() -> None: - tool = RecordingComputer() - await tool.execute({"action": "key", "text": "ctrl+c"}) - assert any(c[0] == "keys" and c[1] == ("Control_L", "c") for c in tool.calls) - - -async def test_mouse_move_and_down() -> None: - tool = RecordingComputer() - await tool.execute({"action": "mouse_move", "coordinate": [5, 6]}) - await tool.execute({"action": "left_mouse_down"}) - assert ("move", 5, 6) in tool.calls - assert ("down", "left") in tool.calls - - -async def test_screenshot_only() -> None: - tool = RecordingComputer() - await tool.execute({"action": "screenshot"}) - assert tool.calls == [("screenshot",)] - - -async def test_key_without_text_errors() -> None: - tool = RecordingComputer() - result = await tool.execute({"action": "key"}) - assert result.isError - - -async def test_unsupported_action_errors() -> None: - tool = RecordingComputer() - result = await tool.execute({"action": "frobnicate"}) - assert result.isError - assert "unsupported" in result_text(result).lower() - - -def test_crop_reports_encoded_mime_type() -> None: - source = BytesIO() - Image.effect_noise((128, 128), 100).convert("RGB").save(source, format="PNG") - - cropped, mime_type = _crop_png( - source.getvalue(), - (0, 0, 128, 128), - WebPScreenshotEncoding(quality=42), - ) - - assert mime_type == "image/webp" - assert cropped.startswith(b"RIFF") - - -async def test_zoom_reports_encoded_mime_type() -> None: - tool = RecordingComputer() - tool.client.screenshot_png = AsyncMock(return_value=(b"png", "image/png")) - - with patch( - "hud.agents.claude.tools.computer._crop_png", - return_value=(b"webp", "image/webp"), - ) as crop: - result = await tool._zoom({"region": [0, 0, 10, 10]}) - - tool.client.screenshot_png.assert_awaited_once_with("image/png") - crop.assert_called_once_with(b"png", (0, 0, 10, 10), tool.screenshot_encoding) - image = next(block for block in result.content if isinstance(block, mcp_types.ImageContent)) - assert image.mimeType == "image/webp" diff --git a/tests/agents/cli_fakes.py b/tests/agents/cli_fakes.py deleted file mode 100644 index a1cb5c246..000000000 --- a/tests/agents/cli_fakes.py +++ /dev/null @@ -1,111 +0,0 @@ -from __future__ import annotations - -import asyncio -from types import SimpleNamespace -from typing import Any - - -class FakeReader: - def __init__(self, value: str, *, pause_after: int | None = None) -> None: - self._raw = value.encode() - self._lines = self._raw.splitlines(keepends=True) - self._pause_after = pause_after - self._index = 0 - self.blocked = asyncio.Event() - self.release = asyncio.Event() - - async def readline(self) -> bytes: - if self._pause_after == self._index: - self.blocked.set() - await self.release.wait() - self._pause_after = None - if self._index == len(self._lines): - return b"" - line = self._lines[self._index] - self._index += 1 - return line - - async def read(self) -> bytes: - return self._raw - - -class FakeWriter: - def __init__(self) -> None: - self.data = bytearray() - self.eof = False - - def write(self, data: bytes) -> None: - self.data.extend(data) - - async def drain(self) -> None: - pass - - def write_eof(self) -> None: - self.eof = True - - -class FakeProcess: - def __init__( - self, - stdout: str, - *, - stderr: str = "", - exit_status: int | None = 0, - returncode: int | None = None, - pause_after: int | None = None, - ) -> None: - self.stdin = FakeWriter() - self.stdout = FakeReader(stdout, pause_after=pause_after) - self.stderr = FakeReader(stderr) - self.exit_status = exit_status - self.returncode = exit_status if returncode is None else returncode - self.closed = False - self.terminated = False - - def terminate(self) -> None: - self.terminated = True - - def close(self) -> None: - self.closed = True - - async def wait_closed(self) -> None: - pass - - -class FakeClient: - """The slice of a run's client the CLI agents use: SSH, the manifest, routed bindings.""" - - def __init__( - self, - ssh: Any, - bindings: list[Any] | None = None, - routed: dict[str, Any] | None = None, - ) -> None: - self.ssh = ssh - self.manifest = SimpleNamespace(bindings=[ssh.capability, *(bindings or [])]) - self.routed = routed or {} - - async def open(self, ref: str) -> Any: - assert ref == "ssh" - return self.ssh - - def binding(self, ref: str) -> Any: - return self.routed[ref] - - -def fake_run( - client: FakeClient | None = None, - prompt_text: str = "", - connections: dict[str, Any] | None = None, -) -> Any: - trace = SimpleNamespace(status=None, content="", extra={}) - steps: list[Any] = [] - return SimpleNamespace( - trace=trace, - record=steps.append, - steps=steps, - client=client, - prompt_text=prompt_text, - runtime_config=None, - connections=connections or {}, - ) diff --git a/tests/agents/gemini/__init__.py b/tests/agents/gemini/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/agents/gemini/tools/__init__.py b/tests/agents/gemini/tools/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/agents/gemini/tools/test_computer.py b/tests/agents/gemini/tools/test_computer.py deleted file mode 100644 index 305fe876b..000000000 --- a/tests/agents/gemini/tools/test_computer.py +++ /dev/null @@ -1,124 +0,0 @@ -"""``GeminiComputerTool`` — predefined computer-use functions dispatched to RFB. - -No live VNC: a recording subclass captures the primitive calls. -""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, patch - -from hud.agents.gemini.tools.computer import GEMINI_COMPUTER_SPEC, GeminiComputerTool -from hud.agents.tools.base import tool_ok - -if TYPE_CHECKING: - from collections.abc import Iterable - - -class RecordingGemini(GeminiComputerTool): - client: Any - - def __init__(self) -> None: - self.calls: list[tuple[Any, ...]] = [] - self.client = SimpleNamespace(width=400, height=300) - self.excluded_predefined_functions = [] - - async def screenshot(self) -> Any: - self.calls.append(("screenshot",)) - return tool_ok("shot") - - async def click( - self, - x: int | None = None, - y: int | None = None, - *, - button: Any = "left", - hold_keys: Iterable[str] | None = None, - count: int = 1, - interval_ms: int = 0, - ) -> None: - self.calls.append(("click", x, y)) - - async def move(self, x: Any, y: Any) -> None: - self.calls.append(("move", x, y)) - - async def type_text(self, text: Any) -> None: - self.calls.append(("type", text)) - - async def press_keys(self, keys: Any, **kw: Any) -> None: - self.calls.append(("keys", tuple(keys))) - - async def scroll( - self, - x: int | None = None, - y: int | None = None, - *, - scroll_x: int = 0, - scroll_y: int = 0, - hold_keys: Iterable[str] | None = None, - ) -> None: - kw = {"scroll_x": scroll_x, "scroll_y": scroll_y, "hold_keys": hold_keys} - self.calls.append(("scroll", x, y, kw)) - - async def drag(self, path: Any, **kw: Any) -> None: - self.calls.append(("drag", tuple(path))) - - -def test_default_spec() -> None: - assert GeminiComputerTool.default_spec("any") is GEMINI_COMPUTER_SPEC - - -async def test_click_at() -> None: - tool = RecordingGemini() - await tool.execute({"action": "click_at", "x": 10, "y": 20}) - assert ("click", 10, 20) in tool.calls - assert tool.calls[-1] == ("screenshot",) - - -async def test_type_text_at_without_clear() -> None: - tool = RecordingGemini() - await tool.execute( - {"action": "type_text_at", "x": 5, "y": 6, "text": "hi", "clear_before_typing": False} - ) - assert ("move", 5, 6) in tool.calls - assert ("type", "hi") in tool.calls - - -async def test_scroll_at_down() -> None: - tool = RecordingGemini() - await tool.execute({"action": "scroll_at", "x": 0, "y": 0, "direction": "down", "magnitude": 3}) - scrolls = [c for c in tool.calls if c[0] == "scroll"] - assert scrolls and scrolls[0][3]["scroll_y"] == 3 - - -async def test_key_combination() -> None: - tool = RecordingGemini() - await tool.execute({"action": "key_combination", "keys": "ctrl+c"}) - assert ("keys", ("Control_L", "c")) in tool.calls - - -async def test_key_combination_requires_string() -> None: - tool = RecordingGemini() - assert (await tool.execute({"action": "key_combination", "keys": 123})).isError - - -async def test_wait_and_drag() -> None: - tool = RecordingGemini() - with patch("hud.agents.tools.rfb.asyncio.sleep", new_callable=AsyncMock) as sleep: - await tool.execute({"action": "wait_5_seconds"}) - await tool.execute( - {"action": "drag_and_drop", "x": 50, "y": 50, "destination_x": 100, "destination_y": 100} - ) - sleep.assert_awaited_once_with(5) - assert any(c[0] == "drag" for c in tool.calls) - - -async def test_missing_action_errors() -> None: - tool = RecordingGemini() - assert (await tool.execute({})).isError - - -async def test_unknown_action_errors() -> None: - tool = RecordingGemini() - assert (await tool.execute({"action": "fly_to_moon"})).isError diff --git a/tests/agents/openai/__init__.py b/tests/agents/openai/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/agents/openai/tools/__init__.py b/tests/agents/openai/tools/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/agents/openai/tools/test_computer.py b/tests/agents/openai/tools/test_computer.py deleted file mode 100644 index a42221ebf..000000000 --- a/tests/agents/openai/tools/test_computer.py +++ /dev/null @@ -1,174 +0,0 @@ -"""``OpenAIComputerTool`` — key mapping + computer-call dispatch to RFB primitives. - -No live VNC: a recording subclass captures the primitive calls. -""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any -from unittest.mock import AsyncMock, patch - -import mcp.types as mcp_types -import pytest - -from hud.agents.openai.tools.computer import ( - OpenAIComputerTool, - _hold_keys, - _map_key, -) -from hud.agents.tools.base import result_text -from hud.types import MCPToolResult - -if TYPE_CHECKING: - from collections.abc import Iterable - - -class RecordingOpenAI(OpenAIComputerTool): - client: Any - - def __init__(self) -> None: - self.calls: list[tuple[Any, ...]] = [] - self.client = SimpleNamespace(width=200, height=100) - - async def screenshot(self) -> MCPToolResult: - self.calls.append(("screenshot",)) - return MCPToolResult( - content=[mcp_types.ImageContent(type="image", data="c2hvdA==", mimeType="image/png")], - ) - - async def click( - self, - x: int | None = None, - y: int | None = None, - *, - button: Any = "left", - hold_keys: Iterable[str] | None = None, - count: int = 1, - interval_ms: int = 0, - ) -> None: - kw = {"button": button, "hold_keys": hold_keys} - if count != 1: - kw["count"] = count - if interval_ms: - kw["interval_ms"] = interval_ms - self.calls.append(("click", x, y, kw)) - - async def move(self, x: Any, y: Any) -> None: - self.calls.append(("move", x, y)) - - async def type_text(self, text: Any) -> None: - self.calls.append(("type", text)) - - async def press_keys(self, keys: Any, **kw: Any) -> None: - self.calls.append(("keys", tuple(keys))) - - async def scroll( - self, - x: int | None = None, - y: int | None = None, - *, - scroll_x: int = 0, - scroll_y: int = 0, - hold_keys: Iterable[str] | None = None, - ) -> None: - kw = {"scroll_x": scroll_x, "scroll_y": scroll_y, "hold_keys": hold_keys} - self.calls.append(("scroll", x, y, kw)) - - async def drag(self, path: Any, **kw: Any) -> None: - self.calls.append(("drag", tuple(path))) - - -def test_key_mapping() -> None: - assert _map_key("ctrl") == "Control_L" - assert _map_key("x") == "x" - assert _map_key("ESC") == "Escape" - assert _map_key("RIGHT") == "Right" - assert _hold_keys(["ctrl", "c"]) == ["Control_L", "c"] - assert _hold_keys(None) is None - - -def test_to_params() -> None: - assert RecordingOpenAI().to_params() == {"type": "computer"} - - -async def test_click_returns_screenshot() -> None: - tool = RecordingOpenAI() - result = await tool.execute({"type": "click", "x": 1, "y": 2, "button": "left"}) - assert ("click", 1, 2, {"button": "left", "hold_keys": None}) in tool.calls - assert not result.isError - - -async def test_type_and_keypress() -> None: - tool = RecordingOpenAI() - await tool.execute({"type": "type", "text": "hi"}) - await tool.execute({"type": "keypress", "keys": ["ctrl", "c"]}) - assert ("type", "hi") in tool.calls - assert ("keys", ("Control_L", "c")) in tool.calls - - -@pytest.mark.parametrize(("ms", "seconds"), [(500, 0.5), (0, 0)]) -async def test_drag_and_wait(ms: int, seconds: float) -> None: - tool = RecordingOpenAI() - await tool.execute({"type": "drag", "path": [{"x": 0, "y": 0}, {"x": 5, "y": 5}]}) - with patch("hud.agents.tools.rfb.asyncio.sleep", new_callable=AsyncMock) as sleep: - result = await tool.execute({"type": "wait", "ms": ms}) - assert ("drag", ((0, 0), (5, 5))) in tool.calls - assert not result.isError - sleep.assert_awaited_once_with(seconds) - - -async def test_response_action_returns_text_and_screenshot() -> None: - tool = RecordingOpenAI() - result = await tool.execute({"type": "response", "text": "all done"}) - assert result_text(result) == "all done" - assert any(isinstance(block, mcp_types.ImageContent) for block in result.content) - assert tool.calls[-1] == ("screenshot",) - - -async def test_actions_list_runs_each() -> None: - tool = RecordingOpenAI() - await tool.execute( - {"actions": [{"type": "move", "x": 3, "y": 4}, {"type": "type", "text": "a"}]} - ) - assert ("move", 3, 4) in tool.calls - assert ("type", "a") in tool.calls - - -@pytest.mark.parametrize( - "arguments", [{"actions": []}, {"actions": "invalid", "type": "move", "x": 3, "y": 4}] -) -async def test_invalid_actions_errors(arguments: dict[str, Any]) -> None: - tool = RecordingOpenAI() - assert (await tool.execute(arguments)).isError - assert tool.calls == [("screenshot",)] - - -async def test_invalid_type_errors() -> None: - tool = RecordingOpenAI() - assert (await tool.execute({"type": "frobnicate"})).isError - assert (await tool.execute({})).isError - - -@pytest.mark.parametrize( - "action", - [ - {"type": "type"}, - {"type": "move", "x": 10}, - {"type": "keypress", "keys": "ESC"}, - {"type": "keypress", "keys": []}, - {"type": "drag", "path": [{"x": 1, "y": 2}, {"x": 3}]}, - {"type": "click", "button": 1}, - {"type": "wait", "ms": -1}, - ], -) -async def test_invalid_action_stops_batch_and_returns_screenshot(action: dict[str, Any]) -> None: - tool = RecordingOpenAI() - result = await tool.execute( - {"actions": [{"type": "move", "x": 3, "y": 4}, action, {"type": "type", "text": "later"}]}, - ) - - assert result.isError - assert result_text(result) - assert any(isinstance(block, mcp_types.ImageContent) for block in result.content) - assert tool.calls == [("move", 3, 4), ("screenshot",)] diff --git a/tests/agents/openai/tools/test_strict_schema.py b/tests/agents/openai/tools/test_strict_schema.py deleted file mode 100644 index f0ff27370..000000000 --- a/tests/agents/openai/tools/test_strict_schema.py +++ /dev/null @@ -1,74 +0,0 @@ -"""``ensure_strict_json_schema`` — coerce a JSON schema to OpenAI strict-mode form.""" - -from __future__ import annotations - -from typing import Any - -from hud.agents.openai.tools.strict_schema import ensure_strict_json_schema - - -def test_empty_schema_becomes_closed_object() -> None: - result = ensure_strict_json_schema({}) - assert result == { - "additionalProperties": False, - "type": "object", - "properties": {}, - "required": [], - } - - -def test_object_gets_additional_properties_false_and_all_required() -> None: - schema: dict[str, Any] = { - "type": "object", - "properties": {"a": {"type": "string"}, "b": {"type": "integer"}}, - } - result = ensure_strict_json_schema(schema) - - assert result["additionalProperties"] is False - assert set(result["required"]) == {"a", "b"} # strict mode requires every property - - -def test_additional_properties_true_is_converted_to_false() -> None: - result = ensure_strict_json_schema( - {"type": "object", "properties": {}, "additionalProperties": True} - ) - assert result["additionalProperties"] is False - - -def test_unsupported_keywords_are_stripped() -> None: - schema: dict[str, Any] = { - "type": "object", - "properties": { - "name": { - "type": "string", - "title": "Name", # unsupported meta keyword - "minLength": 1, # unsupported string constraint - }, - }, - } - name_schema = ensure_strict_json_schema(schema)["properties"]["name"] - assert "title" not in name_schema - assert "minLength" not in name_schema - assert name_schema["type"] == "string" - - -def test_nested_objects_are_recursively_strict() -> None: - schema: dict[str, Any] = { - "type": "object", - "properties": { - "inner": {"type": "object", "properties": {"x": {"type": "number"}}}, - }, - } - inner = ensure_strict_json_schema(schema)["properties"]["inner"] - assert inner["additionalProperties"] is False - assert inner["required"] == ["x"] - - -def test_is_idempotent() -> None: - schema: dict[str, Any] = { - "type": "object", - "properties": {"a": {"type": "string", "title": "A"}}, - } - once = ensure_strict_json_schema(dict(schema)) - twice = ensure_strict_json_schema(once) - assert once == twice diff --git a/tests/agents/test_base.py b/tests/agents/test_base.py deleted file mode 100644 index b0ce8d580..000000000 --- a/tests/agents/test_base.py +++ /dev/null @@ -1,242 +0,0 @@ -"""The agent base contract: the ``Agent`` ABC and gateway routing. - -These cover the model-agnostic surface that doesn't need provider SDKs or -network: the stateless ``Agent`` contract and ``AgentType`` / ``create_agent`` -resolution. -""" - -from __future__ import annotations - -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from hud.agents import ( - ClaudeCLIAgent, - CodexCLIAgent, - OpenAIAgent, - OpenAIChatAgent, - create_agent, -) -from hud.agents.base import Agent -from hud.agents.types import AgentConfig, OpenAIConfig -from hud.types import AgentType -from hud.utils.exceptions import HudAuthenticationError - - -class _FillingAgent(Agent): - async def __call__(self, run: Any) -> None: - run.trace.content = "done" - - -def test_agent_requires_call_implementation() -> None: - with pytest.raises(TypeError): - Agent() - - -async def test_agent_call_fills_trace() -> None: - from types import SimpleNamespace - - run = SimpleNamespace(trace=SimpleNamespace(content="")) - await _FillingAgent()(run) - assert run.trace.content == "done" - - -def test_agent_type_maps_value_to_class_and_provider() -> None: - assert AgentType("openai").cls is OpenAIAgent - assert AgentType("openai_compatible").cls is OpenAIChatAgent - assert isinstance(AgentType("openai").gateway_provider, str) - - -def test_agent_type_registers_cli_agent() -> None: - assert AgentType("claude_cli").cls is ClaudeCLIAgent - assert AgentType.of(ClaudeCLIAgent()) == AgentType.CLAUDE_CLI - assert AgentType("codex_cli").cls is CodexCLIAgent - assert AgentType.of(CodexCLIAgent()) == AgentType.CODEX_CLI - - -@pytest.mark.parametrize("agent", [ClaudeCLIAgent(), CodexCLIAgent()]) -def test_cli_agent_load_rebuilds_dumped_config(agent: ClaudeCLIAgent | CodexCLIAgent) -> None: - loaded = type(agent).load(agent.dump()) - - assert type(loaded) is type(agent) - assert loaded.config == agent.config - - -def test_custom_agent_round_trips_base_config() -> None: - agent = _FillingAgent(AgentConfig(model="custom", timeout_seconds=30)) - - loaded = _FillingAgent.load(agent.dump()) - - assert loaded.config == agent.config - - -def test_missing_provider_dependency_points_at_agents_extra( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Base installs (no [agents] extra) get an actionable error, not a raw import failure.""" - import sys - - import hud.agents - - class _Blocker: - def find_spec(self, fullname: str, path: Any = None, target: Any = None) -> None: - if fullname == "anthropic" or fullname.startswith("anthropic."): - raise ModuleNotFoundError(f"No module named {fullname!r}", name=fullname) - return None - - for module in list(sys.modules): - if module == "anthropic" or module.startswith(("anthropic.", "hud.agents.claude")): - monkeypatch.delitem(sys.modules, module) - monkeypatch.setattr(sys, "meta_path", [_Blocker(), *sys.meta_path]) - if "ClaudeAgent" in vars(hud.agents): # drop any cached lazy export - monkeypatch.delitem(hud.agents.__dict__, "ClaudeAgent") - - with pytest.raises(ImportError, match=r"hud\[agents\]"): - _ = hud.agents.ClaudeAgent - - with pytest.raises(ImportError, match=r"hud\[agents\]"): - _ = AgentType.CLAUDE.cls - - -@pytest.fixture(autouse=True) -def gateway_api_key(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("hud.agents.settings.api_key", "test-key") - - -def test_create_agent_unknown_model_raises(monkeypatch: pytest.MonkeyPatch) -> None: - # No gateway models available -> a bare unknown model can't be resolved. - monkeypatch.setattr("hud.utils.gateway.list_gateway_models", lambda *_: []) - with pytest.raises(ValueError, match="not found"): - create_agent("totally-unknown-model-xyz") - - -def test_create_agent_value_shortcut_leaves_client_out_of_config( - monkeypatch: pytest.MonkeyPatch, -) -> None: - sentinel = object() - - def _build_client(_provider: str) -> object: - return sentinel - - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", _build_client) - - agent = create_agent("openai") # AgentType.OPENAI shortcut - - assert isinstance(agent, OpenAIAgent) - assert agent.config.model == OpenAIConfig().model - assert agent.config.model_client is None - assert agent.openai_client is sentinel - assert agent.dump()["prompt_cache_key"] == agent.config.prompt_cache_key - - -def test_create_agent_uses_the_gateway_even_with_a_provider_key( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr("hud.utils.gateway.settings.openai_api_key", "provider-key") - direct = MagicMock(return_value=object()) - via_gateway = MagicMock(return_value=object()) - monkeypatch.setattr("hud.utils.gateway.AsyncOpenAI", direct) - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", via_gateway) - - agent = create_agent("openai") - - assert isinstance(agent, OpenAIAgent) - assert agent.openai_client is via_gateway.return_value - direct.assert_not_called() - # The same config built directly honours the provider key. - assert OpenAIAgent(OpenAIConfig()).openai_client is direct.return_value - # Routing is config, not a client, so it survives serialization. - reloaded = OpenAIAgent.load(agent.dump()) - assert reloaded.config.gateway is True - assert reloaded.openai_client is via_gateway.return_value - - -def test_create_agent_resolves_gateway_model_metadata( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.utils.gateway import GatewayModelInfo, GatewayProviderInfo - - model = GatewayModelInfo( - id="ft:custom-123", - model_name="gpt-5.5", - sdk_agent_type="openai_compatible", - provider=GatewayProviderInfo(name="openai"), - ) - monkeypatch.setattr("hud.utils.gateway.list_gateway_models", lambda *_: [model]) - - def _build_client(_provider: str) -> object: - return object() - - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", _build_client) - - agent = create_agent("ft:custom-123") - - assert isinstance(agent, OpenAIChatAgent) - assert agent.config.model == "gpt-5.5" # resolved to the model's real name - assert agent.config.model_client is None - - -@pytest.mark.parametrize( - ("alias", "canonical"), - [ - ("deepseek-v4-flash", "deepseek/deepseek-v4-flash"), # slug without provider prefix - ("glm-5.2", "z-ai/glm-5.2"), - ("kimi-k2.6", "moonshotai/kimi-k2.6"), - ("minimax-m3", "MiniMax-M3"), # case-insensitive - ], -) -def test_create_agent_accepts_gateway_model_aliases( - alias: str, - canonical: str, - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.utils.gateway import GatewayModelInfo, GatewayProviderInfo - - model = GatewayModelInfo( - id=canonical, - model_name=canonical, - sdk_agent_type="openai_compatible", - provider=GatewayProviderInfo(name="openai"), - ) - monkeypatch.setattr("hud.utils.gateway.list_gateway_models", lambda *_: [model]) - - def _build_client(_provider: str) -> object: - return object() - - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", _build_client) - - agent = create_agent(alias) - - assert isinstance(agent, OpenAIChatAgent) - assert agent.config.model == canonical - - -def test_create_agent_requires_hud_api_key(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("hud.agents.settings.api_key", None) - - with pytest.raises(HudAuthenticationError, match="HUD_API_KEY"): - create_agent("openai") - - -@pytest.mark.parametrize( - ("credential", "value"), - [ - ("api_key", "provider-key"), - ("api_key", None), - ("base_url", "https://provider.example"), - ("base_url", None), - ], -) -@pytest.mark.parametrize("hud_api_key", ["hud-key", None]) -def test_create_agent_rejects_direct_provider_credentials( - monkeypatch: pytest.MonkeyPatch, - credential: str, - value: str | None, - hud_api_key: str | None, -) -> None: - monkeypatch.setattr("hud.agents.settings.api_key", hud_api_key) - - with pytest.raises(ValueError, match="provider agent directly"): - create_agent("openai_compatible", **{credential: value}) diff --git a/tests/agents/test_claude_agent.py b/tests/agents/test_claude_agent.py deleted file mode 100644 index 0754c85b2..000000000 --- a/tests/agents/test_claude_agent.py +++ /dev/null @@ -1,463 +0,0 @@ -"""``ClaudeAgent`` — ``get_response`` parsing over a fake streaming Messages client, -plus the pure ``_citation`` / ``_cache_last_user_block`` helpers. -""" - -from __future__ import annotations - -import base64 -import random -import struct -import zlib -from io import BytesIO -from types import SimpleNamespace -from typing import Any, cast -from unittest.mock import AsyncMock, Mock, call - -import httpx -import httpx2 -import pytest -from anthropic import APIStatusError -from mcp.types import ImageContent -from PIL import Image - -from hud.agents.claude.agent import ClaudeAgent -from hud.agents.claude.tools.computer import ClaudeComputerTool -from hud.agents.tool_agent import RunState -from hud.capabilities import RFBClient -from hud.types import MCPToolCall, MCPToolResult - - -class FakeStream: - def __init__(self, outcome: Any) -> None: - self._outcome = outcome - - async def __aenter__(self) -> FakeStream: - return self - - async def __aexit__(self, *_a: Any) -> bool: - return False - - def __aiter__(self) -> FakeStream: - return self - - async def __anext__(self) -> Any: - if isinstance(self._outcome, BaseException): - raise self._outcome - raise StopAsyncIteration - - async def get_final_message(self) -> Any: - return self._outcome - - -class FakeMessages: - def __init__(self, *outcomes: Any) -> None: - self._outcomes = list(outcomes) - self.calls = 0 - - def stream(self, **_kwargs: Any) -> FakeStream: - outcome = self._outcomes[self.calls] - self.calls += 1 - return FakeStream(outcome) - - -class FakeAnthropic: - def __init__(self, *outcomes: Any) -> None: - self.beta = SimpleNamespace(messages=FakeMessages(*outcomes)) - - -def _final(*content: Any, stop_reason: str, stop_details: Any = None) -> Any: - """A fake ``BetaMessage``: content blocks plus the always-present envelope.""" - return SimpleNamespace( - content=list(content), - stop_reason=stop_reason, - stop_details=stop_details, - model="claude-test-v9", - usage=SimpleNamespace(input_tokens=11, output_tokens=7, cache_read_input_tokens=3), - ) - - -def _agent(*outcomes: Any) -> ClaudeAgent: - from hud.agents.types import ClaudeConfig - - return ClaudeAgent( - ClaudeConfig(model="claude-test", max_tokens=1024, model_client=FakeAnthropic(*outcomes)) - ) - - -def _state(agent: ClaudeAgent) -> Any: - from hud.agents.tool_agent import RunState - - return RunState(messages=[agent._format_message("user", "go")]) - - -def _inline_error(type_: str, *, headers: dict[str, str] | None = None) -> APIStatusError: - body = {"type": "error", "error": {"type": type_, "message": "stream failed"}} - response = httpx.Response( - 200, - headers=headers, - request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), - ) - return APIStatusError(str(body), response=response, body=body) - - -def test_format_message_shape() -> None: - agent = _agent(SimpleNamespace(content=[], stop_reason="end_turn")) - msg = agent._format_message("assistant", "hi") - assert msg["role"] == "assistant" - - -async def test_get_response_text_and_tool_use() -> None: - final = _final( - SimpleNamespace(type="text", text="hello", citations=None), - SimpleNamespace(type="tool_use", id="t1", name="bash", input={"command": "ls"}), - stop_reason="tool_use", - ) - agent = _agent(final) - state = _state(agent) - - result = await agent.get_response(state) - - assert result.content == "hello" - assert [tc.name for tc in result.tool_calls] == ["bash"] - assert result.tool_calls[0].arguments == {"command": "ls"} - assert result.done is False - assert result.finish_reason == "tool_use" - # Model and usage are normalized off the provider response. - assert result.model == "claude-test-v9" - assert result.usage is not None - assert result.usage.prompt_tokens == 11 - assert result.usage.completion_tokens == 7 - assert result.usage.cached_tokens == 3 - - -async def test_get_response_done_on_text_only() -> None: - final = _final( - SimpleNamespace(type="text", text="done", citations=None), - stop_reason="end_turn", - ) - agent = _agent(final) - result = await agent.get_response(_state(agent)) - assert result.done is True - assert result.content == "done" - assert result.tool_calls == [] - assert result.refusal is None - - -async def test_get_response_surfaces_refusal_explanation() -> None: - explanation = ( - "This request triggered restrictions on violative cyber content and was " - "blocked under Anthropic's Usage Policy." - ) - final = _final( - stop_reason="refusal", - stop_details=SimpleNamespace( - type="refusal", - category="cyber", - explanation=explanation, - ), - ) - agent = _agent(final) - result = await agent.get_response(_state(agent)) - - assert result.finish_reason == "refusal" - assert result.refusal == explanation - assert result.done is True - assert result.tool_calls == [] - - -async def test_get_response_surfaces_refusal_category_when_explanation_missing() -> None: - final = _final( - stop_reason="refusal", - stop_details=SimpleNamespace( - type="refusal", - category="cyber", - explanation=None, - ), - ) - agent = _agent(final) - result = await agent.get_response(_state(agent)) - - assert result.finish_reason == "refusal" - assert result.refusal is not None - assert "cyber" in result.refusal - assert result.done is True - assert result.tool_calls == [] - - -async def test_get_response_collects_thinking() -> None: - final = _final( - SimpleNamespace(type="thinking", thinking="pondering"), - SimpleNamespace(type="text", text="answer", citations=None), - stop_reason="end_turn", - ) - agent = _agent(final) - result = await agent.get_response(_state(agent)) - assert result.reasoning == "pondering" - - -@pytest.mark.parametrize("error_type", ["upstream_error", "timeout_error"]) -async def test_get_response_retries_transient_inline_stream_error( - monkeypatch: pytest.MonkeyPatch, - error_type: str, -) -> None: - final = _final( - SimpleNamespace(type="text", text="recovered", citations=None), - stop_reason="end_turn", - ) - agent = _agent(_inline_error(error_type), _inline_error(error_type), final) - state = _state(agent) - sleep = AsyncMock() - jitter = Mock(side_effect=[0.25, 1.5]) - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", sleep) - monkeypatch.setattr("hud.agents.claude.agent._STREAM_JITTER.uniform", jitter) - - result = await agent.get_response(state) - - messages = cast("FakeMessages", cast("Any", agent.anthropic_client).beta.messages) - assert messages.calls == 3 - assert result.content == "recovered" - assert len(state.messages) == 2 - assert sleep.await_args_list == [call(0.25), call(1.5)] - assert jitter.call_args_list == [call(0.0, 1.0), call(0.0, 2.0)] - - -async def test_get_response_raises_after_transient_stream_retry_is_exhausted( - monkeypatch: pytest.MonkeyPatch, -) -> None: - agent = _agent( - _inline_error("gateway_timeout"), - _inline_error("gateway_timeout"), - _inline_error("gateway_timeout"), - ) - state = _state(agent) - sleep = AsyncMock() - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", sleep) - monkeypatch.setattr("hud.agents.claude.agent._STREAM_JITTER.uniform", Mock(return_value=0.0)) - - with pytest.raises(APIStatusError, match="gateway_timeout"): - await agent.get_response(state) - - messages = cast("FakeMessages", cast("Any", agent.anthropic_client).beta.messages) - assert messages.calls == 3 - assert len(state.messages) == 1 - assert sleep.await_count == 2 - - -@pytest.mark.parametrize( - ("headers", "expected_delay"), - [ - ({"retry-after": "1.75"}, 1.75), - ({"retry-after-ms": "1750"}, 1.75), - ({"retry-after": "60"}, 5.0), - ], - ids=["seconds", "milliseconds", "capped"], -) -async def test_get_response_honors_retry_after( - monkeypatch: pytest.MonkeyPatch, - headers: dict[str, str], - expected_delay: float, -) -> None: - final = _final( - SimpleNamespace(type="text", text="recovered", citations=None), - stop_reason="end_turn", - ) - agent = _agent(_inline_error("overloaded_error", headers=headers), final) - sleep = AsyncMock() - jitter = Mock() - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", sleep) - monkeypatch.setattr("hud.agents.claude.agent._STREAM_JITTER.uniform", jitter) - - result = await agent.get_response(_state(agent)) - - assert result.content == "recovered" - sleep.assert_awaited_once_with(expected_delay) - jitter.assert_not_called() - - -@pytest.mark.parametrize( - "error", - [ - httpx.ReadError( - "stream interrupted", - request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), - ), - httpx.ReadTimeout( - "stream interrupted", - request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"), - ), - httpx2.ReadError( - "stream interrupted", - request=httpx2.Request("POST", "https://api.anthropic.com/v1/messages"), - ), - httpx2.ReadTimeout( - "stream interrupted", - request=httpx2.Request("POST", "https://api.anthropic.com/v1/messages"), - ), - ], - ids=["httpx-read-error", "httpx-read-timeout", "httpx2-read-error", "httpx2-read-timeout"], -) -async def test_get_response_retries_interrupted_stream( - monkeypatch: pytest.MonkeyPatch, - error: httpx.TransportError | httpx2.TransportError, -) -> None: - final = _final( - SimpleNamespace(type="text", text="recovered", citations=None), - stop_reason="end_turn", - ) - agent = _agent(error, final) - state = _state(agent) - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", AsyncMock()) - - result = await agent.get_response(state) - - messages = cast("FakeMessages", cast("Any", agent.anthropic_client).beta.messages) - assert messages.calls == 2 - assert result.content == "recovered" - assert len(state.messages) == 2 - - -async def test_get_response_does_not_retry_non_transient_inline_stream_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - agent = _agent(_inline_error("invalid_request_error")) - state = _state(agent) - sleep = AsyncMock() - monkeypatch.setattr("hud.agents.claude.agent.asyncio.sleep", sleep) - - with pytest.raises(APIStatusError, match="invalid_request_error"): - await agent.get_response(state) - - messages = cast("FakeMessages", cast("Any", agent.anthropic_client).beta.messages) - assert messages.calls == 1 - assert len(state.messages) == 1 - sleep.assert_not_awaited() - - -def test_citation_char_location() -> None: - raw = SimpleNamespace( - type="char_location", - cited_text="quote", - document_index=2, - document_title="doc", - start_char_index=0, - end_char_index=5, - ) - cit = ClaudeAgent._citation(cast("Any", raw)) - assert cit.type == "document_citation" - assert cit.source == "2" - assert cit.start_index == 0 - - -def test_cache_last_user_block_marks_content() -> None: - agent = _agent(SimpleNamespace(content=[], stop_reason="end_turn")) - messages = [agent._format_message("user", "hi")] - out = ClaudeAgent._cache_last_user_block(messages) - content = cast("list[Any]", out[-1]["content"]) - block = cast("dict[str, Any]", content[0]) - assert block.get("cache_control") == {"type": "ephemeral"} - - -def _image_result(data: bytes, *, computer: bool = False) -> Any: - agent = _agent() - state = RunState() - if computer: - spec = ClaudeComputerTool.default_spec("claude") - assert spec is not None - tool = ClaudeComputerTool(spec=spec, client=Mock(spec=RFBClient)) - state.tools[tool.provider_name] = tool - name = tool.provider_name - else: - name = "read_file" - content = ImageContent(type="image", mimeType="image/png", data=base64.b64encode(data).decode()) - result = MCPToolResult(content=[content]) - original = result.model_dump() - message = agent._format_result( - MCPToolCall(id="image-call", name=name, arguments={}), result, state - ) - assert result.model_dump() == original - assert isinstance(message, dict) - return next(iter(message["content"])) - - -@pytest.mark.parametrize("size", [(4000, 2000), (2000, 4000), (2000, 2000), (32, 24)]) -def test_tool_images_fit_claude_limits_without_upscaling(size: tuple[int, int]) -> None: - buffer = BytesIO() - Image.new("RGB", size, "white").save(buffer, format="PNG") - result = _image_result(buffer.getvalue()) - assert not result["is_error"] - source = result["content"][0]["source"] - assert len(source["data"]) <= 5_000_000 - with Image.open(BytesIO(base64.b64decode(source["data"]))) as image: - assert max(image.size) <= 1568 - assert image.width * image.height <= 1_150_000 - assert image.width <= size[0] and image.height <= size[1] - assert image.width / image.height == pytest.approx(size[0] / size[1], rel=0.005) - - -def test_claude_compresses_noisy_transparent_tool_images() -> None: - size = (1072, 1072) - image = Image.frombytes("RGBA", size, random.Random(0).randbytes(size[0] * size[1] * 4)) - buffer = BytesIO() - image.save(buffer, format="PNG") - assert len(base64.b64encode(buffer.getvalue())) > 5_000_000 - result = _image_result(buffer.getvalue()) - assert not result["is_error"] - source = result["content"][0]["source"] - assert source["media_type"] == "image/jpeg" - assert len(source["data"]) <= 5_000_000 - with Image.open(BytesIO(base64.b64decode(source["data"]))) as viewed: - assert viewed.size == size - assert viewed.mode == "RGB" - - -def test_claude_applies_image_orientation() -> None: - image = Image.new("RGB", (40, 20), "red") - exif = Image.Exif() - exif[274] = 6 - buffer = BytesIO() - image.save(buffer, format="JPEG", exif=exif) - source = _image_result(buffer.getvalue())["content"][0]["source"] - with Image.open(BytesIO(base64.b64decode(source["data"]))) as viewed: - assert viewed.size == (20, 40) - assert viewed.getexif().get(274, 1) == 1 - - -def test_claude_preserves_small_png_transparency() -> None: - buffer = BytesIO() - Image.new("RGBA", (32, 24), (255, 0, 0, 128)).save(buffer, format="PNG") - source = _image_result(buffer.getvalue())["content"][0]["source"] - assert source["media_type"] == "image/png" - with Image.open(BytesIO(base64.b64decode(source["data"]))) as viewed: - assert viewed.getpixel((0, 0)) == (255, 0, 0, 128) - - -@pytest.mark.filterwarnings("ignore:Image size.*:PIL.Image.DecompressionBombWarning") -@pytest.mark.parametrize("size", [(4001, 4000), (10000, 10000)]) -def test_claude_rejects_oversized_source_before_decoding(size: tuple[int, int]) -> None: - buffer = BytesIO() - Image.new("RGB", (1, 1)).save(buffer, format="PNG") - data = bytearray(buffer.getvalue()) - data[16:24] = struct.pack(">II", *size) - data[29:33] = struct.pack(">I", zlib.crc32(data[12:29])) - result = _image_result(bytes(data)) - assert result["is_error"] - assert result["content"][0]["type"] == "text" - assert "16,000,000-pixel source limit" in result["content"][0]["text"] - - -def test_claude_returns_invalid_image_as_tool_error() -> None: - result = _image_result(b"invalid image") - assert result["is_error"] - assert result["content"][0]["type"] == "text" - assert "Cannot view image" in result["content"][0]["text"] - - -def test_claude_preserves_computer_screenshot_coordinate_space() -> None: - buffer = BytesIO() - Image.new("RGB", (2000, 1000), "white").save(buffer, format="PNG") - data = buffer.getvalue() - result = _image_result(data, computer=True) - assert not result["is_error"] - source = result["content"][0]["source"] - assert source["media_type"] == "image/png" - assert base64.b64decode(source["data"]) == data diff --git a/tests/agents/test_claude_cli_agent.py b/tests/agents/test_claude_cli_agent.py deleted file mode 100644 index 35f488b37..000000000 --- a/tests/agents/test_claude_cli_agent.py +++ /dev/null @@ -1,921 +0,0 @@ -"""ClaudeCLIAgent remote-command construction over the workspace SSH. - -The agent runs the ``claude`` CLI on the remote workspace. These cover how the -command is assembled per login shell — especially the Windows path, where the -command must ride a batch file invoked via ``cmd /c``. Bare ``.hud_run.bat`` is -rejected by the remote shell (and silently fails under PowerShell), so the -``cmd /c`` prefix is a regression guard for local Windows setups. -""" - -from __future__ import annotations - -import asyncio -import base64 -import json -import re -import sys -from contextlib import asynccontextmanager -from types import SimpleNamespace -from typing import Any, Literal, cast -from unittest.mock import AsyncMock, Mock - -import fastmcp -import pytest -from mcp.types import BlobResourceContents, EmbeddedResource, ImageContent, TextContent - -from hud.agents import create_agent -from hud.agents.claude.sdk import computer_mcp -from hud.agents.claude.sdk.agent import ClaudeCLIAgent -from hud.agents.claude.sdk.events import ClaudeEvents -from hud.agents.types import AgentStep, ClaudeCLIConfig, ToolStep -from hud.capabilities import Capability, Connection, SSHClient -from hud.capabilities.rfb import WebPScreenshotEncoding -from hud.settings import settings -from hud.telemetry.context import set_trace_context -from hud.types import MCPToolResult - -from .cli_fakes import FakeClient as _FakeClient -from .cli_fakes import FakeProcess as _FakeStreamProcess -from .cli_fakes import fake_run as _fake_run - - -@pytest.fixture(autouse=True) -def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", None) - monkeypatch.setattr(settings, "anthropic_api_key", None) - monkeypatch.setattr( - "hud.agents.claude.sdk.agent.resolve_executable", - AsyncMock(return_value="claude"), - ) - - -async def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - - gateway = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig(gateway=True))) - provider = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig(gateway=False))) - - assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in gateway - assert "ANTHROPIC_API_KEY=hud-key" in gateway - assert "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1" in gateway - assert "DISABLE_AUTO_COMPACT" not in gateway - assert "ANTHROPIC_API_KEY=anthropic-key" in provider - assert "ANTHROPIC_BASE_URL" not in provider - assert "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS" not in provider - assert "DISABLE_AUTO_COMPACT" not in provider - assert "ANTHROPIC_MODEL=claude-sonnet-5" in provider - - -async def test_default_routing_uses_gateway_only_without_a_provider_key( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - - without_provider_key = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig())) - monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - with_provider_key = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig())) - - assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in without_provider_key - assert "ANTHROPIC_API_KEY=anthropic-key" in with_provider_key - assert "ANTHROPIC_BASE_URL" not in with_provider_key - - -async def test_create_agent_routes_cli_agent_through_gateway( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - - agent = create_agent("claude_cli") - - assert isinstance(agent, ClaudeCLIAgent) - assert agent.config.model == ClaudeCLIConfig().model - command = await _sent_command(agent) - assert f"ANTHROPIC_BASE_URL={settings.hud_gateway_url}" in command - assert "ANTHROPIC_API_KEY=hud-key" in command - - -async def test_command_uses_process_bound_connection_without_its_credential( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - - with set_trace_context("trace-123"): - command = await _sent_command(ClaudeCLIAgent(), connection=connection) - - assert f"ANTHROPIC_BASE_URL={connection.client_url}" in command - assert "ANTHROPIC_AUTH_TOKEN=hud-process-bound" in command - assert "ANTHROPIC_API_KEY" not in command - assert "scoped-runtime-token" not in command - assert "hud-key" not in command - assert "Trace-Id" not in command - assert "exec env" in command - for name in ( - "ANTHROPIC_MODEL", - "ANTHROPIC_SMALL_FAST_MODEL", - "ANTHROPIC_DEFAULT_SONNET_MODEL", - "ANTHROPIC_DEFAULT_OPUS_MODEL", - "ANTHROPIC_DEFAULT_HAIKU_MODEL", - "CLAUDE_CODE_SUBAGENT_MODEL", - ): - assert f"{name}=claude-sonnet-5" in command - - -async def test_windows_command_encodes_environment_and_arguments( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud&key's") - config = ClaudeCLIConfig( - gateway=True, - max_steps=3, - system_prompt="don't $expand", - ) - command = await _sent_command(ClaudeCLIAgent(config), "powershell") - - encoded = command.rsplit(" ", 1)[1] - script = base64.b64decode(encoded).decode("utf-16-le") - assert "$env:ANTHROPIC_API_KEY='hud&key''s'" in script - assert "$env:CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS='1'" in script - assert "'--system-prompt' 'don''t $expand'" in script - assert "Get-Content -Raw -Encoding UTF8 '.hud_input.jsonl' | & 'claude'" in script - assert "'--input-format=stream-json'" in script - assert "python" not in script - - -class _FakeCompletedProcess: - async def wait(self, *, check: bool, **kwargs: Any) -> Any: - del check - assert kwargs == {"timeout": None} - return SimpleNamespace(stdout=b"", stderr=b"", exit_status=0, returncode=0) - - def terminate(self) -> None: - pass - - def close(self) -> None: - pass - - async def wait_closed(self) -> None: - pass - - -class _FakeConn: - def __init__(self, sink: dict[str, bytes], process: _FakeStreamProcess) -> None: - self._sink = sink - self._process = process - self.ran: list[str] = [] - self.write_commands: list[str] = [] - self.written: dict[str, bytes] = {} - self.deleted: list[str] = [] - - def is_closed(self) -> bool: - return False - - async def create_process(self, cmd: str, **kwargs: Any) -> Any: - input_value = kwargs.get("input") - if cmd.startswith(("rm -f -- ", "cmd /c del /f /q ")): - paths = [path for path in self._sink if path in cmd] - for path in paths: - self._sink.pop(path) - self.deleted.extend(paths) - return _FakeCompletedProcess() - if input_value is not None or cmd.startswith("powershell "): - self.write_commands.append(cmd) - script = cmd - if match := re.search(r"-EncodedCommand (\S+)", cmd): - script = base64.b64decode(match.group(1)).decode("utf-16-le") - name = next( - path - for path in (".hud_input.jsonl", ".hud_run.bat", ".hud_mcp_config.json") - if path in script - ) - if input_value is not None: - self._sink[name] = str(input_value).encode() - elif match := re.search(r"FromBase64String\('([^']+)'\)", script): - self._sink[name] += base64.b64decode(match.group(1)) - else: - self._sink[name] = b"" - self.written[name] = self._sink[name] - return _FakeCompletedProcess() - assert kwargs["encoding"] is None - self.env = kwargs.get("env") - self.ran.append(cmd) - return self._process - - -async def run_claude( - config: ClaudeCLIConfig, - run: Any, - *, - ssh: SSHClient, - prompt: str, - bindings: list[Capability] | None = None, - routed: dict[str, Capability] | None = None, -) -> None: - run.client = _FakeClient(ssh, bindings, routed) - run.prompt_text = prompt - await ClaudeCLIAgent(config)(run) - - -_STREAM_JSON = ( - '{"type":"assistant","message":{"id":"msg-1","type":"message",' - '"role":"assistant","model":"claude-test","content":[{"type":"text",' - '"text":"editing"},{"type":"tool_use","id":"tool-1","name":"Write","input":{}}],' - '"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":11,' - '"output_tokens":7,"cache_read_input_tokens":3}}}\n' - '{"type":"user","message":{"content":[{"type":"tool_result",' - '"tool_use_id":"tool-1","content":[{"type":"text","text":"wrote a.txt"},' - '{"type":"image","source":{"type":"base64","media_type":"image/png",' - '"data":"aW1hZ2U="}}],"is_error":false}]}}\n' - '{"type":"assistant","message":{"id":"msg-2","type":"message",' - '"role":"assistant","model":"claude-test","content":[{"type":"text",' - '"text":"done"}],"stop_reason":"end_turn","stop_sequence":null,' - '"usage":{"input_tokens":11,"output_tokens":7,"cache_read_input_tokens":3}}}\n' - '{"type":"result","is_error":false,"result":"done","session_id":"s",' - '"duration_ms":5,"num_turns":2,"total_cost_usd":0.01}\n' -) - - -def _ssh_with_conn(shell: str, conn: _FakeConn) -> SSHClient: - capability = Capability( - name="shell", - protocol="ssh/2", - url="ssh://localhost:22", - params={"shell": shell, "process_connections": True}, - ) - return SSHClient(capability, cast("Any", conn)) - - -async def _sent_command( - agent: ClaudeCLIAgent, shell: str = "bash", connection: Connection | None = None -) -> str: - """The CLI command the agent sends over SSH (unwrapped from the batch file on Windows).""" - conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - connections = {"inference": connection} if connection is not None else None - await agent(_fake_run(_FakeClient(_ssh_with_conn(shell, conn)), "x", connections)) - if shell in ("cmd", "powershell"): - return conn.written[".hud_run.bat"].decode().split("\r\n")[1] - (command,) = conn.ran - return command - - -async def test_command_passes_the_configured_reasoning_effort() -> None: - command = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig(reasoning_effort="max"))) - assert "--effort max" in command - - -async def test_command_omits_effort_when_unconfigured() -> None: - command = await _sent_command(ClaudeCLIAgent(ClaudeCLIConfig())) - assert "--effort" not in command - - -async def test_exec_on_windows_writes_batch_and_execs_via_cmd() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) - ssh = _ssh_with_conn("cmd", conn) - - run = _fake_run() - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="build it") - - assert conn.ran == ["cmd /c .hud_run.bat"] - assert all(command.startswith("powershell ") for command in conn.write_commands) - assert conn.written[".hud_run.bat"].startswith(b"@echo off\r\n") - assert conn.written[".hud_input.jsonl"].endswith(b"\n") - assert json.loads(conn.written[".hud_input.jsonl"]) == { - "type": "user", - "message": { - "role": "user", - "content": [{"type": "text", "text": "build it"}], - }, - } - assert sink == {} - assert set(conn.deleted) == {".hud_input.jsonl", ".hud_run.bat"} - assert run.trace.status is None - assert run.trace.content == "done" - assert "messages" not in run.trace.extra - - -async def test_exec_on_bash_runs_inline_without_batch() -> None: - sink: dict[str, bytes] = {} - process = _FakeStreamProcess(_STREAM_JSON) - conn = _FakeConn(sink, process) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="build it") - - assert sink == {} - assert conn.write_commands == [] - assert conn.deleted == [] - assert len(conn.ran) == 1 - assert "claude" in conn.ran[0] - assert "--input-format=stream-json" in conn.ran[0] - assert "build it" not in conn.ran[0] - assert process.stdin.data.endswith(b"\n") - assert json.loads(process.stdin.data) == { - "type": "user", - "message": { - "role": "user", - "content": [{"type": "text", "text": "build it"}], - }, - } - assert process.stdin.eof is True - assert run.trace.status is None - assert run.trace.content == "done" - assert "messages" not in run.trace.extra - - -async def test_exec_removes_mcp_config_after_run() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON)) - await run_claude( - ClaudeCLIConfig(), - _fake_run(), - ssh=_ssh_with_conn("bash", conn), - bindings=[ - Capability.mcp(name="database", url="http://db/mcp", transport="streamable-http") - ], - prompt="build it", - ) - - config = json.loads(conn.written[".hud_mcp_config.json"]) - assert config == {"mcpServers": {"database": {"type": "http", "url": "http://db:80/mcp"}}} - assert sink == {} - assert conn.deleted == [".hud_mcp_config.json"] - assert "--mcp-config .hud_mcp_config.json" in conn.ran[0] - - -async def test_exec_records_steps_before_process_exit() -> None: - process = _FakeStreamProcess(_STREAM_JSON, pause_after=1) - conn = _FakeConn({}, process) - ssh = _ssh_with_conn("bash", conn) - run = _fake_run() - - execution = asyncio.create_task( - run_claude( - ClaudeCLIConfig(), - run, - ssh=ssh, - prompt="edit it", - ) - ) - await process.stdout.blocked.wait() - - assert not execution.done() - assert len(run.steps) == 1 - first = run.steps[0] - assert isinstance(first, AgentStep) - assert first.content == "editing" - assert first.tool_calls[0].id == "tool-1" - - process.stdout.release.set() - await execution - - assert [type(step) for step in run.steps] == [AgentStep, ToolStep, AgentStep] - tool = cast("ToolStep", run.steps[1]) - assert tool.started_at == first.ended_at - assert tool.result is not None - text = tool.result.content[0] - assert isinstance(text, TextContent) - assert text.text == "wrote a.txt" - image = tool.result.content[1] - assert isinstance(image, ImageContent) - assert image.mimeType == "image/png" - assert image.data == "aW1hZ2U=" - final = cast("AgentStep", run.steps[2]) - assert final.started_at == tool.ended_at - assert run.trace.status is None - assert run.trace.content == "done" - - -async def test_exec_forwards_trace_id_only_to_hud_gateway( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - monkeypatch.setattr(settings, "anthropic_api_key", "anthropic-key") - - gateway_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - gateway = ClaudeCLIConfig(gateway=True) - with set_trace_context("trace-123"): - await run_claude( - gateway, - _fake_run(), - ssh=_ssh_with_conn("bash", gateway_conn), - prompt="build it", - ) - assert "ANTHROPIC_CUSTOM_HEADERS='Trace-Id: trace-123'" in gateway_conn.ran[0] - - provider_conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - provider = ClaudeCLIConfig(gateway=False) - with set_trace_context("trace-123"): - await run_claude( - provider, - _fake_run(), - ssh=_ssh_with_conn("bash", provider_conn), - prompt="build it", - ) - assert "ANTHROPIC_CUSTOM_HEADERS" not in provider_conn.ran[0] - - -async def test_exec_closes_streaming_process_when_cancelled() -> None: - process = _FakeStreamProcess(_STREAM_JSON, pause_after=0) - conn = _FakeConn({}, process) - execution = asyncio.create_task( - run_claude( - ClaudeCLIConfig(), - _fake_run(), - ssh=_ssh_with_conn("bash", conn), - prompt="build it", - ) - ) - await process.stdout.blocked.wait() - execution.cancel() - - with pytest.raises(asyncio.CancelledError): - await execution - - assert process.closed - - -async def test_exec_nonzero_exit_with_no_stdout_raises() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn(sink, _FakeStreamProcess("", stderr="boom", exit_status=1)) - ssh = _ssh_with_conn("cmd", conn) - - run = _fake_run() - with pytest.raises(RuntimeError, match="boom"): - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.extra["returncode"] == 1 - - -async def test_exec_signal_exit_records_the_returncode() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn( - sink, - _FakeStreamProcess("", exit_status=None, returncode=-15), - ) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - with pytest.raises(RuntimeError, match="return code -15"): - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.extra["returncode"] == -15 - - -async def test_exec_nonzero_exit_with_result_stream_remains_an_error() -> None: - sink: dict[str, bytes] = {} - conn = _FakeConn( - sink, - _FakeStreamProcess(_STREAM_JSON, stderr="transport failed", exit_status=1), - ) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - with pytest.raises(RuntimeError, match="transport failed"): - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.content == "done" - assert run.trace.extra["returncode"] == 1 - assert run.trace.extra["stderr"] == "transport failed" - assert "messages" not in run.trace.extra - - -async def test_exec_nonzero_exit_with_result_stream_and_no_stderr_is_graded() -> None: - """The CLI exits nonzero after a successful result; the run stays gradeable.""" - sink: dict[str, bytes] = {} - conn = _FakeConn(sink, _FakeStreamProcess(_STREAM_JSON, exit_status=1)) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.content == "done" - assert run.trace.extra["returncode"] == 1 - assert "stderr" not in run.trace.extra - - -async def test_exec_zero_exit_without_result_event_is_an_error() -> None: - sink: dict[str, bytes] = {} - stdout = _STREAM_JSON.rsplit('{"type":"result"', 1)[0] - conn = _FakeConn(sink, _FakeStreamProcess(stdout)) - ssh = _ssh_with_conn("bash", conn) - - run = _fake_run() - with pytest.raises(RuntimeError, match="without a result event"): - await run_claude(ClaudeCLIConfig(), run, ssh=ssh, prompt="x") - - assert run.trace.content == "done" - - -@pytest.mark.parametrize( - ("transport", "claude_type"), - [("streamable-http", "http"), ("sse", "sse")], -) -async def test_manifest_mcp_capability_is_written_for_remote_claude( - transport: Literal["streamable-http", "sse"], - claude_type: str, -) -> None: - mcp = Capability.mcp(name="database", url="http://database:8000/mcp", transport=transport) - conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - - await run_claude( - ClaudeCLIConfig(), - _fake_run(), - ssh=_ssh_with_conn("bash", conn), - prompt="call the tool", - bindings=[mcp], - ) - - assert json.loads(conn.written[".hud_mcp_config.json"]) == { - "mcpServers": {"database": {"type": claude_type, "url": "http://database:8000/mcp"}} - } - - -async def test_remote_claude_bridges_computer_mcp_for_the_cli_run( - monkeypatch: pytest.MonkeyPatch, -) -> None: - screen = Capability.rfb(name="screen", url="rfb://localhost:5900", display=0) - routed = Capability.rfb(name="screen", url="rfb://127.0.0.1:41000", display=0) - encoding = WebPScreenshotEncoding(quality=42) - conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - ssh = _ssh_with_conn("bash", conn) - runs_while_bridged: list[int] = [] - - @asynccontextmanager - async def bridge( - bridge_ssh: SSHClient, - capability: Capability, - screenshot_encoding: WebPScreenshotEncoding, - *, - shell: str, - ) -> Any: - assert bridge_ssh is ssh - assert capability == routed - assert screenshot_encoding == encoding - assert shell == "bash" - yield {"type": "stdio", "command": "sh", "args": ["-c", "relay"]} - runs_while_bridged.append(len(conn.ran)) - - monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - - await run_claude( - ClaudeCLIConfig(screenshot_encoding=encoding), - _fake_run(), - ssh=ssh, - prompt="use the computer", - bindings=[screen], - routed={"screen": routed}, - ) - - assert runs_while_bridged == [1] - assert json.loads(conn.written[".hud_mcp_config.json"]) == { - "mcpServers": {"computer-use": {"type": "stdio", "command": "sh", "args": ["-c", "relay"]}} - } - - -async def test_remote_claude_preserves_multiple_rfb_bindings( - monkeypatch: pytest.MonkeyPatch, -) -> None: - screens = [ - Capability.rfb(name="screen-0", url="rfb://display-0:5900", display=0), - Capability.rfb(name="screen-1", url="rfb://display-1:5901", display=1), - ] - routed = { - cap.name: Capability.rfb(name=cap.name, url=f"rfb://127.0.0.1:{41000 + i}", display=i) - for i, cap in enumerate(screens) - } - conn = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - - @asynccontextmanager - async def bridge( - _ssh: SSHClient, - capability: Capability, - _encoding: WebPScreenshotEncoding, - *, - shell: str, - ) -> Any: - yield {"type": "stdio", "command": "sh", "args": ["-c", capability.name]} - - monkeypatch.setattr(computer_mcp, "bridge_computer_mcp", bridge) - - await run_claude( - ClaudeCLIConfig(), - _fake_run(), - ssh=_ssh_with_conn("bash", conn), - prompt="use both screens", - bindings=screens, - routed=routed, - ) - - assert json.loads(conn.written[".hud_mcp_config.json"]) == { - "mcpServers": { - f"computer-use-{name}": {"type": "stdio", "command": "sh", "args": ["-c", name]} - for name in ("screen-0", "screen-1") - } - } - - -async def test_computer_mcp_stdio_owns_rfb_lifetime( - monkeypatch: pytest.MonkeyPatch, -) -> None: - screen = Capability.rfb(name="screen", url="rfb://localhost:5900", display=0) - encoding = WebPScreenshotEncoding(quality=42) - rfb = SimpleNamespace(close=AsyncMock()) - connect = AsyncMock(return_value=rfb) - server = SimpleNamespace(run_async=AsyncMock()) - create = Mock(return_value=server) - monkeypatch.setattr(computer_mcp.RFBClient, "connect", connect) - monkeypatch.setattr(computer_mcp, "create_computer_mcp", create) - - await computer_mcp.run_computer_mcp( - { - computer_mcp.RFB_CAPABILITY_ENV: json.dumps(screen.to_manifest()), - computer_mcp.SCREENSHOT_ENCODING_ENV: encoding.model_dump_json(), - } - ) - - connect.assert_awaited_once_with(screen) - create.assert_called_once_with(rfb, encoding) - server.run_async.assert_awaited_once_with(transport="stdio", show_banner=False) - rfb.close.assert_awaited_once() - - -async def test_computer_mcp_preserves_tool_result(monkeypatch: pytest.MonkeyPatch) -> None: - result = MCPToolResult( - content=[TextContent(type="text", text="failed")], - isError=True, - ) - execute = AsyncMock(return_value=result) - monkeypatch.setattr(computer_mcp.ClaudeComputerTool, "execute", execute) - server = computer_mcp.create_computer_mcp(cast("Any", object()), WebPScreenshotEncoding()) - - async with fastmcp.Client(server) as client: - received = await client.call_tool_mcp( - "computer", - {"action": "left_click", "coordinate": [10, 20]}, - ) - - execute.assert_awaited_once_with({"action": "left_click", "coordinate": [10, 20]}) - assert received.isError is True - assert received.content == result.content - - -class _ByteWriter: - def __init__(self) -> None: - self.closed = False - self.data = bytearray() - - def write(self, data: bytes) -> None: - self.data.extend(data) - - async def drain(self) -> None: - pass - - def close(self) -> None: - self.closed = True - - -class _LocalComputerProcess: - def __init__(self) -> None: - self.stdin = _ByteWriter() - self.stdout = asyncio.StreamReader() - self.stderr = asyncio.StreamReader() - self.returncode: int | None = None - self.terminated = False - self.killed = False - - def terminate(self) -> None: - self.terminated = True - self.returncode = -15 - - def kill(self) -> None: - self.killed = True - self.returncode = -9 - - async def wait(self) -> int: - assert self.returncode is not None - return self.returncode - - -async def test_computer_mcp_bridge_uses_controller_python_and_owns_resources( - monkeypatch: pytest.MonkeyPatch, -) -> None: - bridge_stdout = asyncio.StreamReader() - bridge_stderr = asyncio.StreamReader() - bridge_stderr.feed_data(b"ready\n") - bridge_stdin = _ByteWriter() - bridge = SimpleNamespace( - stdin=bridge_stdin, - stdout=bridge_stdout, - stderr=bridge_stderr, - channel=SimpleNamespace(close=Mock()), - wait_closed=AsyncMock(), - ) - connection = SimpleNamespace(create_process=AsyncMock(return_value=bridge)) - ssh = SimpleNamespace(create_process=connection.create_process) - local = _LocalComputerProcess() - spawn = AsyncMock(return_value=local) - monkeypatch.setattr(computer_mcp.asyncio, "create_subprocess_exec", spawn) - monkeypatch.setattr(computer_mcp.secrets, "token_hex", lambda _length: "bridge-token") - screen = Capability.rfb(name="screen", url="rfb://127.0.0.1:41000", display=0) - encoding = WebPScreenshotEncoding(quality=42) - - async with computer_mcp.bridge_computer_mcp( - cast("Any", ssh), - screen, - encoding, - shell="bash", - ) as config: - assert config == { - "type": "stdio", - "command": "sh", - "args": [ - "-c", - "cat /tmp/hud-computer-bridge-token.response & reader=$!; " - "cat > /tmp/hud-computer-bridge-token.request; wait $reader", - ], - } - assert not bridge_stdin.closed - assert not local.stdin.closed - - bridge_command = connection.create_process.await_args.args[0] - assert "mkfifo -- /tmp/hud-computer-bridge-token.request" in bridge_command - assert "printf 'ready\\n' >&2" in bridge_command - spawn.assert_awaited_once() - spawn_call = spawn.await_args - assert spawn_call is not None - spawn_args = spawn_call.args - assert spawn_args[:3] == ( - sys.executable, - "-m", - "hud.agents.claude.sdk.computer_mcp", - ) - environ = spawn_call.kwargs["env"] - assert json.loads(environ[computer_mcp.RFB_CAPABILITY_ENV]) == screen.to_manifest() - assert environ[computer_mcp.SCREENSHOT_ENCODING_ENV] == encoding.model_dump_json() - bridge.channel.close.assert_called_once() - bridge.wait_closed.assert_awaited_once() - assert bridge_stdin.closed - assert local.stdin.closed - assert local.terminated - assert not local.killed - - -async def test_computer_mcp_bridge_rejects_windows_before_starting_resources() -> None: - screen = Capability.rfb(name="screen", url="rfb://127.0.0.1:41000", display=0) - ssh = SimpleNamespace(create_process=AsyncMock()) - - with pytest.raises(RuntimeError, match="requires a POSIX workspace"): - async with computer_mcp.bridge_computer_mcp( - cast("Any", ssh), - screen, - WebPScreenshotEncoding(), - shell="powershell", - ): - pass - - ssh.create_process.assert_not_awaited() - - -@pytest.mark.skipif(sys.platform == "win32", reason="POSIX FIFO relay") -async def test_computer_mcp_bridge_relays_both_directions(monkeypatch: pytest.MonkeyPatch) -> None: - """The remote FIFO bridge and the relay Claude runs form one duplex pipe to the controller.""" - spawn = asyncio.create_subprocess_exec - - async def remote_shell(command: str) -> Any: - process = await asyncio.create_subprocess_shell( - command, - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - return SimpleNamespace( - stdin=process.stdin, - stdout=process.stdout, - stderr=process.stderr, - channel=SimpleNamespace(close=lambda: None), - wait_closed=process.wait, - ) - - async def echo_controller(*_args: Any, **kwargs: Any) -> asyncio.subprocess.Process: - return await spawn( - "cat", stdin=kwargs["stdin"], stdout=kwargs["stdout"], stderr=kwargs["stderr"] - ) - - monkeypatch.setattr(computer_mcp.asyncio, "create_subprocess_exec", echo_controller) - screen = Capability.rfb(name="screen", url="rfb://127.0.0.1:41000", display=0) - - async with asyncio.timeout(10): - async with computer_mcp.bridge_computer_mcp( - cast("Any", SimpleNamespace(create_process=remote_shell)), - screen, - WebPScreenshotEncoding(), - shell="bash", - ) as config: - relay = await spawn( - config["command"], - *config["args"], - stdin=asyncio.subprocess.PIPE, - stdout=asyncio.subprocess.PIPE, - ) - assert relay.stdin is not None and relay.stdout is not None - relay.stdin.write(b'{"method":"tools/list"}\n') - await relay.stdin.drain() - assert await relay.stdout.readline() == b'{"method":"tools/list"}\n' - relay.stdin.close() - await relay.wait() - - -async def test_concurrent_runs_keep_their_ssh_state_isolated() -> None: - first_process = _FakeStreamProcess(_STREAM_JSON, pause_after=0) - conn_a = _FakeConn({}, first_process) - conn_b = _FakeConn({}, _FakeStreamProcess(_STREAM_JSON)) - run_a, run_b = _fake_run(), _fake_run() - - first = asyncio.create_task( - run_claude(ClaudeCLIConfig(), run_a, ssh=_ssh_with_conn("bash", conn_a), prompt="first") - ) - await first_process.stdout.blocked.wait() - await run_claude( - ClaudeCLIConfig(), run_b, ssh=_ssh_with_conn("powershell", conn_b), prompt="second" - ) - first_process.stdout.release.set() - await first - - assert conn_a.write_commands == [] - assert "--input-format=stream-json" in conn_a.ran[0] - assert json.loads(first_process.stdin.data)["message"]["content"][0]["text"] == "first" - assert conn_b.ran == ["cmd /c .hud_run.bat"] - assert json.loads(conn_b.written[".hud_input.jsonl"])["message"]["content"][0]["text"] == ( - "second" - ) - assert run_a.trace.content == run_b.trace.content == "done" - - -def test_events_record_document_and_unknown_tool_result_blocks() -> None: - """A PDF Read returns a document block; neither it nor an unknown block ends the run.""" - run = _fake_run() - events = ClaudeEvents(run, started_at="2026-01-01T00:00:00Z") - events.consume( - json.dumps( - { - "type": "assistant", - "message": { - "id": "msg-1", - "type": "message", - "role": "assistant", - "model": "claude-test", - "content": [{"type": "tool_use", "id": "tool-1", "name": "Read", "input": {}}], - "stop_reason": "tool_use", - "stop_sequence": None, - "usage": {"input_tokens": 1, "output_tokens": 1}, - }, - } - ) - ) - events.consume( - json.dumps( - { - "type": "user", - "message": { - "content": [ - { - "type": "tool_result", - "tool_use_id": "tool-1", - "content": [ - { - "type": "document", - "source": { - "type": "base64", - "media_type": "application/pdf", - "data": "JVBERi0=", - }, - }, - {"type": "novelty", "payload": 1}, - ], - } - ] - }, - } - ) - ) - - tool = cast("ToolStep", run.steps[1]) - assert tool.result is not None - document, unknown = tool.result.content - assert isinstance(document, EmbeddedResource) - assert isinstance(document.resource, BlobResourceContents) - assert document.resource.mimeType == "application/pdf" - assert document.resource.blob == "JVBERi0=" - assert str(document.resource.uri) == "document://tool-1/0" - assert isinstance(unknown, TextContent) - assert unknown.text == "[unsupported novelty block]" - assert events.error is None diff --git a/tests/agents/test_codex_cli_agent.py b/tests/agents/test_codex_cli_agent.py deleted file mode 100644 index 7b9bcb303..000000000 --- a/tests/agents/test_codex_cli_agent.py +++ /dev/null @@ -1,392 +0,0 @@ -"""CodexCLIAgent command construction and JSONL trajectory mapping.""" - -from __future__ import annotations - -import asyncio -import base64 -from types import SimpleNamespace -from typing import Any, cast -from unittest.mock import AsyncMock - -import pytest -from mcp.types import TextContent - -from hud.agents.cli import resolve_executable -from hud.agents.codex import CodexCLIAgent -from hud.agents.types import AgentStep, CodexCLIConfig, ToolStep -from hud.capabilities import Capability, Connection -from hud.eval.runtime import RuntimeConfig, RuntimeResources -from hud.settings import settings -from hud.telemetry.context import set_trace_context - -from .cli_fakes import FakeClient as _FakeClient -from .cli_fakes import FakeProcess as _FakeProcess -from .cli_fakes import fake_run as _fake_run - - -@pytest.fixture(autouse=True) -def _clear_api_keys(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", None) - monkeypatch.setattr(settings, "openai_api_key", None) - monkeypatch.setattr( - "hud.agents.codex.agent.resolve_executable", - AsyncMock(return_value="codex"), - ) - - -class _FakeSSH: - def __init__( - self, process: _FakeProcess, *, shell: str = "bash", executable_head: bytes = b"\x7fE" - ) -> None: - self.process = process - self.executable_head = executable_head - self.capability = Capability( - name="shell", - protocol="ssh/2", - url="ssh://localhost:22", - params={"shell": shell}, - ) - self.commands: list[str] = [] - - async def run(self, command: str, **_: Any) -> SimpleNamespace: - assert command.startswith("head -c 2 -- ") - return SimpleNamespace(returncode=0, stdout=self.executable_head) - - async def create_process( - self, command: str, *, connections: tuple[Connection, ...] = () - ) -> _FakeProcess: - self.commands.append(command) - return self.process - - -async def _sent_command( - config: CodexCLIConfig, shell: str = "bash", connection: Connection | None = None -) -> str: - ssh = _FakeSSH(_FakeProcess(_STREAM_JSON), shell=shell) - connections = {"inference": connection} if connection is not None else None - await CodexCLIAgent(config)(_fake_run(_FakeClient(ssh), "Fix it", connections)) - (command,) = ssh.commands - return command - - -_STREAM_JSON = ( - '{"type":"thread.started","thread_id":"thread-1"}\n' - '{"type":"turn.started"}\n' - '{"type":"item.started","item":{"id":"cmd-1","type":"command_execution",' - '"command":"pytest -q","aggregated_output":"","exit_code":null,' - '"status":"in_progress"}}\n' - '{"type":"item.completed","item":{"id":"cmd-1","type":"command_execution",' - '"command":"pytest -q","aggregated_output":"1 passed\\n","exit_code":0,' - '"status":"completed"}}\n' - '{"type":"item.completed","item":{"id":"patch-1","type":"file_change",' - '"changes":[{"path":"calc.py","kind":"update"}],"status":"completed"}}\n' - '{"type":"item.started","item":{"id":"mcp-1","type":"mcp_tool_call",' - '"server":"db","tool":"query","arguments":{"sql":"select 42"},' - '"result":null,"error":null,"status":"in_progress"}}\n' - '{"type":"item.completed","item":{"id":"mcp-1","type":"mcp_tool_call",' - '"server":"db","tool":"query","arguments":{"sql":"select 42"},' - '"result":{"content":[{"type":"text","text":"42"}],' - '"structured_content":{"answer":42}},"error":null,"status":"completed"}}\n' - '{"type":"item.completed","item":{"id":"search-1","type":"web_search",' - '"query":"HUD evals","action":{"type":"search"}}}\n' - '{"type":"item.completed","item":{"id":"reason-1","type":"reasoning",' - '"text":"The test now passes."}}\n' - '{"type":"item.completed","item":{"id":"message-1","type":"agent_message",' - '"text":"Implemented and verified."}}\n' - '{"type":"turn.completed","usage":{"input_tokens":20,"cached_input_tokens":5,' - '"output_tokens":8,"reasoning_output_tokens":3}}\n' -) - - -async def test_command_passes_the_configured_reasoning_effort() -> None: - command = await _sent_command(CodexCLIConfig(reasoning_effort="max")) - assert 'model_reasoning_effort="max"' in command - - -async def test_command_omits_reasoning_effort_when_unconfigured() -> None: - command = await _sent_command(CodexCLIConfig()) - assert "model_reasoning_effort" not in command - - -async def test_command_follows_explicit_gateway_routing(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - monkeypatch.setattr(settings, "openai_api_key", "openai-key") - - with set_trace_context("trace-123"): - gateway = await _sent_command(CodexCLIConfig(gateway=True)) - provider = await _sent_command(CodexCLIConfig(gateway=False)) - - assert "HUD_API_KEY=hud-key" in gateway - assert 'model_provider="hud"' in gateway - assert f'model_providers.hud.base_url="{settings.hud_gateway_url}"' in gateway - assert "Trace-Id" in gateway - assert "CODEX_API_KEY=openai-key" in provider - assert "model_provider" not in provider - for command in (gateway, provider): - assert "codex exec" in command - assert "--json" in command - assert "--ephemeral" in command - assert "--ignore-user-config" not in command - assert "mktemp -d" in command - assert 'export CODEX_HOME="$codex_home"' in command - assert "--sandbox workspace-write" in command - assert "--model gpt-5.6-sol" in command - assert command.endswith(" -") - - -@pytest.mark.parametrize("sandbox", ["read-only", "workspace-write", "danger-full-access"]) -async def test_command_uses_process_bound_connection_without_its_credential( - monkeypatch: pytest.MonkeyPatch, sandbox: str -) -> None: - monkeypatch.setattr(settings, "api_key", "hud-key") - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - - with set_trace_context("trace-123"): - command = await _sent_command( - CodexCLIConfig.model_validate({"sandbox": sandbox}), connection=connection - ) - - assert "scoped-runtime-token" not in command - assert "hud-key" not in command - assert "HUD_CONNECTION_CREDENTIAL=hud-process-bound" in command - assert 'model_providers.hud.env_key="HUD_CONNECTION_CREDENTIAL"' in command - assert f'model_providers.hud.base_url="{connection.client_url}"' in command - assert "Trace-Id" not in command - assert "exec env" in command - assert "--sandbox danger-full-access" in command - - -async def test_process_bound_connection_rejects_a_launcher_script_before_starting() -> None: - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - ssh = _FakeSSH(_FakeProcess(_STREAM_JSON), executable_head=b"#!") - - with pytest.raises(RuntimeError, match="launcher script"): - await CodexCLIAgent()(_fake_run(_FakeClient(ssh), "Fix it", {"inference": connection})) - - assert ssh.commands == [] - - -@pytest.mark.parametrize("shell", ["bash", "powershell"]) -async def test_command_preserves_ambient_codex_login_without_explicit_credentials( - shell: str, -) -> None: - command = await _sent_command(CodexCLIConfig(), shell) - script = ( - base64.b64decode(command.rsplit(" ", 1)[1]).decode("utf-16-le") - if shell == "powershell" - else command - ) - - assert "CODEX_HOME" not in script - assert "CODEX_API_KEY" not in script - assert "HUD_API_KEY" not in script - assert "mktemp" not in script - assert "codex exec" in script or "& 'codex' 'exec'" in script - - -async def test_windows_command_encodes_environment_and_arguments( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(settings, "openai_api_key", "key&value's") - config = CodexCLIConfig(gateway=False, sandbox="danger-full-access") - command = await _sent_command(config, "powershell") - - script = base64.b64decode(command.rsplit(" ", 1)[1]).decode("utf-16-le") - assert "$env:CODEX_API_KEY='key&value''s'" in script - assert "$env:CODEX_HOME=$codexHome" in script - assert "[System.Guid]::NewGuid()" in script - assert "Remove-Item -Recurse -Force $codexHome" in script - assert "--ignore-user-config" not in script - assert "'--sandbox' 'danger-full-access'" in script - assert "& 'codex' 'exec'" in script - assert script.endswith(";exit $hudExitCode") - - -async def test_exec_streams_prompt_and_records_codex_items() -> None: - process = _FakeProcess(_STREAM_JSON) - run = _fake_run(_FakeClient(_FakeSSH(process)), "Fix the failing test") - - await CodexCLIAgent()(run) - - assert process.stdin.data == b"Fix the failing test" - assert process.stdin.eof - assert [type(step) for step in run.steps] == [ - ToolStep, - ToolStep, - ToolStep, - ToolStep, - AgentStep, - AgentStep, - ] - command = cast("ToolStep", run.steps[0]) - assert command.call is not None - assert command.call.name == "shell" - assert command.call.arguments == {"command": "pytest -q"} - assert command.result is not None - assert command.result.isError is False - output = command.result.content[0] - assert isinstance(output, TextContent) - assert output.text == "1 passed\n" - patch = cast("ToolStep", run.steps[1]) - assert patch.call is not None - assert patch.call.name == "apply_patch" - mcp = cast("ToolStep", run.steps[2]) - assert mcp.call is not None - assert mcp.call.name == "query" - assert mcp.call.provider_name == "db.query" - assert mcp.result is not None - assert mcp.result.structuredContent == {"answer": 42} - search = cast("ToolStep", run.steps[3]) - assert search.call is not None - assert search.call.name == "web_search" - assert cast("AgentStep", run.steps[4]).reasoning == "The test now passes." - assert cast("AgentStep", run.steps[5]).content == "Implemented and verified." - assert run.trace.content == "Implemented and verified." - assert run.trace.extra["codex_thread_id"] == "thread-1" - assert run.trace.extra["usage"]["cached_input_tokens"] == 5 - assert run.trace.status is None - - -async def test_exec_records_completed_items_before_process_exit() -> None: - process = _FakeProcess(_STREAM_JSON, pause_after=4) - run = _fake_run(_FakeClient(_FakeSSH(process)), "Fix it") - execution = asyncio.create_task(CodexCLIAgent()(run)) - await process.stdout.blocked.wait() - - assert not execution.done() - assert len(run.steps) == 1 - assert isinstance(run.steps[0], ToolStep) - - process.stdout.release.set() - await execution - - -async def test_exec_turn_failure_raises() -> None: - stream = ( - '{"type":"thread.started","thread_id":"thread-1"}\n' - '{"type":"turn.started"}\n' - '{"type":"turn.failed","error":{"message":"model unavailable"}}\n' - ) - run = _fake_run(_FakeClient(_FakeSSH(_FakeProcess(stream))), "Fix it") - - with pytest.raises(RuntimeError, match="model unavailable"): - await CodexCLIAgent()(run) - - -async def test_exec_nonzero_exit_raises_stderr() -> None: - process = _FakeProcess("", stderr="authentication failed", returncode=1) - run = _fake_run(_FakeClient(_FakeSSH(process)), "Fix it") - - with pytest.raises(RuntimeError, match="authentication failed"): - await CodexCLIAgent()(run) - - assert run.trace.extra["returncode"] == 1 - - -async def test_exec_nonzero_exit_prefers_structured_error() -> None: - stream = '{"type":"error","message":"gateway rejected streaming"}\n' - process = _FakeProcess(stream, stderr="noisy warning", returncode=1) - run = _fake_run(_FakeClient(_FakeSSH(process)), "Fix it") - - with pytest.raises(RuntimeError, match="gateway rejected streaming"): - await CodexCLIAgent()(run) - - assert "stderr" not in run.trace.extra - - -async def test_exec_closes_process_when_cancelled() -> None: - process = _FakeProcess(_STREAM_JSON, pause_after=0) - execution = asyncio.create_task( - CodexCLIAgent()(_fake_run(_FakeClient(_FakeSSH(process)), "Fix it")) - ) - await process.stdout.blocked.wait() - execution.cancel() - - with pytest.raises(asyncio.CancelledError): - await execution - - assert process.closed - - -async def test_agent_runs_in_the_workspace_shell_with_the_run_prompt() -> None: - process = _FakeProcess(_STREAM_JSON) - ssh = _FakeSSH(process, shell="powershell") - - await CodexCLIAgent()(_fake_run(_FakeClient(ssh), "Fix it")) - - (command,) = ssh.commands - script = base64.b64decode(command.rsplit(" ", 1)[1]).decode("utf-16-le") - assert "& 'codex' 'exec'" in script - assert process.stdin.data == b"Fix it" - - -async def test_executable_resolution_prefers_matching_managed_bundle() -> None: - ssh = SimpleNamespace( - capability=Capability.ssh(url="ssh://localhost:22", host_pubkey="key", shell="bash"), - run=AsyncMock( - side_effect=[ - SimpleNamespace(returncode=0, stdout=b"Linux\nx86_64\ngnu\n"), - SimpleNamespace(returncode=0, stdout=b""), - ] - ), - ) - - executable = await resolve_executable( - cast("Any", ssh), - "codex", - {"linux-x64": "/usr/local/lib/agents/codex/bin/codex"}, - RuntimeConfig(resources=RuntimeResources(os="linux")), - ) - - assert executable == "/usr/local/lib/agents/codex/bin/codex" - assert ssh.run.await_count == 2 - - -@pytest.mark.parametrize(("reported", "platform"), [("X64", "x64"), ("Arm64", "arm64")]) -async def test_executable_resolution_understands_windows_architectures( - reported: str, platform: str -) -> None: - ssh = SimpleNamespace( - capability=Capability.ssh(url="ssh://localhost:22", host_pubkey="key", shell="powershell"), - run=AsyncMock( - side_effect=[ - SimpleNamespace(returncode=0, stdout=f"{reported}\r\n".encode()), - SimpleNamespace(returncode=0, stdout=b"C:\\bin\\codex.exe\r\n"), - ] - ), - ) - - executable = await resolve_executable( - cast("Any", ssh), - "codex", - {f"linux-{platform}": "/usr/local/lib/agents/codex/bin/codex"}, - RuntimeConfig(resources=RuntimeResources(os="windows")), - ) - - assert executable == "C:\\bin\\codex.exe" - assert ssh.run.await_args_list[1].args[0] == "where.exe codex" - - -async def test_executable_resolution_rejects_runtime_os_mismatch() -> None: - ssh = SimpleNamespace( - capability=Capability.ssh(url="ssh://localhost:22", host_pubkey="key", shell="bash"), - run=AsyncMock(return_value=SimpleNamespace(returncode=0, stdout=b"Linux\nx86_64\ngnu\n")), - ) - - with pytest.raises(RuntimeError, match=r"requested 'windows'.*reports 'linux'"): - await resolve_executable( - cast("Any", ssh), - "codex", - {}, - RuntimeConfig(resources=RuntimeResources(os="windows")), - ) diff --git a/tests/agents/test_gemini_agent.py b/tests/agents/test_gemini_agent.py deleted file mode 100644 index 33a92c0b7..000000000 --- a/tests/agents/test_gemini_agent.py +++ /dev/null @@ -1,279 +0,0 @@ -"""``GeminiAgent`` — ``get_response`` parsing over a fake Generate Content client, -plus ``_make_tool_call`` mapping and ``_grounding_citations``. -""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, cast - -import mcp.types as mcp_types - -from hud.agents.gemini.agent import GeminiAgent, _grounding_citations -from hud.agents.tool_agent import RunState -from hud.agents.types import GeminiConfig -from hud.types import MCPToolResult, Step, Trace - -if TYPE_CHECKING: - from hud.eval.run import Run - - -class FakeModels: - def __init__(self, response: Any) -> None: - self._response = response - - async def generate_content(self, **_kwargs: Any) -> Any: - return self._response - - -class FakeGenai: - def __init__(self, response: Any) -> None: - self.aio = SimpleNamespace(models=FakeModels(response)) - - -def _agent(response: Any) -> GeminiAgent: - return GeminiAgent( - GeminiConfig(model="gemini-test", include_thoughts=False, model_client=FakeGenai(response)) - ) - - -def _state(agent: GeminiAgent) -> Any: - return RunState(messages=[agent._format_message("user", "go")]) - - -def test_format_message_uses_model_role() -> None: - agent = _agent(SimpleNamespace(candidates=[])) - assert agent._format_message("assistant", "hi").role == "model" - assert agent._format_message("user", "hi").role == "user" - - -def _api_response(*candidates: Any) -> Any: - """A fake ``GenerateContentResponse``: candidates plus the response envelope.""" - return SimpleNamespace( - candidates=list(candidates), - model_version="gemini-test-v2", - usage_metadata=SimpleNamespace( - prompt_token_count=5, - candidates_token_count=3, - cached_content_token_count=None, - ), - ) - - -async def test_get_response_text_and_function_call() -> None: - resp_content = SimpleNamespace( - role="model", - parts=[ - SimpleNamespace(function_call=None, text="hi", thought=None), - SimpleNamespace( - function_call=SimpleNamespace(name="bash", args={"command": "ls"}), - text=None, - thought=None, - ), - ], - ) - response = _api_response( - SimpleNamespace( - content=resp_content, - grounding_metadata=None, - finish_reason=SimpleNamespace(name="STOP"), - ) - ) - agent = _agent(response) - - result = await agent.get_response(_state(agent)) - - assert result.content == "hi" - assert [tc.name for tc in result.tool_calls] == ["bash"] - assert result.done is False - assert result.finish_reason == "STOP" - # Model and usage are normalized off the provider response. - assert result.model == "gemini-test-v2" - assert result.usage is not None - assert result.usage.prompt_tokens == 5 - assert result.usage.completion_tokens == 3 - - -async def test_get_response_done_text_only() -> None: - resp_content = SimpleNamespace( - role="model", - parts=[SimpleNamespace(function_call=None, text="answer", thought=None)], - ) - response = _api_response( - SimpleNamespace(content=resp_content, grounding_metadata=None, finish_reason=None) - ) - agent = _agent(response) - result = await agent.get_response(_state(agent)) - assert result.done is True - assert result.content == "answer" - - -async def test_get_response_no_candidates_raises() -> None: - agent = _agent(SimpleNamespace(candidates=[])) - try: - await agent.get_response(_state(agent)) - except RuntimeError: - pass - else: # pragma: no cover - raise AssertionError("expected RuntimeError for empty candidates") - - -def test_make_tool_call_maps_predefined_to_computer() -> None: - agent = _agent(SimpleNamespace(candidates=[])) - fc = SimpleNamespace(name="click_at", args={"x": 1}) - tc = agent._make_tool_call(cast("Any", fc), cast("Any", SimpleNamespace(name="computer_use"))) - assert tc.name == "computer_use" - assert tc.arguments == {"action": "click_at", "x": 1} - assert tc.provider_name == "click_at" - - -def test_make_tool_call_plain_function() -> None: - agent = _agent(SimpleNamespace(candidates=[])) - fc = cast("Any", SimpleNamespace(name="bash", args={"command": "ls"})) - tc = agent._make_tool_call(fc, None) - assert tc.name == "bash" - assert tc.arguments == {"command": "ls"} - - -def test_grounding_citations() -> None: - meta = SimpleNamespace( - grounding_chunks=[SimpleNamespace(web=SimpleNamespace(uri="http://x", title="T"))], - grounding_supports=[ - SimpleNamespace( - segment=SimpleNamespace(text="seg", start_index=0, end_index=3), - grounding_chunk_indices=[0], - ) - ], - ) - cites = _grounding_citations(cast("Any", meta)) - assert len(cites) == 1 - assert cites[0].source == "http://x" - assert cites[0].type == "grounding" - - -async def test_malformed_function_call_is_a_normalized_error() -> None: - response = _api_response( - SimpleNamespace( - content=None, - grounding_metadata=None, - finish_reason=SimpleNamespace(name="MALFORMED_FUNCTION_CALL"), - ) - ) - agent = _agent(response) - result = await agent.get_response(_state(agent)) - assert result.error == "Provider returned a malformed function call" - assert result.stop_reason == "malformed_tool_call" - assert result.finish_reason == "MALFORMED_FUNCTION_CALL" - assert result.usage is not None - - -class RecordingModels: - """Replays scripted responses, recording the contents each request carried.""" - - def __init__(self, responses: list[Any]) -> None: - self._responses = list(responses) - self.requests: list[Any] = [] - - async def generate_content(self, **kwargs: Any) -> Any: - # Snapshot: the agent goes on mutating the list it passed. - self.requests.append(list(kwargs["contents"])) - return self._responses.pop(0) - - -class RecordingGenai: - def __init__(self, responses: list[Any]) -> None: - self.models = RecordingModels(responses) - self.aio = SimpleNamespace(models=self.models) - - -class BigOutputTool: - """Tool stub whose text block exceeds the configured budget.""" - - name = "big" - provider_name = "big" - - def __init__(self, text: str) -> None: - self._text = text - - async def execute(self, arguments: dict[str, Any]) -> MCPToolResult: - del arguments - return MCPToolResult(content=[mcp_types.TextContent(type="text", text=self._text)]) - - -class FakeRun: - """Offline stand-in for ``Run``: records steps onto a local trace only.""" - - def __init__(self) -> None: - self.trace = Trace() - - def record(self, step: Step) -> None: - self.trace.record(step) - - -def _model_turn(*parts: Any) -> Any: - return _api_response( - SimpleNamespace( - content=SimpleNamespace(role="model", parts=list(parts)), - grounding_metadata=None, - finish_reason=None, - ) - ) - - -def _tool_output(contents: Any) -> str: - parts = contents[-1].parts - assert parts is not None - function_response = parts[0].function_response - assert function_response is not None - response = function_response.response - assert response is not None - return cast("str", response["output"]) - - -async def _run_with_tool_output(text: str, budget: int) -> list[Any]: - """Drive one tool call through the loop, returning the requests Gemini saw.""" - client = RecordingGenai( - [ - _model_turn( - SimpleNamespace( - function_call=SimpleNamespace(name="big", args={}), - text=None, - thought=None, - ) - ), - _model_turn(SimpleNamespace(function_call=None, text="done", thought=None)), - ] - ) - agent = GeminiAgent( - GeminiConfig( - model="gemini-test", - include_thoughts=False, - model_client=client, - max_tool_result_chars=budget, - max_steps=3, - ) - ) - state = _state(agent) - state.tools = {"big": cast("Any", BigOutputTool(text))} - - await agent._loop(cast("Run", FakeRun()), state) - - assert len(client.models.requests) == 2 - return client.models.requests - - -async def test_oversized_tool_output_is_bounded_before_the_next_request() -> None: - requests = await _run_with_tool_output("h" * 500 + "t" * 500, budget=100) - - output = _tool_output(requests[1]) - assert "900 omitted" in output - assert "[900 characters omitted]" in output - assert "h" * 20 in output - assert "h" * 21 not in output - assert "t" * 80 in output - - -async def test_tool_output_within_budget_reaches_the_model_intact() -> None: - requests = await _run_with_tool_output("short", budget=100) - - assert _tool_output(requests[1]) == "short" diff --git a/tests/agents/test_openai_agent.py b/tests/agents/test_openai_agent.py deleted file mode 100644 index f733f268c..000000000 --- a/tests/agents/test_openai_agent.py +++ /dev/null @@ -1,340 +0,0 @@ -"""``OpenAIAgent`` — construction + ``get_response`` parsing of the Responses API, -with a fake ``AsyncOpenAI`` client (no network). -""" - -from __future__ import annotations - -import asyncio -from types import SimpleNamespace -from typing import Any, cast -from unittest.mock import AsyncMock, Mock - -import mcp.types as mcp_types -import pytest -from openai.types.responses import ResponseOutputText - -from hud.agents.openai.agent import EmptyShellCallError, OpenAIAgent, OpenAIRunState -from hud.agents.openai.tools.base import format_openai_result -from hud.agents.openai.tools.computer import OPENAI_COMPUTER_SPEC, OpenAIComputerTool -from hud.agents.types import OpenAIConfig -from hud.capabilities import Capability, RFBClient -from hud.clients.client import HudClient -from hud.eval.run import Run -from hud.types import MCPToolCall, MCPToolResult - - -class FakeResponses: - def __init__(self, response: Any) -> None: - self._response = response - self.calls: list[dict[str, Any]] = [] - - async def create(self, **kwargs: Any) -> Any: - self.calls.append(kwargs) - return self._response - - -class FakeOpenAI: - def __init__(self, response: Any) -> None: - self.responses = FakeResponses(response) - - -def _agent(response: Any) -> OpenAIAgent: - return OpenAIAgent(OpenAIConfig(model="gpt-test", model_client=FakeOpenAI(response))) - - -def test_format_message_shapes_user_text() -> None: - agent = _agent(SimpleNamespace(id="r", output=[])) - msg = cast("dict[str, Any]", agent._format_message("user", "hello")) - assert msg["role"] == "user" - - -def test_format_openai_result_empty_output_emits_one_text_item() -> None: - # The Responses API rejects a function_call_output with an empty output - # list, so a contentless tool result must still produce one input_text item. - call = MCPToolCall(id="call_1", name="noop") - formatted = cast("dict[str, Any]", format_openai_result(call, MCPToolResult(content=[]))) - - assert formatted["type"] == "function_call_output" - assert formatted["call_id"] == "call_1" - assert formatted["output"] == [{"type": "input_text", "text": ""}] - - -def test_format_computer_result_preserves_screenshot_mime_type() -> None: - agent = _agent(SimpleNamespace(id="r", output=[])) - tool = OpenAIComputerTool(spec=OPENAI_COMPUTER_SPEC, client=cast("Any", object())) - state = OpenAIRunState(tools={"computer": tool}) - result = MCPToolResult( - content=[ - mcp_types.ImageContent( - type="image", - data="d2VicA==", - mimeType="image/webp", - ), - ], - ) - - formatted = cast( - "dict[str, Any]", - agent._format_result(MCPToolCall(id="call_1", name="computer"), result, state), - ) - - assert formatted["output"]["image_url"] == "data:image/webp;base64,d2VicA==" - - -def _api_response( - id: str, output: list[Any], usage: Any = None, incomplete_details: Any = None -) -> Any: - """A fake Responses-API payload: output items plus the response envelope.""" - return SimpleNamespace( - id=id, - output=output, - model="gpt-test-v1", - usage=usage, - incomplete_details=incomplete_details, - ) - - -@pytest.fixture -def computer_client() -> Mock: - screen = Mock( - spec=RFBClient, - drain=AsyncMock(), - screenshot_png=AsyncMock(return_value=(b"screen", "image/png")), - ) - return Mock( - spec=HudClient, - manifest=SimpleNamespace( - bindings=[Capability.rfb(name="screen", url="rfb://localhost:5900")], - ), - open=AsyncMock(return_value=screen), - start_task=AsyncMock(return_value={"prompt": "Use the computer."}), - grade=AsyncMock(return_value={"score": 1.0}), - ) - - -@pytest.mark.parametrize( - ("actions", "error"), - [ - ([{"type": "screenshot"}, {"type": "response", "text": "all done"}], None), - ([], "actions list is empty"), - ([{"type": "keypress", "keys": ["UNKNOWN"]}, {"type": "type", "text": "later"}], "UNKNOWN"), - ], - ids=["response", "empty-call", "failed-keypress"], -) -async def test_computer_result_reaches_next_model_turn( - computer_client: Mock, actions: list[dict[str, Any]], error: str | None -) -> None: - screen = computer_client.open.return_value - screen.conn.keyboard.press.side_effect = KeyError("UNKNOWN") - call = SimpleNamespace( - type="computer_call", - call_id="call_1", - actions=[SimpleNamespace(to_dict=lambda action=action: action) for action in actions], - action=None, - pending_safety_checks=[], - ) - create = AsyncMock(side_effect=[_api_response("resp_1", [call]), _api_response("resp_2", [])]) - agent = OpenAIAgent( - OpenAIConfig(model="gpt-test", model_client=Mock(responses=Mock(create=create))), - ) - - async with Run(computer_client, "computer-task", {}) as run: - await agent(run) - - assert run.trace.status == "completed" - assert create.await_count == 2 - feedback = create.call_args_list[1].kwargs["input"] - assert feedback[0]["type"] == "computer_call_output" - assert feedback[0]["call_id"] == "call_1" - assert feedback[0]["output"]["image_url"].startswith("data:image/png;base64,") - if error: - assert error in feedback[1]["content"][0]["text"] - assert ( - "Remaining actions in this call were not executed" in feedback[1]["content"][0]["text"] - ) - screen.conn.keyboard.write.assert_not_called() - - -async def test_computer_error_preserves_failed_screenshot_context(computer_client: Mock) -> None: - computer_client.open.return_value.screenshot_png.side_effect = RuntimeError( - "display unavailable" - ) - call = SimpleNamespace( - type="computer_call", - call_id="call_1", - actions=[], - action=SimpleNamespace(to_dict=lambda: {"type": "frobnicate"}), - pending_safety_checks=[], - ) - agent = _agent(_api_response("resp_1", [call])) - - async with Run(computer_client, "computer-task", {}) as run: - await agent(run) - - assert run.trace.status == "error" - error = next(step.error for step in run.trace.steps if step.error) - assert "frobnicate" in error - assert "display unavailable" in error - - -async def test_computer_screenshot_timeout_preserves_action_context(computer_client: Mock) -> None: - timeout = TimeoutError("capture timed out") - screen = computer_client.open.return_value - screen.screenshot_png.side_effect = timeout - tool = OpenAIComputerTool(spec=OPENAI_COMPUTER_SPEC, client=screen) - - with pytest.raises(TimeoutError, match="capture timed out") as raised: - await tool.execute({"type": "frobnicate"}) - - assert raised.value is timeout - assert any("frobnicate" in note for note in raised.value.__notes__) - - -@pytest.mark.parametrize("error_type", [TimeoutError, asyncio.CancelledError]) -async def test_computer_action_interruption_stops_batch( - computer_client: Mock, error_type: type[BaseException] -) -> None: - screen = computer_client.open.return_value - error = error_type("input interrupted") - screen.drain = AsyncMock(side_effect=error) - tool = OpenAIComputerTool(spec=OPENAI_COMPUTER_SPEC, client=screen) - - with pytest.raises(error_type) as raised: - await tool.execute( - {"actions": [{"type": "move", "x": 3, "y": 4}, {"type": "type", "text": "later"}]}, - ) - - assert raised.value is error - screen.conn.keyboard.write.assert_not_called() - screen.screenshot_png.assert_not_awaited() - - -async def test_get_response_parses_text_and_function_call() -> None: - response = _api_response( - "resp_1", - [ - SimpleNamespace( - type="message", - content=[ResponseOutputText(type="output_text", text="hi", annotations=[])], - ), - SimpleNamespace( - type="function_call", - name="shell", - arguments='{"command": ["ls"]}', - call_id="call_1", - ), - ], - usage=SimpleNamespace( - input_tokens=9, - output_tokens=4, - input_tokens_details=SimpleNamespace(cached_tokens=2), - ), - ) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "go")]) - - result = await agent.get_response(state) - - assert result.content == "hi" - assert [tc.name for tc in result.tool_calls] == ["shell"] - assert result.tool_calls[0].arguments == {"command": ["ls"]} - assert result.done is False - assert state.last_response_id == "resp_1" - # Model and usage are normalized off the provider response. - assert result.model == "gpt-test-v1" - assert result.usage is not None - assert result.usage.prompt_tokens == 9 - assert result.usage.completion_tokens == 4 - assert result.usage.cached_tokens == 2 - - -async def test_get_response_done_when_no_tool_calls() -> None: - response = _api_response("resp_2", []) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "hi")]) - - result = await agent.get_response(state) - assert result.done is True - assert result.tool_calls == [] - assert result.usage is None # provider omitted usage - assert result.finish_reason is None - - -async def test_get_response_reuses_prompt_cache_key() -> None: - agent = _agent(_api_response("resp_cache", [])) - state = OpenAIRunState(messages=[agent._format_message("user", "first")]) - - await agent.get_response(state) - state.messages.append(agent._format_message("user", "second")) - await agent.get_response(state) - - calls = cast("Any", agent.openai_client.responses).calls - assert calls[0]["prompt_cache_key"] == agent.config.prompt_cache_key - assert calls[1]["prompt_cache_key"] == agent.config.prompt_cache_key - - -async def test_get_response_surfaces_token_cap_truncation() -> None: - # Responses has no finish_reason; incomplete_details.reason carries the cap. - response = _api_response( - "resp_3", [], incomplete_details=SimpleNamespace(reason="max_output_tokens") - ) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "hi")]) - - result = await agent.get_response(state) - assert result.finish_reason == "max_output_tokens" - - -async def test_get_response_short_circuits_on_consumed_messages() -> None: - agent = _agent(SimpleNamespace(id="unused", output=[])) - state = OpenAIRunState( - messages=[agent._format_message("user", "go")], - last_response_id="prev", - ) - state.message_cursor = len(state.messages) # nothing new to send - - result = await agent.get_response(state) - assert result.done is True - # No API call should have been made. - assert cast("Any", agent.openai_client.responses).calls == [] - - -async def test_get_response_parses_shell_call() -> None: - response = _api_response( - "resp_3", - [ - SimpleNamespace( - type="shell_call", - action=SimpleNamespace(to_dict=lambda: {"command": ["pwd"]}), - call_id="call_sh", - ), - ], - ) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "run")]) - - result = await agent.get_response(state) - assert [tc.name for tc in result.tool_calls] == ["shell"] - - -async def test_get_response_rejects_empty_shell_call() -> None: - # A shell_call with an empty commands array poisons the stored response - # chain (OpenAI 500s every continuation via previous_response_id), so the - # turn must fail fast and its response id must never be committed. - response = _api_response( - "resp_poisoned", - [ - SimpleNamespace( - type="shell_call", - action=SimpleNamespace(to_dict=lambda: {"commands": [], "timeout_ms": 10000}), - call_id="call_empty", - ), - ], - ) - agent = _agent(response) - state = OpenAIRunState(messages=[agent._format_message("user", "run")]) - - with pytest.raises(EmptyShellCallError, match="empty commands array"): - await agent.get_response(state) - - assert state.last_response_id is None diff --git a/tests/agents/test_openai_compatible_agent.py b/tests/agents/test_openai_compatible_agent.py deleted file mode 100644 index ea607d773..000000000 --- a/tests/agents/test_openai_compatible_agent.py +++ /dev/null @@ -1,113 +0,0 @@ -"""``OpenAIChatAgent`` — chat.completions ``get_response`` parsing + error path.""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any, cast - -import pytest - -from hud.agents.openai_compatible.agent import OpenAIChatAgent, OpenAIChatRunState -from hud.agents.types import OpenAIChatConfig - - -class FakeCompletions: - def __init__(self, response: Any, error: Exception | None = None) -> None: - self._response = response - self._error = error - - async def create(self, **_kwargs: Any) -> Any: - if self._error is not None: - raise self._error - return self._response - - -class FakeOpenAI: - def __init__(self, response: Any, error: Exception | None = None) -> None: - self.chat = SimpleNamespace(completions=FakeCompletions(response, error)) - - -def _agent(response: Any, error: Exception | None = None) -> OpenAIChatAgent: - client = cast("Any", FakeOpenAI(response, error)) - return OpenAIChatAgent(OpenAIChatConfig(model="m", model_client=client)) - - -def _response(content: str, tool_calls: list[Any]) -> Any: - dump: dict[str, Any] = {"role": "assistant", "content": content} - if tool_calls: - dump["tool_calls"] = [ - { - "id": tc.id, - "type": "function", - "function": {"name": tc.function.name, "arguments": tc.function.arguments}, - } - for tc in tool_calls - ] - message = SimpleNamespace( - content=content, - tool_calls=tool_calls, - refusal=None, - model_dump=lambda exclude_none=True: dump, - ) - choice = SimpleNamespace(message=message, finish_reason="stop", logprobs=None) - return SimpleNamespace( - choices=[choice], - model="m-v1", - usage=SimpleNamespace(prompt_tokens=6, completion_tokens=2, prompt_tokens_details=None), - ) - - -def _state(agent: OpenAIChatAgent) -> Any: - return OpenAIChatRunState(messages=[agent._format_message("user", "go")]) - - -async def test_get_response_text_only() -> None: - agent = _agent(_response("hi", [])) - result = await agent.get_response(_state(agent)) - assert result.content == "hi" - assert result.done is True - assert result.tool_calls == [] - # Model and usage are normalized off the provider response. - assert result.model == "m-v1" - assert result.usage is not None - assert result.usage.prompt_tokens == 6 - assert result.usage.completion_tokens == 2 - - -async def test_get_response_with_tool_call() -> None: - tc = SimpleNamespace( - type="function", - id="c1", - function=SimpleNamespace(name="read", arguments='{"path": "x"}'), - ) - agent = _agent(_response("", [tc])) - result = await agent.get_response(_state(agent)) - assert [c.name for c in result.tool_calls] == ["read"] - assert result.tool_calls[0].arguments == {"path": "x"} - assert result.done is False - - -async def test_get_response_error_path() -> None: - agent = _agent(None, error=RuntimeError("boom")) - with pytest.raises(RuntimeError, match="boom"): - await agent.get_response(_state(agent)) - - -async def test_get_response_malformed_tool_args() -> None: - """Unparseable arguments (e.g. truncated at the token limit) yield a call carrying - the raw string, and the recorded message replays "{}" so history stays parseable.""" - tc = SimpleNamespace( - type="function", - id="c1", - function=SimpleNamespace(name="bash", arguments='{"command": "'), - ) - agent = _agent(_response("", [tc])) - state = _state(agent) - result = await agent.get_response(state) - assert result.error is None - assert result.done is False - (call,) = result.tool_calls - assert call.id == "c1" - assert call.arguments == '{"command": "' - recorded = state.messages[-1] - assert recorded["tool_calls"][0]["function"]["arguments"] == "{}" diff --git a/tests/agents/test_provider_native_tools.py b/tests/agents/test_provider_native_tools.py deleted file mode 100644 index 3ff92bbb8..000000000 --- a/tests/agents/test_provider_native_tools.py +++ /dev/null @@ -1,700 +0,0 @@ -"""Provider native tool adapters: translate a provider tool call into SSH execution. - -Each provider exposes its own LLM-facing schema (``to_params``) but executes over a -shared ``SSHClient`` (``self.bash`` -> ``conn.run``). These tests inject a fake SSH -client and assert the command translation + result shape, fully offline. -""" - -from __future__ import annotations - -import base64 -import shlex -from io import BytesIO -from types import SimpleNamespace -from typing import Any, cast - -import asyncssh -import mcp.types as mcp_types -import pytest -from PIL import Image - -from hud.agents.claude.tools.coding import ClaudeBashTool, ClaudeTextEditorTool -from hud.agents.gemini.tools.coding import GeminiEditTool, GeminiShellTool -from hud.agents.openai.tools.coding import OpenAIShellTool -from hud.agents.openai_compatible.agent import OpenAIChatAgent -from hud.agents.openai_compatible.tools import BashTool, EditTool, ReadTool, WriteTool -from hud.agents.tool_agent import RunState -from hud.agents.tools.base import result_text -from hud.agents.tools.ssh import bound_shell_output -from hud.agents.types import OpenAIChatConfig -from hud.capabilities import Capability, SSHClient -from hud.types import MCPToolCall - - -class _Completed: - def __init__(self, *, stdout: str | bytes = "", stderr: str = "", exit_status: int = 0) -> None: - self.stdout = stdout - self.stderr = stderr - self.exit_status = exit_status - self.returncode = exit_status - - -class _Conn: - def __init__(self, completed: _Completed, store: dict[str, bytes]) -> None: - self._completed = completed - self._store = store - self.commands: list[str] = [] - - def is_closed(self) -> bool: - return False - - async def run( - self, - command: str, - *, - input: str | None = None, - check: bool = False, - encoding: str | None = "utf-8", - ) -> _Completed: - self.commands.append(command) - parts = shlex.split(command) - if len(parts) == 3 and parts[:2] == ["cat", "--"]: - if parts[2] not in self._store: - if check: - raise asyncssh.ProcessError( - env=None, - command=command, - subsystem=None, - exit_status=1, - exit_signal=None, - returncode=1, - stdout="", - stderr=f"cat: {parts[2]}: No such file or directory", - ) - return _Completed( - stderr=f"cat: {parts[2]}: No such file or directory", exit_status=1 - ) - data = self._store[parts[2]] - return _Completed(stdout=data if encoding is None else data.decode(encoding)) - if len(parts) == 3 and parts[:2] == ["cat", ">"]: - assert input is not None - self._store[parts[2]] = input.encode() - return _Completed() - if len(parts) == 4 and parts[:3] == ["ls", "-1A", "--"]: - prefix = parts[3].rstrip("/") - prefix = "/" if not prefix else prefix + "/" - names = { - rest.split("/", 1)[0] - for file_path in self._store - if file_path.startswith(prefix) and (rest := file_path[len(prefix) :]) - } - return _Completed(stdout="\n".join(sorted(names))) - if len(parts) == 3 and parts[:2] in (["test", "-d"], ["test", "-e"]): - path = parts[2] - exists = path in self._store or any( - file_path.startswith(path.rstrip("/") + "/") for file_path in self._store - ) - if parts[1] == "-d": - exists = any( - file_path.startswith(path.rstrip("/") + "/") for file_path in self._store - ) - return _Completed(exit_status=0 if exists else 1) - if len(parts) >= 3 and parts[:2] == ["mkdir", "-p"]: - return _Completed(exit_status=0) - return self._completed - - async def create_process(self, command: str, **kwargs: Any) -> _Process: - return _Process(await self.run(command, **kwargs)) - - -class _Process: - def __init__(self, completed: _Completed) -> None: - self.completed = completed - self.closed = False - - async def wait(self, *, check: bool, timeout: float) -> _Completed: # noqa: ASYNC109 - del timeout - if check and self.completed.exit_status: - raise asyncssh.ProcessError( - env=None, - command=None, - subsystem=None, - exit_status=self.completed.exit_status, - exit_signal=None, - returncode=self.completed.exit_status, - stdout=self.completed.stdout, - stderr=self.completed.stderr, - ) - return self.completed - - def close(self) -> None: - self.closed = True - - def terminate(self) -> None: - self.closed = True - - async def wait_closed(self) -> None: - pass - - -class _FakeSSH(SSHClient): - """SSH client with an in-memory exec-channel filesystem.""" - - def __init__( - self, - *, - stdout: str = "ok", - stderr: str = "", - exit_status: int = 0, - files: dict[str, bytes] | None = None, - cwd: str | None = None, - ) -> None: - self.files: dict[str, bytes] = files or {} - params = {"cwd": cwd} if cwd else {} - super().__init__( - Capability(name="shell", protocol="ssh/2", url="ssh://localhost:22", params=params), - cast( - "Any", - _Conn( - _Completed(stdout=stdout, stderr=stderr, exit_status=exit_status), - self.files, - ), - ), - ) - - -def _ssh(**kwargs: Any) -> SSHClient: - return cast("SSHClient", _FakeSSH(**kwargs)) - - -def _commands(tool: Any) -> list[str]: - return tool.client.conn.commands - - -class _OpenAIChatAgentForTest(OpenAIChatAgent): - async def build_tools_for_test(self, ssh: SSHClient) -> tuple[dict[str, Any], list[Any]]: - return await self._build_tools({"ssh": ssh}) - - -# ─── OpenAI shell ───────────────────────────────────────────────────── - - -@pytest.mark.parametrize( - ("command", "timeout_ms", "expected"), - [ - ("pwd", 2500, "timeout 2.5s bash -lc pwd"), - ("pwd", 500, "timeout 0.5s bash -lc pwd"), - ( - "echo ready; sleep infinity", - 500, - "timeout 0.5s bash -lc 'echo ready; sleep infinity'", - ), - ], -) -async def test_openai_shell_applies_requested_timeout_to_entire_command( - command: str, - timeout_ms: int, - expected: str, -) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - - result = await tool.execute({"commands": [command], "timeout_ms": timeout_ms}) - - assert _commands(tool) == [expected] - assert result.isError is False - assert result.structuredContent is not None - assert result.structuredContent["provider_tool"] == "shell" - assert len(result.structuredContent["output"]) == 1 - - -async def test_openai_shell_runs_each_command_without_timeout() -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - - await tool.execute({"commands": ["echo a", "echo b"]}) - - assert _commands(tool) == ["echo a", "echo b"] - - -async def test_openai_shell_has_no_hidden_timeout_across_command_batch( - monkeypatch: pytest.MonkeyPatch, -) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - calls: list[dict[str, Any]] = [] - - async def run(*args: object, **kwargs: Any) -> _Completed: - del args - calls.append(kwargs) - return _Completed() - - monkeypatch.setattr(tool.client, "run", run) - agent = _OpenAIChatAgentForTest( - OpenAIChatConfig(model="test", model_client=cast("Any", object())) - ) - - await agent._dispatch_call( - MCPToolCall(name="shell", arguments={"commands": ["echo a", "echo b"]}), - RunState(tools={"shell": tool}), - ) - - assert len(calls) == 2 - assert all("timeout" not in kwargs for kwargs in calls) - - -async def test_openai_shell_applies_limit_independently_to_each_command() -> None: - limit = 80 - tool = OpenAIShellTool( - spec=OpenAIShellTool.default_spec("gpt-5.5"), - client=_ssh(stdout="stdout-start-" + "a" * 100, stderr="b" * 100 + "-stderr-end"), - ) - - result = await tool.execute( - {"commands": ["first-command", "second-command"], "max_output_length": limit} - ) - - assert result.structuredContent is not None - outputs = result.structuredContent["output"] - assert len(outputs) == 2 - assert all(len(output["stdout"]) + len(output["stderr"]) == limit for output in outputs) - assert all(output["stdout"].startswith("stdout-start-") for output in outputs) - assert all(output["stderr"].endswith("-stderr-end") for output in outputs) - assert sum(len(output["stdout"]) + len(output["stderr"]) for output in outputs) == 2 * limit - text_blocks = [ - block.text for block in result.content if isinstance(block, mcp_types.TextContent) - ] - assert len(text_blocks) == 2 - assert all(len(text) == limit for text in text_blocks) - assert text_blocks == [output["stdout"] + output["stderr"] for output in outputs] - - -@pytest.mark.parametrize(("limit", "expected"), [(1, "["), (10, "[truncated")]) -def test_shared_output_bound_handles_limits_smaller_than_marker( - limit: int, - expected: str, -) -> None: - assert bound_shell_output("x" * 100, "", limit) == (expected, "") - - -@pytest.mark.parametrize("max_output_length", [0, -1, "20000", 20_000.0, True]) -async def test_openai_shell_rejects_invalid_output_limits_without_running( - max_output_length: Any, -) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - - result = await tool.execute( - {"commands": ["echo should-not-run"], "max_output_length": max_output_length} - ) - - assert result.isError is True - assert _commands(tool) == [] - assert result.structuredContent is not None - assert "max_output_length" not in result.structuredContent - assert result_text(result) == "max_output_length must be a positive integer" - - -@pytest.mark.parametrize("max_output_length", [None, 20 * 1024 * 1024]) -async def test_openai_shell_uses_safe_effective_limit(max_output_length: int | None) -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - arguments: dict[str, Any] = {"commands": ["echo ok"]} - if max_output_length is not None: - arguments["max_output_length"] = max_output_length - - result = await tool.execute(arguments) - - assert result.structuredContent is not None - assert result.structuredContent["max_output_length"] == 10 * 1024 * 1024 - - -async def test_openai_shell_rejects_non_list_commands_without_running() -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - - result = await tool.execute({"commands": 123}) - - assert result.isError is True - assert _commands(tool) == [] - - -def test_openai_shell_to_params_is_shell_type() -> None: - tool = OpenAIShellTool(spec=OpenAIShellTool.default_spec("gpt-5.5"), client=_ssh()) - assert tool.to_params()["type"] == "shell" - - -# ─── OpenAI-compatible OpenCode workspace tools ─────────────────────── - - -async def test_openai_compatible_catalog_matches_opencode_workspace_tools() -> None: - agent = _OpenAIChatAgentForTest( - OpenAIChatConfig(model="qwen3.6-plus", model_client=cast("Any", object())) - ) - - tools, params = await agent.build_tools_for_test(_ssh()) - - assert list(tools) == ["bash", "read", "glob", "grep", "edit", "write"] - assert [param["function"]["name"] for param in params] == [ - "bash", - "read", - "glob", - "grep", - "edit", - "write", - ] - - -async def test_openai_compatible_bash_uses_workdir_and_timeout() -> None: - tool = BashTool(spec=BashTool.default_spec("qwen"), client=_ssh()) - - await tool.execute({"command": "echo hi", "workdir": "/tmp/my dir", "timeout": 2500}) - - assert _commands(tool) == ["cd '/tmp/my dir' && timeout 3s bash -lc 'echo hi'"] - - -async def test_shared_ssh_tool_has_no_hidden_command_timeout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - ssh = _ssh() - seen_kwargs: dict[str, Any] = {} - - async def run(*args: object, **kwargs: Any) -> _Completed: - del args - seen_kwargs.update(kwargs) - return _Completed() - - monkeypatch.setattr(ssh, "run", run) - tool = BashTool(spec=BashTool.default_spec("qwen"), client=ssh) - - result = await tool.execute({"command": "sleep forever"}) - - assert result.isError is False - assert "timeout" not in seen_kwargs - - -async def test_shared_ssh_tool_reports_a_signal_returncode( - monkeypatch: pytest.MonkeyPatch, -) -> None: - ssh = _ssh() - - async def signaled(*args: object, **kwargs: Any) -> SimpleNamespace: - del args, kwargs - return SimpleNamespace( - stdout="", - stderr="terminated", - exit_status=None, - returncode=-15, - ) - - monkeypatch.setattr(ssh, "run", signaled) - tool = BashTool(spec=BashTool.default_spec("qwen"), client=ssh) - - result = await tool.execute({"command": "sleep forever"}) - - assert result.isError is True - assert result_text(result).endswith("(exit -15)") - - -async def test_shared_ssh_tool_bounds_combined_output( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr("hud.agents.tools.ssh.MAX_SHELL_OUTPUT_LENGTH", 80) - tool = BashTool( - spec=BashTool.default_spec("qwen"), - client=_ssh( - stdout="stdout-start-" + "a" * 100, - stderr="b" * 100 + "-stderr-end", - ), - ) - - result = await tool.execute({"command": "noisy"}) - - text = result_text(result) - output = text.split("\n", 1)[1].rsplit("\n(exit 0)", 1)[0] - stdout, stderr = output.split("\nstderr:\n", 1) - assert len(stdout) + len(stderr) == 80 - assert stdout.startswith("stdout-start-") - assert stderr.endswith("-stderr-end") - assert "[truncated]" in output - - -async def test_openai_compatible_write_stores_file_via_ssh_exec() -> None: - ssh = _FakeSSH() - tool = WriteTool(spec=WriteTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - result = await tool.execute({"filePath": "/REPORT.md", "content": "done"}) - - assert result.isError is False - assert ssh.files["/REPORT.md"] == b"done" - - -async def test_paths_reach_the_session_verbatim() -> None: - """A path means what it says in the session's own namespace: file helpers - and shell commands must never disagree about what a path names, so - nothing is anchored or rewritten on the way through.""" - ssh = _FakeSSH(cwd="/app", files={"/app/f.txt": b"inside"}) - tool = WriteTool(spec=WriteTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - await tool.execute({"filePath": "/tmp/probe.txt", "content": "done"}) - - assert ssh.files["/tmp/probe.txt"] == b"done" - assert "/app/tmp/probe.txt" not in ssh.files - assert await cast("SSHClient", ssh).read_text("/app/f.txt") == "inside" - - -async def test_read_maps_the_directory_predicate_and_listing_together() -> None: - """`test -d`, listing, and reads must agree on the same path, or - workspace dirs are misclassified as files.""" - ssh = _FakeSSH(cwd="/workspace", files={"/workspace/pkg/mod.py": b"x = 1\n"}) - tool = ReadTool(spec=ReadTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - result = await tool.execute({"filePath": "/workspace/pkg"}) - - text = result_text(result) - assert "directory" in text - assert "mod.py" in text - assert "test -d /workspace/pkg" in cast("Any", ssh).conn.commands - - -async def test_openai_compatible_edit_rewrites_unique_match() -> None: - ssh = _FakeSSH(files={"/f.txt": b"hello old world"}) - tool = EditTool(spec=EditTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - result = await tool.execute( - {"filePath": "/f.txt", "oldString": "old", "newString": "new"}, - ) - - assert result.isError is False - assert ssh.files["/f.txt"] == b"hello new world" - - -async def test_openai_compatible_edit_rejects_ambiguous_match() -> None: - ssh = _FakeSSH(files={"/f.txt": b"a a a"}) - tool = EditTool(spec=EditTool.default_spec("qwen"), client=cast("SSHClient", ssh)) - - result = await tool.execute( - {"filePath": "/f.txt", "oldString": "a", "newString": "b"}, - ) - - assert result.isError is True - assert ssh.files["/f.txt"] == b"a a a" - - -async def test_openai_compatible_read_lists_directories() -> None: - tool = ReadTool( - spec=ReadTool.default_spec("qwen"), - client=_ssh(files={"/work/a.txt": b"a", "/work/nested/b.txt": b"b"}), - ) - - result = await tool.execute({"filePath": "/work"}) - - text = result_text(result) - assert "directory" in text - assert "a.txt" in text - assert "nested" in text - - -async def test_openai_compatible_read_accepts_zero_offset_for_first_page() -> None: - tool = ReadTool( - spec=ReadTool.default_spec("qwen"), - client=_ssh(files={"/f.txt": b"alpha\nbeta\n"}), - ) - - result = await tool.execute({"filePath": "/f.txt", "offset": 0, "limit": 1}) - - text = result_text(result) - assert "1: alpha" in text - assert "2: beta" not in text - - -# ─── Gemini shell ───────────────────────────────────────────────────── - - -async def test_gemini_shell_scopes_command_to_quoted_directory() -> None: - tool = GeminiShellTool(spec=GeminiShellTool.default_spec("gemini"), client=_ssh()) - - await tool.execute({"command": "ls -la", "dir_path": "/tmp/my dir"}) - - assert _commands(tool) == ["cd '/tmp/my dir' && ls -la"] - - -async def test_gemini_shell_runs_bare_command() -> None: - tool = GeminiShellTool(spec=GeminiShellTool.default_spec("gemini"), client=_ssh()) - - await tool.execute({"command": "ls"}) - - assert _commands(tool) == ["ls"] - - -async def test_gemini_shell_requires_command() -> None: - tool = GeminiShellTool(spec=GeminiShellTool.default_spec("gemini"), client=_ssh()) - - with pytest.raises(ValueError, match="command is required"): - await tool.execute({"command": ""}) - - -# ─── Claude bash ────────────────────────────────────────────────────── - - -async def test_claude_bash_runs_command() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude-sonnet-4-6"), client=_ssh()) - - await tool.execute({"command": "echo hi"}) - - assert _commands(tool) == ["echo hi"] - - -async def test_claude_bash_restart_is_a_noop() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude-sonnet-4-6"), client=_ssh()) - - result = await tool.execute({"restart": True}) - - assert result.isError is False - assert result_text(result) == ( - "restart is unnecessary; each command runs in a fresh shell session" - ) - assert _commands(tool) == [] # restart never touches the shell - - -async def test_claude_bash_requires_command() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude-sonnet-4-6"), client=_ssh()) - - result = await tool.execute({}) - - assert result.isError is True - assert _commands(tool) == [] - - -def test_claude_bash_to_params_carries_native_schema() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude-sonnet-4-6"), client=_ssh()) - params = tool.to_params() - assert params == {"type": "bash_20250124", "name": "bash"} - - -def test_claude_bash_generic_spec_for_non_anthropic_model() -> None: - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("qwen/qwen3.8-max"), client=_ssh()) - params: Any = tool.to_params() - assert "type" not in params - assert params["name"] == "bash" - assert params["input_schema"]["required"] == ["command"] - - -def test_claude_editor_generic_spec_for_non_anthropic_model() -> None: - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("qwen/qwen3.8-max"), - client=_ssh(), - ) - params: Any = tool.to_params() - assert "type" not in params - assert params["name"] == "str_replace_based_edit_tool" - assert params["input_schema"]["required"] == ["command", "path"] - - -# ─── editor tools over SSH exec ─────────────────────────────────────── - - -async def test_claude_text_editor_creates_file() -> None: - ssh = _FakeSSH() - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), client=cast("SSHClient", ssh) - ) - - result = await tool.execute({"command": "create", "path": "/f.txt", "file_text": "hello"}) - - assert result.isError is False - assert ssh.files["/f.txt"] == b"hello" - - -async def test_claude_text_editor_str_replace_rewrites_file() -> None: - ssh = _FakeSSH(files={"/f.txt": b"hello old world"}) - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), client=cast("SSHClient", ssh) - ) - - result = await tool.execute( - {"command": "str_replace", "path": "/f.txt", "old_str": "old", "new_str": "new"}, - ) - - assert result.isError is False - assert ssh.files["/f.txt"] == b"hello new world" - - -async def test_claude_text_editor_str_replace_errors_when_not_unique() -> None: - ssh = _FakeSSH(files={"/f.txt": b"a a a"}) - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), client=cast("SSHClient", ssh) - ) - - result = await tool.execute( - {"command": "str_replace", "path": "/f.txt", "old_str": "a", "new_str": "b"}, - ) - - assert result.isError is True # ambiguous match must not write - assert ssh.files["/f.txt"] == b"a a a" - - -async def test_gemini_edit_creates_file_when_old_string_empty() -> None: - ssh = _FakeSSH() - tool = GeminiEditTool(spec=GeminiEditTool.default_spec("gemini"), client=cast("SSHClient", ssh)) - - await tool.execute({"file_path": "/n.txt", "old_string": "", "new_string": "fresh"}) - - assert ssh.files["/n.txt"] == b"fresh" - - -async def test_reading_a_missing_file_is_a_tool_error_not_a_raised_traceback() -> None: - """Reading before creating is the first thing an editor tool does; that failure - must come back as a tool result carrying the shell's message.""" - ssh = _FakeSSH() - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), client=cast("SSHClient", ssh) - ) - - result = await tool.execute({"command": "view", "path": "/nope.txt"}) - - assert result.isError is True - assert "No such file or directory" in result_text(result) - - -@pytest.mark.parametrize("image_format", ["PNG", "JPEG", "GIF", "WEBP"]) -@pytest.mark.parametrize("tool_class", [ClaudeTextEditorTool, ReadTool]) -async def test_file_view_returns_images_as_image_content( - image_format: str, tool_class: type[ClaudeTextEditorTool] | type[ReadTool] -) -> None: - buffer = BytesIO() - Image.new("RGB", (32, 24), "red").save(buffer, format=image_format) - data = buffer.getvalue() - ssh = _FakeSSH(files={"/image with no extension": data}) - tool = tool_class(spec=tool_class.default_spec("claude"), client=ssh) - - result = await tool.execute( - { - "command": "view", - "path": "/image with no extension", - "filePath": "/image with no extension", - } - ) - - assert not result.isError - assert len(result.content) == 1 - content = result.content[0] - assert isinstance(content, mcp_types.ImageContent) - assert content.mimeType == Image.MIME[image_format] - assert base64.b64decode(content.data) == data - - -@pytest.mark.parametrize("data", [b"\x00binary", b"\xff\xfeinvalid"]) -async def test_file_view_rejects_non_image_binary(data: bytes) -> None: - ssh = _FakeSSH(files={"/binary": data}) - tool = ClaudeTextEditorTool(spec=ClaudeTextEditorTool.default_spec("claude"), client=ssh) - - result = await tool.execute({"command": "view", "path": "/binary"}) - - assert result.isError - assert "expected UTF-8 text or an image" in result_text(result) - - -async def test_file_view_preserves_unicode_text() -> None: - text = "Hello, 世界\n" - ssh = _FakeSSH(files={"/text.png": text.encode()}) - tool = ClaudeTextEditorTool(spec=ClaudeTextEditorTool.default_spec("claude"), client=ssh) - - result = await tool.execute({"command": "view", "path": "/text.png"}) - - assert not result.isError - assert result_text(result) == text diff --git a/tests/agents/test_robot_dataset_writer.py b/tests/agents/test_robot_dataset_writer.py deleted file mode 100644 index 9829f6f51..000000000 --- a/tests/agents/test_robot_dataset_writer.py +++ /dev/null @@ -1,162 +0,0 @@ -"""DatasetWriter shares by compatible schema/FPS, not a single process-global.""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock - -import numpy as np -import pytest - -from hud.agents.robot import dataset as dataset_mod -from hud.agents.robot.dataset import DatasetWriter - - -def _contract( - *, state_shape: tuple[int, ...], action_shape: tuple[int, ...], robot: str -) -> dict[str, Any]: - return { - "robot_type": robot, - "features": { - "state": {"role": "observation", "dtype": "float32", "shape": list(state_shape)}, - "action": {"role": "action", "dtype": "float32", "shape": list(action_shape)}, - }, - } - - -@pytest.fixture(autouse=True) -def reset_dataset_writer_globals(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: - DatasetWriter._datasets.clear() - DatasetWriter._open.clear() - DatasetWriter._atexit_registered = False - monkeypatch.setenv("RECORD_DIR", str(tmp_path)) - monkeypatch.delenv("HF_REPO", raising=False) - - -def _install_fake_lerobot(monkeypatch: pytest.MonkeyPatch) -> list[MagicMock]: - created: list[MagicMock] = [] - - def create(**kwargs: Any) -> MagicMock: - ds = MagicMock(name=f"ds-{len(created)}") - ds.create_kwargs = kwargs - created.append(ds) - return ds - - monkeypatch.setitem( - __import__("sys").modules, - "lerobot.datasets.lerobot_dataset", - SimpleNamespace(LeRobotDataset=SimpleNamespace(create=create)), - ) - - def _found(_name: str) -> object: - return object() - - monkeypatch.setattr(dataset_mod.importlib.util, "find_spec", _found) - return created - - -def test_matching_writers_share_one_dataset(monkeypatch: pytest.MonkeyPatch) -> None: - created = _install_fake_lerobot(monkeypatch) - contract = _contract(state_shape=(3,), action_shape=(2,), robot="arm") - obs, act = {"state": np.zeros(3, dtype=np.float32)}, np.zeros(2, dtype=np.float32) - DatasetWriter(contract, fps=10).add(obs, act, task="t") - DatasetWriter(contract, fps=10).add(obs, act, task="t") - assert len(created) == 1 - assert len(DatasetWriter._datasets) == 1 - - -def test_different_fps_or_features_get_separate_datasets( - monkeypatch: pytest.MonkeyPatch, -) -> None: - created = _install_fake_lerobot(monkeypatch) - c1 = _contract(state_shape=(3,), action_shape=(2,), robot="arm") - c2 = _contract(state_shape=(4,), action_shape=(2,), robot="arm") - act = np.zeros(2, dtype=np.float32) - DatasetWriter(c1, fps=10).add({"state": np.zeros(3, dtype=np.float32)}, act, task="t") - DatasetWriter(c1, fps=20).add({"state": np.zeros(3, dtype=np.float32)}, act, task="t") - DatasetWriter(c2, fps=10).add({"state": np.zeros(4, dtype=np.float32)}, act, task="t") - assert len(created) == 3 - assert created[0].create_kwargs["fps"] == 10 - assert created[1].create_kwargs["fps"] == 20 - assert tuple(created[2].create_kwargs["features"]["observation.state"]["shape"]) == (4,) - - -def test_different_feature_names_get_separate_datasets( - monkeypatch: pytest.MonkeyPatch, -) -> None: - created = _install_fake_lerobot(monkeypatch) - first = _contract(state_shape=(2,), action_shape=(1,), robot="arm") - second = _contract(state_shape=(2,), action_shape=(1,), robot="arm") - first["features"]["state"]["names"] = ["shoulder", "elbow"] - second["features"]["state"]["names"] = ["x", "y"] - obs = {"state": np.zeros(2, dtype=np.float32)} - action = np.zeros(1, dtype=np.float32) - - DatasetWriter(first, fps=10).add(obs, action, task="t") - DatasetWriter(second, fps=10).add(obs, action, task="t") - - assert len(created) == 2 - assert created[0].create_kwargs["features"]["observation.state"]["names"] == [ - "shoulder", - "elbow", - ] - assert created[1].create_kwargs["features"]["observation.state"]["names"] == ["x", "y"] - - -def test_finalize_clears_all_datasets(monkeypatch: pytest.MonkeyPatch) -> None: - created = _install_fake_lerobot(monkeypatch) - act = np.zeros(2, dtype=np.float32) - a = DatasetWriter(_contract(state_shape=(3,), action_shape=(2,), robot="a"), fps=10) - b = DatasetWriter(_contract(state_shape=(5,), action_shape=(2,), robot="b"), fps=10) - a.add({"state": np.zeros(3, dtype=np.float32)}, act, task="t") - b.add({"state": np.zeros(5, dtype=np.float32)}, act, task="t") - a.end_episode() - b.end_episode() - DatasetWriter.finalize() - assert DatasetWriter._datasets == {} - for ds in created: - ds.finalize.assert_called_once() - - -def test_nested_image_keys_stay_distinct(monkeypatch: pytest.MonkeyPatch) -> None: - created = _install_fake_lerobot(monkeypatch) - contract = { - "robot_type": "stereo", - "features": { - "left/image": {"role": "observation", "dtype": "image", "shape": [2, 2, 3]}, - "right/image": {"role": "observation", "dtype": "image", "shape": [2, 2, 3]}, - "state": {"role": "observation", "dtype": "float32", "shape": [1]}, - "action": {"role": "action", "dtype": "float32", "shape": [1]}, - }, - } - writer = DatasetWriter(contract, fps=10) - left = np.zeros((2, 2, 3), dtype=np.uint8) - right = np.full((2, 2, 3), 255, dtype=np.uint8) - writer.add( - {"left/image": left, "right/image": right, "state": np.zeros(1, dtype=np.float32)}, - np.zeros(1, dtype=np.float32), - task="t", - ) - writer.end_episode() - - features = created[0].create_kwargs["features"] - assert "observation.images.left_image" in features - assert "observation.images.right_image" in features - frame = created[0].add_frame.call_args.args[0] - assert frame["observation.images.left_image"] is left - assert frame["observation.images.right_image"] is right - - -def test_duplicate_mapped_keys_raise() -> None: - contract = { - "robot_type": "bad", - "features": { - # Both wire names become the same LeRobot key after path flattening. - "cam/rgb": {"role": "observation", "dtype": "image", "shape": [2, 2, 3]}, - "cam_rgb": {"role": "observation", "dtype": "image", "shape": [2, 2, 3]}, - "action": {"role": "action", "dtype": "float32", "shape": [1]}, - }, - } - with pytest.raises(ValueError, match="both map to LeRobot key"): - DatasetWriter(contract, fps=10) diff --git a/tests/agents/test_tool_agent.py b/tests/agents/test_tool_agent.py deleted file mode 100644 index 19ff303b8..000000000 --- a/tests/agents/test_tool_agent.py +++ /dev/null @@ -1,826 +0,0 @@ -"""``ToolAgent`` plumbing: catalog→clients, message formatting, dispatch + loop. - -The provider-specific bits are abstract; this drives a tiny concrete subclass with a -scripted ``get_response`` so the loop, dispatch, and message formatting run offline. -""" - -from __future__ import annotations - -import asyncio -from contextlib import AsyncExitStack -from types import SimpleNamespace -from typing import TYPE_CHECKING, Any, Literal, cast -from unittest.mock import AsyncMock, Mock - -import fastmcp -import mcp.types as mcp_types -import pytest -from fastmcp.client.transports import SSETransport, StreamableHttpTransport - -from hud.agents.claude.agent import ClaudeAgent -from hud.agents.claude.tools.coding import ClaudeBashTool, ClaudeTextEditorTool -from hud.agents.openai.tools.coding import OpenAIShellTool -from hud.agents.openai.tools.mcp_proxy import OpenAIMCPProxyTool -from hud.agents.tool_agent import DegenerateTurnError, RunState, ToolAgent -from hud.agents.tools.base import AgentToolSpec, result_text -from hud.agents.tools.rfb import RFBTool -from hud.agents.tools.ssh import SSHInfrastructureErrorResult -from hud.agents.types import ( - AgentStep, - ClaudeCLIConfig, - ClaudeConfig, - ToolAgentConfig, - ToolStep, -) -from hud.capabilities import ( - Capability, - CapabilityClient, - MCPClient, - RFBClient, - SSHClient, -) -from hud.capabilities.mcp import get_mcp_trace_id -from hud.capabilities.rfb import PngScreenshotEncoding, WebPScreenshotEncoding -from hud.capabilities.ssh import SSHConnectionError -from hud.telemetry.context import set_trace_context -from hud.types import MCPToolCall, MCPToolResult, Step, Trace - -if TYPE_CHECKING: - from hud.eval.run import Run - -_Msg = dict[str, Any] - - -class _FakeRun: - """Offline stand-in for ``Run``: records steps onto a local trace only.""" - - def __init__(self) -> None: - self.trace = Trace() - - def record(self, step: Step) -> None: - self.trace.record(step) - - -class DictAgent(ToolAgent[_Msg, ToolAgentConfig]): - """Minimal concrete ToolAgent over plain-dict messages.""" - - config_cls = ToolAgentConfig - - def __init__(self, turns: list[AgentStep], **config: Any) -> None: - super().__init__(ToolAgentConfig(model="test-model", **config)) - self._turns = list(turns) - - async def _initialize_state(self, *, prompt: Any) -> RunState[_Msg]: - return RunState(messages=self._initial_messages(prompt)) - - async def get_response( - self, state: RunState[_Msg], *, system_prompt: Any = None, citations_enabled: bool = False - ) -> AgentStep: - return self._turns.pop(0) - - def _format_message(self, role: str, text: str) -> _Msg: - return {"role": role, "content": text} - - def _format_result( - self, call: MCPToolCall, result: MCPToolResult, state: RunState[_Msg] - ) -> _Msg: - return {"role": "tool", "name": call.name, "isError": result.isError} - - -# ─── catalog → clients derivation ───────────────────────────────────── - - -def test_init_subclass_derives_clients_from_catalog() -> None: - class WithCatalog(DictAgent): - tool_catalog = (OpenAIShellTool,) - - assert WithCatalog.clients == (SSHClient,) - - -def test_claude_defaults_to_configurable_webp_screenshots() -> None: - assert ToolAgentConfig().screenshot_encoding == PngScreenshotEncoding() - assert ClaudeConfig().screenshot_encoding == WebPScreenshotEncoding() - assert ClaudeCLIConfig().screenshot_encoding == WebPScreenshotEncoding() - - configured = ClaudeConfig.model_validate( - {"screenshot_encoding": {"mime_type": "image/webp", "quality": 42}}, - ) - assert configured.screenshot_encoding == WebPScreenshotEncoding(quality=42) - - with pytest.raises(ValueError): - ClaudeConfig.model_validate( - {"screenshot_encoding": {"mime_type": "image/webp", "quality": 101}}, - ) - - -def test_only_claude_provider_has_a_default_tool_timeout() -> None: - config = ClaudeConfig(timeout_seconds=600) - - assert config.timeout_seconds == 600 - assert config.tool_timeout_seconds == 120 - assert ClaudeConfig(tool_timeout_seconds=None).tool_timeout_seconds is None - assert ToolAgentConfig().tool_timeout_seconds is None - - -async def test_agent_passes_screenshot_encoding_to_rfb_tools() -> None: - class ScreenTool(RFBTool): - name = "screen" - - @classmethod - def default_spec(cls, model: str) -> AgentToolSpec: - del model - return AgentToolSpec(api_type="screen", api_name="screen") - - async def execute(self, arguments: dict[str, Any]) -> MCPToolResult: - del arguments - return MCPToolResult(content=[]) - - def to_params(self) -> dict[str, str]: - return {"name": self.name} - - class ScreenAgent(DictAgent): - tool_catalog = (ScreenTool,) - - encoding = {"mime_type": "image/webp", "quality": 42} - agent = ScreenAgent([AgentStep(content="done", done=True)], screenshot_encoding=encoding) - rfb = object.__new__(RFBClient) - - tools, _ = await agent._build_tools({"screen": rfb}) - - tool = tools["screen"] - assert isinstance(tool, ScreenTool) - assert tool.screenshot_encoding == WebPScreenshotEncoding(quality=42) - - -async def test_agent_opens_every_mcp_capability_by_name() -> None: - capabilities = [ - Capability.mcp(name="database", url="http://database:8000/mcp"), - Capability.mcp(name="search", url="http://search:8000/mcp"), - ] - opened: list[str] = [] - - class Client: - manifest = SimpleNamespace(bindings=capabilities) - - async def open(self, ref: str) -> CapabilityClient: - opened.append(ref) - return cast("CapabilityClient", object()) - - class MultiMCPAgent(DictAgent): - clients = (MCPClient,) - - class LiveRun(_FakeRun): - def __init__(self) -> None: - super().__init__() - self.client = Client() - self.prompt_messages: list[Any] = [] - - await MultiMCPAgent([AgentStep(content="done", done=True)])(cast("Any", LiveRun())) - - assert opened == ["database", "search"] - - -async def test_agent_opens_only_one_non_mcp_capability_per_protocol() -> None: - capabilities = [ - Capability.rfb(name="screen-0", url="rfb://display-0", display=0), - Capability.rfb(name="screen-1", url="rfb://display-1", display=1), - ] - opened: list[str] = [] - - class Client: - manifest = SimpleNamespace(bindings=capabilities) - - async def open(self, ref: str) -> CapabilityClient: - opened.append(ref) - return cast("CapabilityClient", object()) - - class ComputerAgent(DictAgent): - clients = (RFBClient,) - - class LiveRun(_FakeRun): - def __init__(self) -> None: - super().__init__() - self.client = Client() - self.prompt_messages: list[Any] = [] - - await ComputerAgent([AgentStep(content="done", done=True)])(cast("Any", LiveRun())) - - assert opened == ["screen-0"] - - -async def test_mcp_capability_names_do_not_collide_with_protocol_keys() -> None: - capabilities = [ - Capability.mcp(name="ssh/2", url="http://database:8000/mcp"), - Capability.ssh(name="shell", url="ssh://workspace", host_pubkey="key"), - ] - opened: list[str] = [] - - class Client: - manifest = SimpleNamespace(bindings=capabilities) - - async def open(self, ref: str) -> CapabilityClient: - opened.append(ref) - return cast("CapabilityClient", object()) - - class MCPAndShellAgent(DictAgent): - clients = (MCPClient, SSHClient) - - class LiveRun(_FakeRun): - def __init__(self) -> None: - super().__init__() - self.client = Client() - self.prompt_messages: list[Any] = [] - - await MCPAndShellAgent([AgentStep(content="done", done=True)])(cast("Any", LiveRun())) - - assert opened == ["ssh/2", "shell"] - - -@pytest.mark.parametrize( - ("transport", "expected_type"), - [("sse", SSETransport), ("streamable-http", StreamableHttpTransport)], -) -async def test_mcp_client_uses_the_declared_http_transport( - monkeypatch: pytest.MonkeyPatch, - transport: Literal["sse", "streamable-http"], - expected_type: type[SSETransport] | type[StreamableHttpTransport], -) -> None: - transports: list[Any] = [] - - class Client: - def __init__(self, selected: Any, **_kwargs: Any) -> None: - transports.append(selected) - - async def __aenter__(self) -> Client: - return self - - async def __aexit__(self, *_exc: object) -> None: - return None - - monkeypatch.setattr(fastmcp, "Client", Client) - capability = Capability.mcp( - url="https://tools.example/events", - transport=transport, - ) - - client = await MCPClient.connect(capability) - await client.close() - - assert isinstance(transports[0], expected_type) - - -async def test_mcp_client_propagates_active_trace_context() -> None: - calls: list[dict[str, Any]] = [] - - class Client: - async def call_tool_mcp(self, **kwargs: Any) -> mcp_types.CallToolResult: - calls.append(kwargs) - return mcp_types.CallToolResult(content=[]) - - capability = Capability.mcp(url="https://tools.example/mcp") - client = MCPClient(capability, cast("Any", Client()), AsyncExitStack()) - - with set_trace_context("child-trace"): - await client.call_tool("verify", {}) - - assert calls == [ - { - "name": "verify", - "arguments": {}, - "meta": {"hud/trace-id": "child-trace"}, - } - ] - - -def test_mcp_server_reads_propagated_trace_context(monkeypatch: pytest.MonkeyPatch) -> None: - request_context = SimpleNamespace( - meta=SimpleNamespace(model_extra={"hud/trace-id": "child-trace"}) - ) - monkeypatch.setattr( - "fastmcp.server.dependencies.get_context", - lambda: SimpleNamespace(request_context=request_context), - ) - - assert get_mcp_trace_id() == "child-trace" - - -def test_mcp_server_reads_propagated_trace_header(monkeypatch: pytest.MonkeyPatch) -> None: - request_context = SimpleNamespace(meta=SimpleNamespace(model_extra={})) - monkeypatch.setattr( - "fastmcp.server.dependencies.get_context", - lambda: SimpleNamespace(request_context=request_context), - ) - monkeypatch.setattr( - "fastmcp.server.dependencies.get_http_headers", - lambda: { - "trace-id": "child-trace", - }, - ) - - assert get_mcp_trace_id() == "child-trace" - - -async def test_multiple_mcp_capabilities_qualify_tool_names() -> None: - class Client(MCPClient): - def __init__(self, *names: str) -> None: - self.tools = [ - mcp_types.Tool( - name=name, - description=f"Run {name}", - inputSchema={"type": "object", "properties": {}}, - ) - for name in names - ] - self.calls: list[str] = [] - - async def list_tools(self) -> list[mcp_types.Tool]: - return self.tools - - async def call_tool(self, name: str, arguments: dict[str, Any]) -> MCPToolResult: - self.calls.append(name) - return MCPToolResult(content=[]) - - class MultiMCPAgent(DictAgent): - tool_catalog = (OpenAIMCPProxyTool,) - - database = Client("lookup", "write") - search = Client("lookup", "find") - tools, params = await MultiMCPAgent([])._build_tools({"database": database, "search": search}) - - assert list(tools) == [ - "database__lookup", - "database__write", - "search__lookup", - "search__find", - ] - assert [param["name"] for param in params] == list(tools) - - await tools["database__lookup"].execute({}) - assert database.calls == ["lookup"] - assert search.calls == [] - - single, _ = await MultiMCPAgent([])._build_tools({"database": Client("lookup")}) - assert list(single) == ["lookup"] - - -async def test_multiple_mcp_capabilities_reject_qualified_name_collisions() -> None: - class Client(MCPClient): - def __init__(self, name: str) -> None: - self.tool = mcp_types.Tool( - name=name, - description=f"Run {name}", - inputSchema={"type": "object", "properties": {}}, - ) - - async def list_tools(self) -> list[mcp_types.Tool]: - return [self.tool] - - async def call_tool(self, name: str, arguments: dict[str, Any]) -> MCPToolResult: - raise AssertionError("colliding tools must not be callable") - - class MultiMCPAgent(DictAgent): - tool_catalog = (OpenAIMCPProxyTool,) - - with pytest.raises(ValueError, match=r"MCP tool name collision.*a__b__c"): - await MultiMCPAgent([])._build_tools({"a": Client("b__c"), "a__b": Client("c")}) - - -async def test_qualified_mcp_names_are_valid_provider_tool_names() -> None: - class Client(MCPClient): - def __init__(self, name: str) -> None: - self.tool = mcp_types.Tool( - name=name, - description=f"Run {name}", - inputSchema={"type": "object", "properties": {}}, - ) - - async def list_tools(self) -> list[mcp_types.Tool]: - return [self.tool] - - async def call_tool(self, name: str, arguments: dict[str, Any]) -> MCPToolResult: - return MCPToolResult(content=[]) - - class MultiMCPAgent(DictAgent): - tool_catalog = (OpenAIMCPProxyTool,) - - long_name = "lookup_" * 10 - tools, params = await MultiMCPAgent([])._build_tools( - { - "company.tools": Client(long_name + "a"), - "company-tools": Client(long_name + "b"), - } - ) - - assert len(tools) == 2 - assert set(tools) == {param["name"] for param in params} - assert all( - len(name) <= 64 and name.replace("_", "").replace("-", "").isalnum() for name in tools - ) - - -# ─── initial messages / user text formatting ────────────────────────── - - -def test_initial_messages_formats_each_turn() -> None: - agent = DictAgent([]) - turn = mcp_types.PromptMessage( - role="user", content=mcp_types.TextContent(type="text", text="a") - ) - assert agent._initial_messages([turn]) == [{"role": "user", "content": "a"}] - assert agent._format_user_text("hey") == {"role": "user", "content": "hey"} - - -# ─── dispatch + loop ────────────────────────────────────────────────── - - -async def test_dispatch_unknown_tool_returns_error_result() -> None: - agent = DictAgent([]) - result = await agent._dispatch_call(MCPToolCall(name="ghost"), RunState()) - assert result.isError is True - - -async def test_dispatch_unparsed_arguments_returns_error_result() -> None: - # A call whose provider arguments never parsed is answered, not executed. - agent = DictAgent([]) - call = MCPToolCall(name="bash", arguments='{"command": "') - result = await agent._dispatch_call(call, RunState()) - assert result.isError is True - content = result.content[0] - assert isinstance(content, mcp_types.TextContent) - assert "not executed" in content.text - assert '{"command": "' in content.text # the raw prefix re-anchors the model - - -async def test_dispatch_marks_exhausted_ssh_reconnect_as_infrastructure_error() -> None: - agent = DictAgent([]) - tool = SimpleNamespace( - execute=AsyncMock(side_effect=SSHConnectionError("SSH reconnect failed after 3 attempts")) - ) - state: RunState[_Msg] = RunState(tools={"bash": cast("Any", tool)}) - - result = await agent._dispatch_call(MCPToolCall(name="bash"), state) - - assert result.isError is True - assert isinstance(result, SSHInfrastructureErrorResult) - - -async def test_dispatch_reraises_tool_timeout_before_deadline() -> None: - ssh = SimpleNamespace(run=AsyncMock(side_effect=TimeoutError("remote timed out"))) - tool = ClaudeBashTool( - spec=ClaudeBashTool.default_spec("claude"), - client=cast("Any", ssh), - ) - agent = DictAgent([], tool_timeout_seconds=120) - state: RunState[_Msg] = RunState(tools={"bash": tool}) - - with pytest.raises(TimeoutError, match="remote timed out"): - await agent._dispatch_call( - MCPToolCall(name="bash", arguments={"command": "sleep forever"}), - state, - ) - - -async def test_claude_bash_timeout_terminates_process_and_continues_loop( - monkeypatch: pytest.MonkeyPatch, -) -> None: - started = asyncio.Event() - - async def wait(*, check: bool, timeout: None) -> None: # noqa: ASYNC109 - del check, timeout - started.set() - await asyncio.Event().wait() - - process = SimpleNamespace( - wait=AsyncMock(side_effect=wait), - terminate=Mock(), - close=Mock(side_effect=AssertionError("graceful termination should succeed")), - wait_closed=AsyncMock(), - ) - connection = SimpleNamespace( - is_closed=Mock(return_value=False), - create_process=AsyncMock(return_value=process), - ) - ssh = SSHClient( - Capability.ssh(url="ssh://workspace", host_pubkey="key"), - cast("Any", connection), - ) - tool = ClaudeBashTool(spec=ClaudeBashTool.default_spec("claude"), client=ssh) - agent = ClaudeAgent( - ClaudeConfig( - model="claude-test", - model_client=cast("Any", object()), - tool_timeout_seconds=0.01, - ) - ) - responses = AsyncMock( - side_effect=[ - AgentStep( - content="", - done=False, - tool_calls=[ - MCPToolCall(id="tool-1", name="bash", arguments={"command": "sleep forever"}) - ], - ), - AgentStep(content="recovered", done=True), - ] - ) - monkeypatch.setattr(agent, "get_response", responses) - state = RunState(messages=[], tools={"bash": tool}) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, state) - - assert started.is_set() - process.terminate.assert_called_once_with() - process.wait_closed.assert_awaited_once_with() - assert responses.await_count == 2 - assert run.trace.status == "completed" - assert run.trace.content == "recovered" - tool_result = cast("list[Any]", state.messages[0]["content"])[0] - error_block = cast("list[Any]", tool_result["content"])[0] - assert error_block["text"] == ( - "Error: bash timed out after 0.01s; retry with a shorter command" - ) - - -async def test_composite_editor_uses_one_tool_deadline( - monkeypatch: pytest.MonkeyPatch, -) -> None: - deadline = Mock(wraps=asyncio.timeout) - monkeypatch.setattr(asyncio, "timeout", deadline) - ssh = SimpleNamespace( - read_text=AsyncMock(return_value="old"), - write_text=AsyncMock(), - ) - tool = ClaudeTextEditorTool( - spec=ClaudeTextEditorTool.default_spec("claude"), - client=cast("Any", ssh), - ) - agent = DictAgent([], tool_timeout_seconds=120) - state: RunState[_Msg] = RunState(tools={tool.provider_name: tool}) - - result = await agent._dispatch_call( - MCPToolCall( - name=tool.provider_name, - arguments={ - "command": "str_replace", - "path": "/file.txt", - "old_str": "old", - "new_str": "new", - }, - ), - state, - ) - - assert result.isError is False - assert result_text(result) == "wrote 3 bytes to /file.txt" - deadline.assert_called_once_with(120) - ssh.read_text.assert_awaited_once_with("/file.txt") - ssh.write_text.assert_awaited_once_with("/file.txt", "new") - - -async def test_loop_finishes_on_done_response() -> None: - agent = DictAgent([AgentStep(content="final answer", done=True)]) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.status == "completed" - assert run.trace.content == "final answer" - assert run.trace.is_error is False - assert run.trace.stop_reason == "done" - assert run.trace.is_truncated is False - # The agent turn was recorded directly, with loop-stamped fallbacks. - (step,) = run.trace.steps - assert isinstance(step, AgentStep) - assert step.source == "agent" - assert step.content == "final answer" - assert step.model == "test-model" - assert step.started_at is not None - - -class _DegenerateDictAgent(DictAgent): - """DictAgent whose scripted turns may be ``DegenerateTurnError`` instances.""" - - def __init__(self, turns: list[AgentStep | DegenerateTurnError], **config: Any) -> None: - super().__init__(cast("list[AgentStep]", turns), **config) - - async def get_response( - self, state: RunState[_Msg], *, system_prompt: Any = None, citations_enabled: bool = False - ) -> AgentStep: - turn = await super().get_response( - state, system_prompt=system_prompt, citations_enabled=citations_enabled - ) - if isinstance(turn, DegenerateTurnError): - raise turn - return turn - - -async def test_loop_discards_degenerate_turn_and_resamples() -> None: - agent = _DegenerateDictAgent( - [ - DegenerateTurnError("empty shell_call"), - AgentStep(content="recovered", done=True), - ] - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - # The degenerate turn is discarded (consuming a step) and never recorded. - assert run.trace.status == "completed" - assert run.trace.content == "recovered" - assert [step.source for step in run.trace.steps] == ["agent"] - - -async def test_loop_fails_when_degenerate_turn_exhausts_steps() -> None: - agent = _DegenerateDictAgent([DegenerateTurnError("empty shell_call")] * 2) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 2 - await agent._loop(run, RunState()) - - assert run.trace.status == "error" - assert run.trace.error == "empty shell_call" - - -async def test_loop_dispatches_tool_calls_then_finishes() -> None: - agent = DictAgent( - [ - AgentStep(content="", done=False, tool_calls=[MCPToolCall(name="ghost")]), - AgentStep(content="done now", done=True), - ] - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.content == "done now" - assert [step.source for step in run.trace.steps] == ["agent", "tool", "agent"] - # the (unknown) tool call produced an observed tool step in the trajectory - tool_step = run.trace.steps[1] - assert isinstance(tool_step, ToolStep) - assert tool_step.call is not None - assert tool_step.call.name == "ghost" - assert tool_step.result is not None - assert tool_step.result.isError is True # unknown tool → error result - - -async def test_loop_resets_infrastructure_error_count_after_other_result( - monkeypatch: pytest.MonkeyPatch, -) -> None: - turns = [ - AgentStep(content="", done=False, tool_calls=[MCPToolCall(name="bash")]) for _ in range(5) - ] - agent = DictAgent(turns) - infrastructure_error = SSHInfrastructureErrorResult(content=[], isError=True) - ordinary_error = MCPToolResult(content=[], isError=True) - dispatch = AsyncMock( - side_effect=[ - infrastructure_error, - ordinary_error, - infrastructure_error, - infrastructure_error, - infrastructure_error, - ] - ) - monkeypatch.setattr(agent, "_dispatch_call", dispatch) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 10 - await agent._loop(run, RunState()) - - assert dispatch.await_count == 5 - assert run.trace.status == "error" - assert run.trace.stop_reason is None - assert run.trace.error == ("SSH tool failure limit reached after 3 consecutive errors") - - -async def test_loop_max_steps_is_normal_termination() -> None: - # Always returns a tool call → never "done" → hits max_steps. Exhausting the - # configured budget is a stop reason, not an agent error (the platform must - # not paint the rollout or its last tool call as failed). - never_done = [ - AgentStep(content="", done=False, tool_calls=[MCPToolCall(name="ghost")]) for _ in range(5) - ] - agent = DictAgent(never_done) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 2 - await agent._loop(run, RunState()) - - assert run.trace.is_error is False - assert run.trace.status == "completed" - assert run.trace.stop_reason == "max_steps" - assert run.trace.is_truncated is True - # No synthetic error step — the trajectory ends on the real agent/tool steps. - assert all(step.source != "system" for step in run.trace.steps) - - -async def test_loop_marks_length_finish_as_truncated() -> None: - # A final turn cut off at the provider token cap (e.g. mid-tool-call) ends the - # rollout normally but is a truncation, not a natural finish — across every - # provider's finish-reason vocabulary. - for finish_reason in ("length", "max_output_tokens", "max_tokens", "MAX_TOKENS"): - agent = DictAgent([AgentStep(content="partial", done=True, finish_reason=finish_reason)]) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.status == "completed" - assert run.trace.stop_reason == "length" - assert run.trace.is_truncated is True - - -async def test_loop_answers_malformed_call_by_default() -> None: - # Default "retry": the malformed call gets an error result and the loop continues. - agent = DictAgent( - [ - AgentStep( - content="", - done=False, - tool_calls=[MCPToolCall(name="bash", arguments='{"command": "')], - ), - AgentStep(content="recovered", done=True), - ] - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.content == "recovered" - tool_step = run.trace.steps[1] - assert isinstance(tool_step, ToolStep) - assert tool_step.result is not None - assert tool_step.result.isError is True - - -async def test_loop_stops_on_malformed_call_when_configured() -> None: - # The rollout ends at the malformed-call turn with nothing dispatched, and the - # fired condition is the recorded stop reason. - agent = DictAgent( - [ - AgentStep( - content="", - done=False, - tool_calls=[MCPToolCall(name="bash", arguments='{"command": "')], - ) - ], - stop_on={"malformed_tool_call"}, - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.status == "completed" - assert run.trace.stop_reason == "malformed_tool_call" - assert run.trace.is_truncated is True - assert all(not isinstance(step, ToolStep) for step in run.trace.steps) - - -async def test_loop_stops_on_length_when_configured() -> None: - # A token-capped turn ends the rollout even when its tool calls parsed. - agent = DictAgent( - [ - AgentStep( - content="", - done=False, - finish_reason="length", - tool_calls=[MCPToolCall(name="bash", arguments={"command": "ls"})], - ) - ], - stop_on={"length", "malformed_tool_call"}, - ) - run = cast("Run", _FakeRun()) - - agent.config.max_steps = 3 - await agent._loop(run, RunState()) - - assert run.trace.stop_reason == "length" - assert run.trace.is_truncated is True - assert all(not isinstance(step, ToolStep) for step in run.trace.steps) - - -@pytest.mark.parametrize("auto_respond", [False, True]) -@pytest.mark.parametrize("reason", [None, "malformed_tool_call"]) -async def test_provider_error_stops_without_auto_response( - auto_respond: bool, reason: Literal["malformed_tool_call"] | None -) -> None: - agent = DictAgent( - [AgentStep(error="provider failure", stop_reason=reason)], - auto_respond=auto_respond, - max_steps=3, - ) - run = cast("Run", _FakeRun()) - await agent._loop(run, RunState()) - assert run.trace.status == "error" - assert run.trace.stop_reason == reason - assert len(run.trace.steps) == 1 - assert run.trace.steps[0].error == "provider failure" diff --git a/tests/agents/test_trace.py b/tests/agents/test_trace.py deleted file mode 100644 index 43e5246ea..000000000 --- a/tests/agents/test_trace.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Tool-agent family layer tests: the flat ``Step`` subclasses.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any -from unittest.mock import patch - -from mcp import types as mcp_types - -from hud.agents.types import ( - AgentStep, - Citation, - Sample, - SubagentStep, - ToolStep, - Usage, -) -from hud.telemetry.context import set_trace_context -from hud.types import MCPToolCall, MCPToolResult, Step, Trace - - -def test_agent_step_raw_serializes_safely(): - """AgentStep captures raw provider payloads in JSON-safe dumps.""" - - @dataclass - class RawResponse: - raw_data: str - - step = AgentStep(raw=RawResponse(raw_data="value")) - data = step.model_dump(mode="json") - - assert step.raw == RawResponse(raw_data="value") - assert data["raw"] == {"raw_data": "value"} - - -def test_agent_step_dump_uses_canonical_field_names(): - """AgentStep dumps use the normalized SDK field names, flat on the step.""" - step = AgentStep(raw={"raw_data": "value"}) - step.reasoning = "because" - step.citations = [Citation(source="https://example.com")] - - data = step.model_dump(exclude_none=True, mode="json") - - assert data["source"] == "agent" - assert data["reasoning"] == "because" - assert data["citations"] == [{"type": "citation", "text": "", "source": "https://example.com"}] - assert data["raw"] == {"raw_data": "value"} - - -def test_agent_step_citations_roundtrip(): - """Citations survive serialize/deserialize as typed Citations.""" - cit = Citation(type="url_citation", source="https://example.com", title="Example") - step = AgentStep(content="hello", citations=[cit]) - data = step.model_dump(mode="json") - restored = AgentStep(**data) - assert restored.citations == [cit] - assert restored.citations[0].source == "https://example.com" - - -def test_citation_defaults(): - """Citation defaults to a generic, empty-span citation.""" - c = Citation() - assert c.type == "citation" - assert c.text == "" - assert c.start_index is None - - -def test_final_query_reads_the_final_agent_turns_citations(): - """The chat-surface read: trace.final() with family vocabulary at the call site.""" - - def reply_citations(step: Step) -> list[Citation] | None: - return step.citations if isinstance(step, AgentStep) else None - - cited = Citation(source="https://example.com") - trace = Trace() - trace.record(AgentStep(content="draft", citations=[cited])) - trace.record(ToolStep()) - assert trace.final(reply_citations) == [cited] - - trace.record(AgentStep(content="final", done=True)) - assert trace.final(reply_citations) == [] - - -def test_agent_step_timing_and_error_live_on_the_skeleton(): - """The turn uses the skeleton's channels: record() stamps ended_at, error is error.""" - trace = Trace() - step = AgentStep(error="rate limited", done=True) - trace.record(step) - - assert step.ended_at is not None - assert trace.error == "rate limited" - - -def test_trace_dump_keeps_family_payloads(): - """Family subclass fields survive a whole-trace dump (SerializeAsAny).""" - trace = Trace() - trace.record( - AgentStep( - content="answer", - usage=Usage(prompt_tokens=10, completion_tokens=3), - sample=Sample(prompt_token_ids=[1], output_token_ids=[2], output_logprobs=[-0.5]), - ) - ) - trace.record(SubagentStep(subagent=Trace(content="inner"))) - - dump = trace.model_dump(mode="json", exclude_none=True) - assert dump["steps"][0]["content"] == "answer" - assert dump["steps"][0]["usage"]["prompt_tokens"] == 10 - assert dump["steps"][0]["sample"]["output_token_ids"] == [2] - assert dump["steps"][1]["subagent"]["content"] == "inner" - - -def test_step_emit_carries_tool_call_and_result(): - captured: list[dict[str, Any]] = [] - call = MCPToolCall(name="bash", arguments={"command": "ls"}) - result = MCPToolResult( - content=[mcp_types.TextContent(type="text", text="file.txt")], - isError=False, - ) - - with ( - patch("hud.types.queue_span", side_effect=captured.append), - set_trace_context("run-1"), - ): - ToolStep(call=call, result=result).emit() - - (span,) = captured - assert span["name"] == "step.tool" - assert span["attributes"]["hud.schema"] == "hud.step.v1" - payload = span["attributes"]["hud.payload"] - assert payload["call"]["name"] == "bash" - assert payload["result"]["content"][0]["text"] == "file.txt" - assert payload["result"]["isError"] is False From f52a91031c6b745b6a7bec1d451414bd1743e8f3 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 052/248] test(agents): satisfy ty in the agent scenarios Stub helpers pass their behavior through typed keyword arguments, loop rows always answer in arrival order, robot contracts take their features as a typed mapping, and a cut-relay server no longer reuses the name of a list parameter. --- tests/agents/test_cli_agents.py | 6 ++++-- tests/agents/test_loop.py | 15 ++++++--------- tests/agents/test_robot_recording.py | 8 +++++--- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/agents/test_cli_agents.py b/tests/agents/test_cli_agents.py index f48392406..6f6294799 100644 --- a/tests/agents/test_cli_agents.py +++ b/tests/agents/test_cli_agents.py @@ -101,11 +101,13 @@ def result(text: str = "done", *, is_error: bool = False) -> dict[str, Any]: def claude_stub(root: Path, **behavior: Any) -> Stub: - return Stub(root / "stub", "claude", **{"stdout": CLAUDE_STREAM, **behavior}) + behavior.setdefault("stdout", CLAUDE_STREAM) + return Stub(root / "stub", "claude", **behavior) def codex_stub(root: Path, **behavior: Any) -> Stub: - return Stub(root / "stub", "codex", **{"stdout": CODEX_STREAM, **behavior}) + behavior.setdefault("stdout", CODEX_STREAM) + return Stub(root / "stub", "codex", **behavior) def cli_env(root: Path, stub: Stub, **options: Any) -> Environment: diff --git a/tests/agents/test_loop.py b/tests/agents/test_loop.py index b72117b45..138f751ff 100644 --- a/tests/agents/test_loop.py +++ b/tests/agents/test_loop.py @@ -76,7 +76,7 @@ def sequence(*turns: Turn) -> Callable[[ModelRequest], Turn]: @dataclass(frozen=True) class Row: agent: Callable[[], Agent] - script: list[Turn] | Callable[[ModelRequest], Turn] + turns: list[Turn] ending: Any @@ -331,7 +331,7 @@ class Row: ), "responses-empty-shell-call-resampled": Row( responses(), - sequence(shell_call(), say("done")), + [shell_call(), say("done")], snapshot( { "status": "completed", @@ -345,7 +345,7 @@ class Row: ), "responses-empty-shell-call-on-last-step": Row( responses(max_steps=1), - sequence(shell_call()), + [shell_call()], snapshot( { "status": "error", @@ -365,10 +365,7 @@ async def test_a_rollout_ends_for_the_reason_its_last_turn_gives( row: Row, models: Models, hud_env: HudEnv, tmp_path: Path ) -> None: hud_env.set(HUD_API_KEY="k") - if callable(row.script): - models.respond(row.script) - else: - models.script(row.script) + models.respond(sequence(*row.turns)) run = await run_task(workspace_env(tmp_path / "ws"), row.agent()) @@ -570,8 +567,8 @@ async def reach_the_workspace_through_the_relay() -> None: port = await relay.start(address.hostname, address.port) env.add_capability(replace(shell, url=f"ssh://127.0.0.1:{port}")) - async with mcp_server(cutter(relay)) as tools: - env.add_capability(tools) + async with mcp_server(cutter(relay)) as cut: + env.add_capability(cut) run = await run_task(env, chat(max_steps=10)()) assert ending(run, models.requests()) == expected diff --git a/tests/agents/test_robot_recording.py b/tests/agents/test_robot_recording.py index 671c851ce..9b9de532f 100644 --- a/tests/agents/test_robot_recording.py +++ b/tests/agents/test_robot_recording.py @@ -41,7 +41,9 @@ TICKS = 3 -def contract(*, rate: int = 5, robot: str = "arm", **features: dict[str, Any]) -> dict[str, Any]: +def contract( + *, rate: int = 5, robot: str = "arm", features: dict[str, dict[str, Any]] | None = None +) -> dict[str, Any]: return { "control_rate": rate, "robot_type": robot, @@ -264,7 +266,7 @@ async def test_a_failed_rollout_keeps_the_frames_it_recorded(recording: Path) -> async def test_nested_camera_names_stay_distinct_features(recording: Path) -> None: stereo = contract( - **{ + features={ "left/image": {"role": "observation", "type": "rgb"}, "right/image": {"role": "observation", "type": "rgb"}, "state": {"role": "observation", "names": ["tick", "half"]}, @@ -286,7 +288,7 @@ async def test_nested_camera_names_stay_distinct_features(recording: Path) -> No async def test_features_that_flatten_to_one_key_fail_the_rollout(recording: Path) -> None: clash = contract( - **{ + features={ "cam/rgb": {"role": "observation", "type": "rgb"}, "cam_rgb": {"role": "observation", "type": "rgb"}, "state": {"role": "observation", "names": ["tick", "half"]}, From e9e4867cdfc4c1d231fecee07b8944bc6a16108b Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 053/248] test: answer image commands from a store in the fake docker Harbor adaptation builds, pulls and inspects images. FakeDocker.images() adds a small image store between the explicit rules and the defaults: a build stores its tag with the record declared for its build context, a compose pull stores the declared record for its image, and image inspect and version answer from what is stored. Without a store nothing changes. --- tests/harness/docker.py | 159 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 147 insertions(+), 12 deletions(-) diff --git a/tests/harness/docker.py b/tests/harness/docker.py index 30cb38858..2c6a2f3df 100644 --- a/tests/harness/docker.py +++ b/tests/harness/docker.py @@ -4,6 +4,8 @@ appends its argv, and the contents of each file it names with ``-f``/``--file``, to a log; then it answers from the first rule whose regex matches the argv joined by spaces. Tests add rules ahead of the defaults with :meth:`FakeDocker.on`. +:meth:`FakeDocker.images` adds an image store that answers ``version``, +``build``, ``compose build|pull`` and ``image inspect`` between the two. """ from __future__ import annotations @@ -20,7 +22,7 @@ import pytest SCRIPT = """#!{python} -import json, os, pathlib, re, sys, time +import json, os, pathlib, re, runpy, sys, time state = pathlib.Path(os.environ["FAKE_DOCKER_STATE"]) argv = sys.argv[1:] @@ -31,12 +33,95 @@ with (state / "calls.jsonl").open("a") as log: log.write(json.dumps({{"argv": argv, "files": files}}) + "\\n") joined = " ".join(argv) -for rule in json.loads((state / "rules.json").read_text()): - if re.search(rule["match"], joined): - time.sleep(rule["delay"]) - sys.stdout.write(rule["stdout"]) - sys.stderr.write(rule["stderr"]) - sys.exit(rule["exit"]) +rules = json.loads((state / "rules.json").read_text()) +for default in (False, True): + for rule in rules: + if rule.get("default", False) == default and re.search(rule["match"], joined): + time.sleep(rule["delay"]) + sys.stdout.write(rule["stdout"]) + sys.stderr.write(rule["stderr"]) + sys.exit(rule["exit"]) + if not default and (state / "store.py").is_file(): + runpy.run_path(str(state / "store.py"), init_globals={{"state": state, "argv": argv}}) +""" + +# Runs inside the fake ``docker`` with ``state`` and ``argv`` bound; exits when it +# answers, returns to the default rules otherwise. +STORE = """ +import json, pathlib, sys + +store = json.loads((state / "store.json").read_text()) +root = pathlib.Path(store["root"]) + + +def done(stdout="", stderr="", code=0): + sys.stdout.write(stdout) + sys.stderr.write(stderr) + sys.exit(code) + + +def save(): + (state / "store.json").write_text(json.dumps(store)) + + +def built(tag, context): + try: + key = pathlib.Path(context).resolve().relative_to(root).as_posix() + except ValueError: + key = str(context) + record = store["builds"].get(key, {}) + store["images"][tag] = { + "config": record.get("config", {}), + "platform": record.get("platform", store["platform"]), + } + save() + + +def option(name): + return argv[argv.index(name) + 1] if name in argv else None + + +if argv[:1] == ["version"]: + fmt = option("--format") + answers = { + "{{.Server.Version}}": store["version"], + "{{.Server.Os}}/{{.Server.Arch}}": store["platform"], + } + done(answers.get(fmt, store["version"]) + "\\n") +if argv[:2] == ["image", "inspect"]: + image = argv[-1] + record = store["images"].get(image) + if record is None: + done(stderr=f"Error response from daemon: No such image: {image}\\n", code=1) + if option("--format") == "{{.Os}}/{{.Architecture}}": + done(record["platform"] + "\\n") + done(json.dumps([{"Config": record["config"]}])) +if argv[:1] == ["build"]: + built(option("--tag"), argv[-1]) + done() +if argv[:1] == ["compose"] and argv[-2] in ("build", "pull"): + operation, name = argv[-2:] + directory = pathlib.Path(option("--project-directory") or ".") + service = {} + for flag, value in zip(argv, argv[1:]): + if flag in ("-f", "--file"): + document = json.loads(pathlib.Path(value).read_text()) + service.update(document.get("services", {}).get(name, {})) + image = service.get("image") + if operation == "build": + build = service.get("build", {}) + context = build if isinstance(build, str) else build.get("context", ".") + built(image, directory / context) + done() + if image in store["pulls"]: + record = store["pulls"][image] + store["images"][image] = { + "config": record.get("config", {}), + "platform": record.get("platform", store["platform"]), + } + save() + done() + done(stderr=f"Error response from daemon: pull access denied for {image}\\n", code=1) """ DF_OUTPUT = ( @@ -72,10 +157,10 @@ def __init__(self, root: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}") monkeypatch.setenv("FAKE_DOCKER_STATE", str(self._state)) self._rules: list[dict[str, Any]] = [] - self.on(r".", stdout="") - self.on(r"^volume create ", stdout="volume\n") - self.on(r"^exec .* df -Pk", stdout=DF_OUTPUT) - self.on(r"^run ", stdout="cid-1\n") + self._rule(r".", default=True) + self._rule(r"^volume create ", stdout="volume\n", default=True) + self._rule(r"^exec .* df -Pk", stdout=DF_OUTPUT, default=True) + self._rule(r"^run ", stdout="cid-1\n", default=True) def on( self, @@ -87,8 +172,58 @@ def on( delay: float = 0.0, ) -> None: """Answer invocations whose argv matches the regex ``match``; newest rules win.""" + self._rule(match, stdout=stdout, stderr=stderr, exit=exit, delay=delay, default=False) + + def images( + self, + root: Path, + *, + builds: dict[str, dict[str, Any]] | None = None, + pulls: dict[str, dict[str, Any]] | None = None, + platform: str = "linux/amd64", + version: str = "28.3.3", + ) -> None: + """Answer image commands from a store, after the rules added with :meth:`on`. + + A ``build`` (``docker build`` or ``docker compose build``) stores its tag + with the record in ``builds`` keyed by the build context's path relative + to ``root``; a ``compose pull`` stores the record in ``pulls`` for its + image and fails for any other. A record is ``{"config": , + "platform": "os/arch"}``; both keys are optional, and the platform + defaults to the server's. ``image inspect`` answers from the stored + images and ``version`` reports ``version`` and ``platform``. + """ + store = { + "root": str(root.resolve()), + "builds": builds or {}, + "pulls": pulls or {}, + "images": {}, + "platform": platform, + "version": version, + } + (self._state / "store.json").write_text(json.dumps(store)) + (self._state / "store.py").write_text(STORE) + + def _rule( + self, + match: str, + *, + stdout: str = "", + stderr: str = "", + exit: int = 0, + delay: float = 0.0, + default: bool, + ) -> None: self._rules.insert( - 0, {"match": match, "stdout": stdout, "stderr": stderr, "exit": exit, "delay": delay} + 0, + { + "match": match, + "stdout": stdout, + "stderr": stderr, + "exit": exit, + "delay": delay, + "default": default, + }, ) (self._state / "rules.json").write_text(json.dumps(self._rules)) From c2579beeb3098d711a3c7ccd615e03c69a10e0c9 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 054/248] test: adapt Harbor case directories against the fake docker Each directory under cases/ is a Harbor dataset with a docker.json saying what the daemon holds. The scenario adapts a copy and compares the failures, the generated tree, every compose.yaml and tasks.json, and the docker command log with the case's expected.json, and checks that every compose project resolves inside itself and that no host template value reaches the written files. Tables cover the no-task and no-daemon errors and which edits change the environment image identity. This replaces test_contract.py, whose autouse fake replaced resolve_images and require_docker and whose controller config.json assertions pinned an unexported schema; those contracts move to the controller's Docker fixture suite. --- .../containerfile/environment/Containerfile | 1 + .../environment/docker-compose.yaml | 7 + .../containerfile/instruction.md | 1 + .../containerfile/task.toml | 0 .../containerfile/tests/test.sh | 2 + .../cases/compose-main-build/docker.json | 1 + .../cases/compose-main-build/expected.json | 252 +++ .../environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 3 + .../image-and-dockerfile/instruction.md | 1 + .../image-and-dockerfile/task.toml | 0 .../image-and-dockerfile/tests/test.sh | 2 + .../cases/compose-variables/docker.json | 5 + .../cases/compose-variables/expected.json | 171 ++ .../task-a/environment/Dockerfile | 2 + .../task-a/environment/docker-compose.yaml | 16 + .../compose-variables/task-a/instruction.md | 1 + .../cases/compose-variables/task-a/task.toml | 7 + .../compose-variables/task-a/tests/test.sh | 2 + .../build-pmars/environment/Dockerfile | 2 + .../build-pmars/instruction.md | 1 + .../dataset-grouping/build-pmars/task.toml | 18 + .../build-pmars/tests/test.sh | 2 + .../caffe-cifar-10/environment/Dockerfile | 2 + .../caffe-cifar-10/instruction.md | 1 + .../dataset-grouping/caffe-cifar-10/task.toml | 2 + .../caffe-cifar-10/tests/test.sh | 2 + .../cancel-async-tasks/environment/Dockerfile | 2 + .../cancel-async-tasks/instruction.md | 1 + .../cancel-async-tasks/task.toml | 5 + .../cancel-async-tasks/tests/test.sh | 2 + .../chess-best-move/environment/Dockerfile | 2 + .../chess-best-move/instruction.md | 1 + .../chess-best-move/task.toml | 2 + .../chess-best-move/tests/test.sh | 2 + .../harbor/cases/dataset-grouping/docker.json | 1 + .../cases/dataset-grouping/expected.json | 447 +++++ .../sam-cell-seg/environment/Dockerfile | 2 + .../sam-cell-seg/instruction.md | 1 + .../dataset-grouping/sam-cell-seg/task.toml | 2 + .../sam-cell-seg/tests/test.sh | 2 + .../harbor/cases/env-templates/docker.json | 1 + .../harbor/cases/env-templates/expected.json | 126 ++ .../templated/environment/Dockerfile | 2 + .../env-templates/templated/instruction.md | 1 + .../cases/env-templates/templated/task.toml | 6 + .../env-templates/templated/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../instruction.md | 1 + .../artifact-absolute-destination/task.toml | 1 + .../tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../instruction.md | 1 + .../artifact-backslash-destination/task.toml | 1 + .../tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../instruction.md | 1 + .../artifact-dotdot-destination/task.toml | 1 + .../artifact-dotdot-destination/tests/test.sh | 2 + .../artifact-dotdot/environment/Dockerfile | 2 + .../findings/artifact-dotdot/instruction.md | 1 + .../cases/findings/artifact-dotdot/task.toml | 1 + .../findings/artifact-dotdot/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../artifact-double-slash/instruction.md | 1 + .../findings/artifact-double-slash/task.toml | 1 + .../artifact-double-slash/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../instruction.md | 1 + .../artifact-manifest-destination/task.toml | 1 + .../tests/test.sh | 2 + .../artifact-root/environment/Dockerfile | 2 + .../findings/artifact-root/instruction.md | 1 + .../cases/findings/artifact-root/task.toml | 1 + .../findings/artifact-root/tests/test.sh | 2 + .../bad-config/environment/Dockerfile | 2 + .../cases/findings/bad-config/instruction.md | 1 + .../cases/findings/bad-config/task.toml | 3 + .../cases/findings/bad-config/tests/test.sh | 2 + .../findings/bad-toml/environment/Dockerfile | 2 + .../cases/findings/bad-toml/instruction.md | 1 + .../harbor/cases/findings/bad-toml/task.toml | 1 + .../cases/findings/bad-toml/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 1 + .../compose-not-mapping/instruction.md | 1 + .../findings/compose-not-mapping/task.toml | 0 .../compose-not-mapping/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 3 + .../compose-path-escape/instruction.md | 1 + .../findings/compose-path-escape/task.toml | 0 .../compose-path-escape/tests/test.sh | 2 + .../harbor/cases/findings/docker.json | 1 + .../harbor/cases/findings/expected.json | 331 ++++ .../environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 5 + .../healthcheck-duration/instruction.md | 1 + .../findings/healthcheck-duration/task.toml | 0 .../healthcheck-duration/tests/test.sh | 2 + .../healthcheck-test/environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 4 + .../findings/healthcheck-test/instruction.md | 1 + .../cases/findings/healthcheck-test/task.toml | 0 .../findings/healthcheck-test/tests/test.sh | 2 + .../host-variable/environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 3 + .../findings/host-variable/instruction.md | 1 + .../cases/findings/host-variable/task.toml | 0 .../findings/host-variable/tests/test.sh | 2 + .../main-build-escape/environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 3 + .../findings/main-build-escape/instruction.md | 1 + .../findings/main-build-escape/task.toml | 0 .../findings/main-build-escape/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 5 + .../main-build-path-type/instruction.md | 1 + .../findings/main-build-path-type/task.toml | 0 .../main-build-path-type/tests/test.sh | 2 + .../mcp-servers/environment/Dockerfile | 2 + .../cases/findings/mcp-servers/instruction.md | 1 + .../cases/findings/mcp-servers/task.toml | 18 + .../cases/findings/mcp-servers/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../findings/missing-instruction/task.toml | 0 .../missing-instruction/tests/test.sh | 2 + .../environment/docker-compose.yaml | 5 + .../missing-main-dockerfile/instruction.md | 1 + .../missing-main-dockerfile/task.toml | 0 .../missing-main-dockerfile/tests/test.sh | 2 + .../missing-tests/environment/Dockerfile | 2 + .../findings/missing-tests/instruction.md | 1 + .../cases/findings/missing-tests/task.toml | 0 .../environment/Dockerfile | 2 + .../instruction.md | 1 + .../missing-verifier-dockerfile/task.toml | 2 + .../missing-verifier-dockerfile/tests/test.sh | 2 + .../multi-step/environment/Dockerfile | 2 + .../multi-step/steps/plan/instruction.md | 1 + .../multi-step/steps/plan/tests/test.sh | 2 + .../cases/findings/multi-step/task.toml | 2 + .../cases/findings/no-recipe/instruction.md | 1 + .../harbor/cases/findings/no-recipe/task.toml | 0 .../cases/findings/no-recipe/tests/test.sh | 2 + .../reserved-ports/environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 3 + .../findings/reserved-ports/instruction.md | 1 + .../cases/findings/reserved-ports/task.toml | 0 .../findings/reserved-ports/tests/test.sh | 2 + .../reserved-services/environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 6 + .../findings/reserved-services/instruction.md | 1 + .../findings/reserved-services/task.toml | 0 .../findings/reserved-services/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 3 + .../sidecar-without-recipe/instruction.md | 1 + .../findings/sidecar-without-recipe/task.toml | 0 .../sidecar-without-recipe/tests/test.sh | 2 + .../findings/supported/environment/Dockerfile | 2 + .../cases/findings/supported/instruction.md | 1 + .../harbor/cases/findings/supported/task.toml | 0 .../cases/findings/supported/tests/test.sh | 2 + .../broken/environment/Dockerfile | 1 + .../image-failures/broken/instruction.md | 1 + .../cases/image-failures/broken/task.toml | 0 .../cases/image-failures/broken/tests/test.sh | 2 + .../harbor/cases/image-failures/docker.json | 18 + .../harbor/cases/image-failures/expected.json | 282 +++ .../environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 3 + .../missing-sidecar-image/instruction.md | 1 + .../missing-sidecar-image/task.toml | 0 .../missing-sidecar-image/tests/test.sh | 2 + .../verifier-on-arm/environment/Dockerfile | 2 + .../verifier-on-arm/instruction.md | 1 + .../image-failures/verifier-on-arm/task.toml | 2 + .../verifier-on-arm/tests/Dockerfile | 2 + .../verifier-on-arm/tests/test.sh | 2 + .../working/environment/Dockerfile | 2 + .../image-failures/working/instruction.md | 1 + .../cases/image-failures/working/task.toml | 0 .../image-failures/working/tests/test.sh | 2 + .../harbor/cases/plain-image/docker.json | 9 + .../harbor/cases/plain-image/expected.json | 132 ++ .../task-a/environment/.dockerignore | 2 + .../plain-image/task-a/environment/Dockerfile | 2 + .../task-a/environment/docker-compose.yml | 1 + .../task-a/environment/ignored.txt | 1 + .../cases/plain-image/task-a/instruction.md | 1 + .../harbor/cases/plain-image/task-a/task.toml | 5 + .../cases/plain-image/task-a/tests/test.sh | 2 + .../harbor/cases/prebuilt-image/docker.json | 9 + .../harbor/cases/prebuilt-image/expected.json | 107 ++ .../prebuilt-image/task-a/instruction.md | 1 + .../cases/prebuilt-image/task-a/task.toml | 2 + .../cases/prebuilt-image/task-a/tests/test.sh | 2 + .../budget/environment/Dockerfile | 2 + .../budget/instruction.md | 1 + .../resources-and-budgets/budget/task.toml | 5 + .../budget/tests/test.sh | 2 + .../cpu-only/environment/Dockerfile | 2 + .../cpu-only/instruction.md | 1 + .../resources-and-budgets/cpu-only/task.toml | 2 + .../cpu-only/tests/test.sh | 2 + .../cases/resources-and-budgets/docker.json | 1 + .../cases/resources-and-budgets/expected.json | 495 +++++ .../gpu/environment/Dockerfile | 2 + .../resources-and-budgets/gpu/instruction.md | 1 + .../cases/resources-and-budgets/gpu/task.toml | 6 + .../resources-and-budgets/gpu/tests/test.sh | 2 + .../windows-tpu/environment/Dockerfile | 2 + .../windows-tpu/instruction.md | 1 + .../windows-tpu/task.toml | 3 + .../windows-tpu/tests/test.sh | 2 + .../cases/separate-verifier/docker.json | 10 + .../cases/separate-verifier/expected.json | 550 ++++++ .../placement/environment/Dockerfile | 2 + .../placement/environment/docker-compose.yaml | 5 + .../placement/instruction.md | 1 + .../separate-verifier/placement/task.toml | 22 + .../placement/tests/Dockerfile | 2 + .../separate-verifier/placement/tests/test.sh | 2 + .../plain/environment/Dockerfile | 2 + .../plain/environment/docker-compose.yaml | 5 + .../separate-verifier/plain/instruction.md | 1 + .../cases/separate-verifier/plain/task.toml | 2 + .../separate-verifier/plain/tests/Dockerfile | 2 + .../separate-verifier/plain/tests/test.sh | 2 + .../same-placement/environment/Dockerfile | 2 + .../same-placement/instruction.md | 1 + .../same-placement/task.toml | 10 + .../same-placement/tests/Dockerfile | 2 + .../same-placement/tests/test.sh | 2 + .../cases/shared-collect-sidecar/docker.json | 5 + .../shared-collect-sidecar/expected.json | 156 ++ .../task-a/environment/Dockerfile | 2 + .../task-a/environment/docker-compose.yaml | 5 + .../task-a/instruction.md | 1 + .../shared-collect-sidecar/task-a/task.toml | 5 + .../task-a/tests/test.sh | 2 + .../harbor/cases/sidecars/docker.json | 22 + .../harbor/cases/sidecars/expected.json | 556 ++++++ .../implicit-main/environment/Dockerfile | 2 + .../environment/cache/Dockerfile | 1 + .../environment/docker-compose.yaml | 21 + .../implicit-main/environment/seed/Dockerfile | 2 + .../sidecars/implicit-main/instruction.md | 1 + .../cases/sidecars/implicit-main/task.toml | 0 .../sidecars/implicit-main/tests/test.sh | 2 + .../main-process/environment/Dockerfile | 2 + .../environment/docker-compose.yaml | 13 + .../sidecars/main-process/instruction.md | 1 + .../cases/sidecars/main-process/task.toml | 0 .../cases/sidecars/main-process/tests/test.sh | 2 + .../source-builds-and-volumes/docker.json | 1 + .../source-builds-and-volumes/expected.json | 187 ++ .../task-a/environment/Dockerfile | 2 + .../task-a/environment/database/Dockerfile | 1 + .../task-a/environment/database/data/.keep | 0 .../task-a/environment/database/db.env | 1 + .../task-a/environment/docker-compose.yaml | 18 + .../task-a/environment/main-data/.keep | 0 .../task-a/environment/main.env | 1 + .../task-a/environment/readonly/.keep | 0 .../task-a/instruction.md | 1 + .../task-a/task.toml | 0 .../task-a/tests/test.sh | 2 + .../harbor/cases/symlinks/docker.json | 1 + .../harbor/cases/symlinks/expected.json | 128 ++ .../symlinks/task-a/environment/Dockerfile | 2 + .../task-a/environment/Dockerfile.link | 1 + .../cases/symlinks/task-a/instruction.md | 1 + .../harbor/cases/symlinks/task-a/task.toml | 0 .../cases/symlinks/task-a/tests/host-file | 1 + .../cases/symlinks/task-a/tests/test.sh | 2 + tests/integrations/harbor/test_adapt.py | 281 +++ tests/integrations/harbor/test_contract.py | 1684 ----------------- 279 files changed, 4841 insertions(+), 1684 deletions(-) create mode 100644 tests/integrations/harbor/cases/compose-main-build/containerfile/environment/Containerfile create mode 100644 tests/integrations/harbor/cases/compose-main-build/containerfile/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/compose-main-build/containerfile/instruction.md create mode 100644 tests/integrations/harbor/cases/compose-main-build/containerfile/task.toml create mode 100644 tests/integrations/harbor/cases/compose-main-build/containerfile/tests/test.sh create mode 100644 tests/integrations/harbor/cases/compose-main-build/docker.json create mode 100644 tests/integrations/harbor/cases/compose-main-build/expected.json create mode 100644 tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/instruction.md create mode 100644 tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/task.toml create mode 100644 tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/tests/test.sh create mode 100644 tests/integrations/harbor/cases/compose-variables/docker.json create mode 100644 tests/integrations/harbor/cases/compose-variables/expected.json create mode 100644 tests/integrations/harbor/cases/compose-variables/task-a/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/compose-variables/task-a/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/compose-variables/task-a/instruction.md create mode 100644 tests/integrations/harbor/cases/compose-variables/task-a/task.toml create mode 100644 tests/integrations/harbor/cases/compose-variables/task-a/tests/test.sh create mode 100644 tests/integrations/harbor/cases/dataset-grouping/build-pmars/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/dataset-grouping/build-pmars/instruction.md create mode 100644 tests/integrations/harbor/cases/dataset-grouping/build-pmars/task.toml create mode 100644 tests/integrations/harbor/cases/dataset-grouping/build-pmars/tests/test.sh create mode 100644 tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/instruction.md create mode 100644 tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/task.toml create mode 100644 tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/tests/test.sh create mode 100644 tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/instruction.md create mode 100644 tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/task.toml create mode 100644 tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/tests/test.sh create mode 100644 tests/integrations/harbor/cases/dataset-grouping/chess-best-move/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/dataset-grouping/chess-best-move/instruction.md create mode 100644 tests/integrations/harbor/cases/dataset-grouping/chess-best-move/task.toml create mode 100644 tests/integrations/harbor/cases/dataset-grouping/chess-best-move/tests/test.sh create mode 100644 tests/integrations/harbor/cases/dataset-grouping/docker.json create mode 100644 tests/integrations/harbor/cases/dataset-grouping/expected.json create mode 100644 tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/instruction.md create mode 100644 tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/task.toml create mode 100644 tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/tests/test.sh create mode 100644 tests/integrations/harbor/cases/env-templates/docker.json create mode 100644 tests/integrations/harbor/cases/env-templates/expected.json create mode 100644 tests/integrations/harbor/cases/env-templates/templated/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/env-templates/templated/instruction.md create mode 100644 tests/integrations/harbor/cases/env-templates/templated/task.toml create mode 100644 tests/integrations/harbor/cases/env-templates/templated/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/artifact-absolute-destination/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/artifact-absolute-destination/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/artifact-absolute-destination/task.toml create mode 100644 tests/integrations/harbor/cases/findings/artifact-absolute-destination/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/artifact-backslash-destination/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/artifact-backslash-destination/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/artifact-backslash-destination/task.toml create mode 100644 tests/integrations/harbor/cases/findings/artifact-backslash-destination/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/artifact-dotdot-destination/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/artifact-dotdot-destination/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/artifact-dotdot-destination/task.toml create mode 100644 tests/integrations/harbor/cases/findings/artifact-dotdot-destination/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/artifact-dotdot/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/artifact-dotdot/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/artifact-dotdot/task.toml create mode 100644 tests/integrations/harbor/cases/findings/artifact-dotdot/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/artifact-double-slash/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/artifact-double-slash/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/artifact-double-slash/task.toml create mode 100644 tests/integrations/harbor/cases/findings/artifact-double-slash/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/artifact-manifest-destination/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/artifact-manifest-destination/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/artifact-manifest-destination/task.toml create mode 100644 tests/integrations/harbor/cases/findings/artifact-manifest-destination/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/artifact-root/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/artifact-root/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/artifact-root/task.toml create mode 100644 tests/integrations/harbor/cases/findings/artifact-root/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/bad-config/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/bad-config/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/bad-config/task.toml create mode 100644 tests/integrations/harbor/cases/findings/bad-config/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/bad-toml/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/bad-toml/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/bad-toml/task.toml create mode 100644 tests/integrations/harbor/cases/findings/bad-toml/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/compose-not-mapping/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/compose-not-mapping/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/compose-not-mapping/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/compose-not-mapping/task.toml create mode 100644 tests/integrations/harbor/cases/findings/compose-not-mapping/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/compose-path-escape/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/compose-path-escape/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/compose-path-escape/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/compose-path-escape/task.toml create mode 100644 tests/integrations/harbor/cases/findings/compose-path-escape/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/docker.json create mode 100644 tests/integrations/harbor/cases/findings/expected.json create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-duration/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-duration/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-duration/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-duration/task.toml create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-duration/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-test/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-test/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-test/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-test/task.toml create mode 100644 tests/integrations/harbor/cases/findings/healthcheck-test/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/host-variable/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/host-variable/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/host-variable/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/host-variable/task.toml create mode 100644 tests/integrations/harbor/cases/findings/host-variable/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/main-build-escape/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/main-build-escape/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/main-build-escape/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/main-build-escape/task.toml create mode 100644 tests/integrations/harbor/cases/findings/main-build-escape/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/main-build-path-type/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/main-build-path-type/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/main-build-path-type/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/main-build-path-type/task.toml create mode 100644 tests/integrations/harbor/cases/findings/main-build-path-type/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/mcp-servers/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/mcp-servers/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/mcp-servers/task.toml create mode 100644 tests/integrations/harbor/cases/findings/mcp-servers/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/missing-instruction/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/missing-instruction/task.toml create mode 100644 tests/integrations/harbor/cases/findings/missing-instruction/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/missing-main-dockerfile/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/missing-main-dockerfile/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/missing-main-dockerfile/task.toml create mode 100644 tests/integrations/harbor/cases/findings/missing-main-dockerfile/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/missing-tests/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/missing-tests/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/missing-tests/task.toml create mode 100644 tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/task.toml create mode 100644 tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/multi-step/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/multi-step/steps/plan/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/multi-step/steps/plan/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/multi-step/task.toml create mode 100644 tests/integrations/harbor/cases/findings/no-recipe/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/no-recipe/task.toml create mode 100644 tests/integrations/harbor/cases/findings/no-recipe/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/reserved-ports/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/reserved-ports/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/reserved-ports/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/reserved-ports/task.toml create mode 100644 tests/integrations/harbor/cases/findings/reserved-ports/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/reserved-services/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/reserved-services/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/reserved-services/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/reserved-services/task.toml create mode 100644 tests/integrations/harbor/cases/findings/reserved-services/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/sidecar-without-recipe/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/sidecar-without-recipe/task.toml create mode 100644 tests/integrations/harbor/cases/findings/sidecar-without-recipe/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/supported/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/supported/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/supported/task.toml create mode 100644 tests/integrations/harbor/cases/findings/supported/tests/test.sh create mode 100644 tests/integrations/harbor/cases/image-failures/broken/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/image-failures/broken/instruction.md create mode 100644 tests/integrations/harbor/cases/image-failures/broken/task.toml create mode 100644 tests/integrations/harbor/cases/image-failures/broken/tests/test.sh create mode 100644 tests/integrations/harbor/cases/image-failures/docker.json create mode 100644 tests/integrations/harbor/cases/image-failures/expected.json create mode 100644 tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/image-failures/missing-sidecar-image/instruction.md create mode 100644 tests/integrations/harbor/cases/image-failures/missing-sidecar-image/task.toml create mode 100644 tests/integrations/harbor/cases/image-failures/missing-sidecar-image/tests/test.sh create mode 100644 tests/integrations/harbor/cases/image-failures/verifier-on-arm/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/image-failures/verifier-on-arm/instruction.md create mode 100644 tests/integrations/harbor/cases/image-failures/verifier-on-arm/task.toml create mode 100644 tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/Dockerfile create mode 100644 tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/test.sh create mode 100644 tests/integrations/harbor/cases/image-failures/working/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/image-failures/working/instruction.md create mode 100644 tests/integrations/harbor/cases/image-failures/working/task.toml create mode 100644 tests/integrations/harbor/cases/image-failures/working/tests/test.sh create mode 100644 tests/integrations/harbor/cases/plain-image/docker.json create mode 100644 tests/integrations/harbor/cases/plain-image/expected.json create mode 100644 tests/integrations/harbor/cases/plain-image/task-a/environment/.dockerignore create mode 100644 tests/integrations/harbor/cases/plain-image/task-a/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/plain-image/task-a/environment/docker-compose.yml create mode 100644 tests/integrations/harbor/cases/plain-image/task-a/environment/ignored.txt create mode 100644 tests/integrations/harbor/cases/plain-image/task-a/instruction.md create mode 100644 tests/integrations/harbor/cases/plain-image/task-a/task.toml create mode 100644 tests/integrations/harbor/cases/plain-image/task-a/tests/test.sh create mode 100644 tests/integrations/harbor/cases/prebuilt-image/docker.json create mode 100644 tests/integrations/harbor/cases/prebuilt-image/expected.json create mode 100644 tests/integrations/harbor/cases/prebuilt-image/task-a/instruction.md create mode 100644 tests/integrations/harbor/cases/prebuilt-image/task-a/task.toml create mode 100644 tests/integrations/harbor/cases/prebuilt-image/task-a/tests/test.sh create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/budget/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/budget/instruction.md create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/budget/task.toml create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/budget/tests/test.sh create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/cpu-only/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/cpu-only/instruction.md create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/cpu-only/task.toml create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/cpu-only/tests/test.sh create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/docker.json create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/expected.json create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/gpu/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/gpu/instruction.md create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/gpu/task.toml create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/gpu/tests/test.sh create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/instruction.md create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/task.toml create mode 100644 tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/tests/test.sh create mode 100644 tests/integrations/harbor/cases/separate-verifier/docker.json create mode 100644 tests/integrations/harbor/cases/separate-verifier/expected.json create mode 100644 tests/integrations/harbor/cases/separate-verifier/placement/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/separate-verifier/placement/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/separate-verifier/placement/instruction.md create mode 100644 tests/integrations/harbor/cases/separate-verifier/placement/task.toml create mode 100644 tests/integrations/harbor/cases/separate-verifier/placement/tests/Dockerfile create mode 100644 tests/integrations/harbor/cases/separate-verifier/placement/tests/test.sh create mode 100644 tests/integrations/harbor/cases/separate-verifier/plain/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/separate-verifier/plain/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/separate-verifier/plain/instruction.md create mode 100644 tests/integrations/harbor/cases/separate-verifier/plain/task.toml create mode 100644 tests/integrations/harbor/cases/separate-verifier/plain/tests/Dockerfile create mode 100644 tests/integrations/harbor/cases/separate-verifier/plain/tests/test.sh create mode 100644 tests/integrations/harbor/cases/separate-verifier/same-placement/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/separate-verifier/same-placement/instruction.md create mode 100644 tests/integrations/harbor/cases/separate-verifier/same-placement/task.toml create mode 100644 tests/integrations/harbor/cases/separate-verifier/same-placement/tests/Dockerfile create mode 100644 tests/integrations/harbor/cases/separate-verifier/same-placement/tests/test.sh create mode 100644 tests/integrations/harbor/cases/shared-collect-sidecar/docker.json create mode 100644 tests/integrations/harbor/cases/shared-collect-sidecar/expected.json create mode 100644 tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/shared-collect-sidecar/task-a/instruction.md create mode 100644 tests/integrations/harbor/cases/shared-collect-sidecar/task-a/task.toml create mode 100644 tests/integrations/harbor/cases/shared-collect-sidecar/task-a/tests/test.sh create mode 100644 tests/integrations/harbor/cases/sidecars/docker.json create mode 100644 tests/integrations/harbor/cases/sidecars/expected.json create mode 100644 tests/integrations/harbor/cases/sidecars/implicit-main/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/sidecars/implicit-main/environment/cache/Dockerfile create mode 100644 tests/integrations/harbor/cases/sidecars/implicit-main/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/sidecars/implicit-main/environment/seed/Dockerfile create mode 100644 tests/integrations/harbor/cases/sidecars/implicit-main/instruction.md create mode 100644 tests/integrations/harbor/cases/sidecars/implicit-main/task.toml create mode 100644 tests/integrations/harbor/cases/sidecars/implicit-main/tests/test.sh create mode 100644 tests/integrations/harbor/cases/sidecars/main-process/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/sidecars/main-process/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/sidecars/main-process/instruction.md create mode 100644 tests/integrations/harbor/cases/sidecars/main-process/task.toml create mode 100644 tests/integrations/harbor/cases/sidecars/main-process/tests/test.sh create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/docker.json create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/expected.json create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/Dockerfile create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/data/.keep create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/db.env create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main-data/.keep create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main.env create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/readonly/.keep create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/instruction.md create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/task.toml create mode 100644 tests/integrations/harbor/cases/source-builds-and-volumes/task-a/tests/test.sh create mode 100644 tests/integrations/harbor/cases/symlinks/docker.json create mode 100644 tests/integrations/harbor/cases/symlinks/expected.json create mode 100644 tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile create mode 120000 tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile.link create mode 100644 tests/integrations/harbor/cases/symlinks/task-a/instruction.md create mode 100644 tests/integrations/harbor/cases/symlinks/task-a/task.toml create mode 120000 tests/integrations/harbor/cases/symlinks/task-a/tests/host-file create mode 100644 tests/integrations/harbor/cases/symlinks/task-a/tests/test.sh create mode 100644 tests/integrations/harbor/test_adapt.py delete mode 100644 tests/integrations/harbor/test_contract.py diff --git a/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/Containerfile b/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/Containerfile new file mode 100644 index 000000000..0a93e44d3 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/Containerfile @@ -0,0 +1 @@ +FROM python:3.12 diff --git a/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/docker-compose.yaml b/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/docker-compose.yaml new file mode 100644 index 000000000..f5063eeed --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/containerfile/environment/docker-compose.yaml @@ -0,0 +1,7 @@ +services: + main: + build: + context: . + dockerfile: Containerfile + args: + FLAVOR: compose diff --git a/tests/integrations/harbor/cases/compose-main-build/containerfile/instruction.md b/tests/integrations/harbor/cases/compose-main-build/containerfile/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/containerfile/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/compose-main-build/containerfile/task.toml b/tests/integrations/harbor/cases/compose-main-build/containerfile/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/compose-main-build/containerfile/tests/test.sh b/tests/integrations/harbor/cases/compose-main-build/containerfile/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/containerfile/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/compose-main-build/docker.json b/tests/integrations/harbor/cases/compose-main-build/docker.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/docker.json @@ -0,0 +1 @@ +{} diff --git a/tests/integrations/harbor/cases/compose-main-build/expected.json b/tests/integrations/harbor/cases/compose-main-build/expected.json new file mode 100644 index 000000000..23928ec96 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/expected.json @@ -0,0 +1,252 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /containerfile/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "build": { + "args": { + "FLAVOR": "compose" + }, + "context": ".", + "dockerfile": "Containerfile" + }, + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /image-and-dockerfile/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "image": "authored/main:latest", + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [], + "projects": { + "compose-main-build-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:compose-main-build-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "image-and-dockerfile", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "compose-main-build-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "image-and-dockerfile" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/environment/docker-compose.yaml", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/image-and-dockerfile/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + }, + "compose-main-build-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "args": { + "FLAVOR": "compose" + }, + "context": "./compose-project/environment", + "dockerfile": "Containerfile" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:compose-main-build-

-
", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "containerfile", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "compose-main-build-
", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "containerfile" + } + ], + "tree": [ + "compose-project/environment/Containerfile", + "compose-project/environment/docker-compose.yaml", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/containerfile/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/Dockerfile b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/docker-compose.yaml b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/docker-compose.yaml new file mode 100644 index 000000000..8a3d24f45 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + image: authored/main:latest diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/instruction.md b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/task.toml b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/tests/test.sh b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-main-build/image-and-dockerfile/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/compose-variables/docker.json b/tests/integrations/harbor/cases/compose-variables/docker.json new file mode 100644 index 000000000..f3f75b0e3 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/docker.json @@ -0,0 +1,5 @@ +{ + "pulls": { + "redis:7-alpine": {} + } +} diff --git a/tests/integrations/harbor/cases/compose-variables/expected.json b/tests/integrations/harbor/cases/compose-variables/expected.json new file mode 100644 index 000000000..48008d244 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/expected.json @@ -0,0 +1,171 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /task-a/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "build": { + "context": "." + }, + "deploy": { + "resources": { + "limits": { + "cpus": "2", + "memory": "4096M" + } + } + }, + "environment": { + "FROM_DOTENV": "dotenv", + "FROM_ENV": "from-task" + }, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "volumes": [] + }, + "redis": { + "environment": {}, + "expose": [ + "6379" + ], + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [], + "projects": { + "compose-variables-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "deploy": { + "resources": { + "limits": { + "cpus": "2", + "memory": "4096M" + } + } + }, + "entrypoint": [], + "environment": { + "FROM_DOTENV": "dotenv", + "FROM_ENV": "from-task" + }, + "expose": [], + "image": "hud-harbor:compose-variables-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + }, + "redis": { + "environment": {}, + "expose": [ + "6379" + ], + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "task-a", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "compose-variables-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": { + "cpu": 2.0, + "memory_mb": 4096 + } + }, + "slug": "task-a" + } + ], + "tree": [ + "compose-project/environment/.env", + "compose-project/environment/Dockerfile", + "compose-project/environment/docker-compose.yaml", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/task-a/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/compose-variables/task-a/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/environment/docker-compose.yaml b/tests/integrations/harbor/cases/compose-variables/task-a/environment/docker-compose.yaml new file mode 100644 index 000000000..1e566f3c4 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/environment/docker-compose.yaml @@ -0,0 +1,16 @@ +services: + main: + build: + context: ${CONTEXT_DIR} + image: ${MAIN_IMAGE_NAME} + environment: + FROM_ENV: ${LITERAL} + FROM_DOTENV: ${DOTENV_ONLY} + deploy: + resources: + limits: + cpus: ${CPUS} + memory: ${MEMORY} + redis: + image: redis:${SIDECAR_TAG} + expose: [6379] diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/instruction.md b/tests/integrations/harbor/cases/compose-variables/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/task.toml b/tests/integrations/harbor/cases/compose-variables/task-a/task.toml new file mode 100644 index 000000000..8c3a7cda0 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/task.toml @@ -0,0 +1,7 @@ +[environment] +cpus = 2 +memory_mb = 4096 + +[environment.env] +LITERAL = "from-task" +TEMPLATED = "${HARBOR_JUDGE_KEY}" diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/tests/test.sh b/tests/integrations/harbor/cases/compose-variables/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/build-pmars/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/build-pmars/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/build-pmars/task.toml b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/task.toml new file mode 100644 index 000000000..8eb24600d --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/task.toml @@ -0,0 +1,18 @@ +artifacts = [ + "/tmp/result", + { source = "/app/outputs", destination = "results/outputs", exclude = ["*.tmp", "cache"] }, +] + +[task] +description = "Build pMARS." +keywords = ["c", "build"] + +[metadata] +category = "systems" +difficulty = "medium" + +[agent] +timeout_sec = 45 + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/cases/dataset-grouping/build-pmars/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/build-pmars/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/environment/Dockerfile new file mode 100644 index 000000000..8b58f79a3 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM nvidia/cuda:12.0-runtime-ubuntu22.04 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/task.toml b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/task.toml new file mode 100644 index 000000000..c900fd66a --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/task.toml @@ -0,0 +1,2 @@ +[metadata] +category = "machine-learning" diff --git a/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/caffe-cifar-10/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/task.toml b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/task.toml new file mode 100644 index 000000000..5f9766ded --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/task.toml @@ -0,0 +1,5 @@ +[metadata] +category = "systems" + +[verifier] +environment_mode = "shared" diff --git a/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/cancel-async-tasks/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/task.toml b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/task.toml new file mode 100644 index 000000000..070c3446e --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/task.toml @@ -0,0 +1,2 @@ +[metadata] +category = "games" diff --git a/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/chess-best-move/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/dataset-grouping/docker.json b/tests/integrations/harbor/cases/dataset-grouping/docker.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/docker.json @@ -0,0 +1 @@ +{} diff --git a/tests/integrations/harbor/cases/dataset-grouping/expected.json b/tests/integrations/harbor/cases/dataset-grouping/expected.json new file mode 100644 index 000000000..d03e64df1 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/expected.json @@ -0,0 +1,447 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /cancel-async-tasks/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /build-pmars/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /caffe-cifar-10/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [], + "projects": { + "dataset-grouping-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:dataset-grouping-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "cancel-async-tasks", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "columns": { + "category": "systems" + }, + "env": "dataset-grouping-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "cancel-async-tasks" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/cancel-async-tasks/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + }, + "dataset-grouping-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:dataset-grouping-

-
", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "agent_config": { + "timeout_seconds": 45.0 + }, + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [ + { + "service": "main", + "source": "/tmp/result" + }, + { + "destination": "results/outputs", + "exclude": [ + "*.tmp", + "cache" + ], + "service": "main", + "source": "/app/outputs" + } + ], + "collect": [], + "description": "Build pMARS.", + "id": "build-pmars", + "separate_verifier": false, + "verifier_timeout": 30.0 + } + }, + "columns": { + "category": "systems", + "difficulty": "medium", + "keywords": [ + "c", + "build" + ] + }, + "env": "dataset-grouping-
", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "build-pmars" + }, + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "chess-best-move", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "columns": { + "category": "games" + }, + "env": "dataset-grouping-
", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "chess-best-move" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/build-pmars/test.sh", + "compose-project/tests/chess-best-move/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + }, + "dataset-grouping-": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:dataset-grouping--", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "caffe-cifar-10", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "columns": { + "category": "machine-learning" + }, + "env": "dataset-grouping-", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "caffe-cifar-10" + }, + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "sam-cell-seg", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "columns": { + "category": "machine-learning" + }, + "env": "dataset-grouping-", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "sam-cell-seg" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/caffe-cifar-10/test.sh", + "compose-project/tests/sam-cell-seg/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/environment/Dockerfile b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/environment/Dockerfile new file mode 100644 index 000000000..8b58f79a3 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM nvidia/cuda:12.0-runtime-ubuntu22.04 +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/instruction.md b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/task.toml b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/task.toml new file mode 100644 index 000000000..c900fd66a --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/task.toml @@ -0,0 +1,2 @@ +[metadata] +category = "machine-learning" diff --git a/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/tests/test.sh b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/dataset-grouping/sam-cell-seg/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/env-templates/docker.json b/tests/integrations/harbor/cases/env-templates/docker.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/docker.json @@ -0,0 +1 @@ +{} diff --git a/tests/integrations/harbor/cases/env-templates/expected.json b/tests/integrations/harbor/cases/env-templates/expected.json new file mode 100644 index 000000000..280f26c34 --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/expected.json @@ -0,0 +1,126 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /templated/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [], + "projects": { + "env-templates-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:env-templates-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "templated", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "env-templates-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "templated" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/templated/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/env-templates/templated/environment/Dockerfile b/tests/integrations/harbor/cases/env-templates/templated/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/templated/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/env-templates/templated/instruction.md b/tests/integrations/harbor/cases/env-templates/templated/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/templated/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/env-templates/templated/task.toml b/tests/integrations/harbor/cases/env-templates/templated/task.toml new file mode 100644 index 000000000..4690c7964 --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/templated/task.toml @@ -0,0 +1,6 @@ +[environment.env] +JUDGE_KEY = "${HARBOR_JUDGE_KEY}" +MODEL = "${HARBOR_JUDGE_MODEL:-gpt-4o}" + +[verifier.env] +VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" diff --git a/tests/integrations/harbor/cases/env-templates/templated/tests/test.sh b/tests/integrations/harbor/cases/env-templates/templated/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/env-templates/templated/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-absolute-destination/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-absolute-destination/instruction.md b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-absolute-destination/task.toml b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/task.toml new file mode 100644 index 000000000..3f81952f9 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/task.toml @@ -0,0 +1 @@ +artifacts = [{ source = "/output", destination = "/tmp/out" }] diff --git a/tests/integrations/harbor/cases/findings/artifact-absolute-destination/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-absolute-destination/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-backslash-destination/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-backslash-destination/instruction.md b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-backslash-destination/task.toml b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/task.toml new file mode 100644 index 000000000..280c896b4 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/task.toml @@ -0,0 +1 @@ +artifacts = [{ source = "/output", destination = "a\\b" }] diff --git a/tests/integrations/harbor/cases/findings/artifact-backslash-destination/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-backslash-destination/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/instruction.md b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/task.toml b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/task.toml new file mode 100644 index 000000000..9d60c4c7f --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/task.toml @@ -0,0 +1 @@ +artifacts = [{ source = "/output", destination = "../out" }] diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot-destination/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-dotdot/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot/instruction.md b/tests/integrations/harbor/cases/findings/artifact-dotdot/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot/task.toml b/tests/integrations/harbor/cases/findings/artifact-dotdot/task.toml new file mode 100644 index 000000000..b3b91ef1e --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot/task.toml @@ -0,0 +1 @@ +artifacts = ["/workspace/../secret"] diff --git a/tests/integrations/harbor/cases/findings/artifact-dotdot/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-dotdot/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-dotdot/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-double-slash/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-double-slash/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-double-slash/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-double-slash/instruction.md b/tests/integrations/harbor/cases/findings/artifact-double-slash/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-double-slash/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-double-slash/task.toml b/tests/integrations/harbor/cases/findings/artifact-double-slash/task.toml new file mode 100644 index 000000000..f08ceddb7 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-double-slash/task.toml @@ -0,0 +1 @@ +artifacts = ["//"] diff --git a/tests/integrations/harbor/cases/findings/artifact-double-slash/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-double-slash/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-double-slash/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-manifest-destination/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-manifest-destination/instruction.md b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-manifest-destination/task.toml b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/task.toml new file mode 100644 index 000000000..cf19a6e99 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/task.toml @@ -0,0 +1 @@ +artifacts = [{ source = "/output", destination = "manifest.json" }] diff --git a/tests/integrations/harbor/cases/findings/artifact-manifest-destination/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-manifest-destination/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/artifact-root/environment/Dockerfile b/tests/integrations/harbor/cases/findings/artifact-root/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-root/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/artifact-root/instruction.md b/tests/integrations/harbor/cases/findings/artifact-root/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-root/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/artifact-root/task.toml b/tests/integrations/harbor/cases/findings/artifact-root/task.toml new file mode 100644 index 000000000..3132b7263 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-root/task.toml @@ -0,0 +1 @@ +artifacts = ["/"] diff --git a/tests/integrations/harbor/cases/findings/artifact-root/tests/test.sh b/tests/integrations/harbor/cases/findings/artifact-root/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/artifact-root/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/bad-config/environment/Dockerfile b/tests/integrations/harbor/cases/findings/bad-config/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-config/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/bad-config/instruction.md b/tests/integrations/harbor/cases/findings/bad-config/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-config/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/bad-config/task.toml b/tests/integrations/harbor/cases/findings/bad-config/task.toml new file mode 100644 index 000000000..4a3e19fbd --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-config/task.toml @@ -0,0 +1,3 @@ +[environment] +cpus = -1 +network_mode = "sometimes" diff --git a/tests/integrations/harbor/cases/findings/bad-config/tests/test.sh b/tests/integrations/harbor/cases/findings/bad-config/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-config/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/bad-toml/environment/Dockerfile b/tests/integrations/harbor/cases/findings/bad-toml/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-toml/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/bad-toml/instruction.md b/tests/integrations/harbor/cases/findings/bad-toml/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-toml/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/bad-toml/task.toml b/tests/integrations/harbor/cases/findings/bad-toml/task.toml new file mode 100644 index 000000000..8758ff120 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-toml/task.toml @@ -0,0 +1 @@ +not = [valid diff --git a/tests/integrations/harbor/cases/findings/bad-toml/tests/test.sh b/tests/integrations/harbor/cases/findings/bad-toml/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/bad-toml/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/Dockerfile b/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/docker-compose.yaml new file mode 100644 index 000000000..340f75d7a --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-not-mapping/environment/docker-compose.yaml @@ -0,0 +1 @@ +- main diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/instruction.md b/tests/integrations/harbor/cases/findings/compose-not-mapping/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-not-mapping/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/task.toml b/tests/integrations/harbor/cases/findings/compose-not-mapping/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/compose-not-mapping/tests/test.sh b/tests/integrations/harbor/cases/findings/compose-not-mapping/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-not-mapping/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/environment/Dockerfile b/tests/integrations/harbor/cases/findings/compose-path-escape/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-path-escape/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/compose-path-escape/environment/docker-compose.yaml new file mode 100644 index 000000000..2c04472ef --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-path-escape/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + env_file: ../../secret.env diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/instruction.md b/tests/integrations/harbor/cases/findings/compose-path-escape/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-path-escape/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/task.toml b/tests/integrations/harbor/cases/findings/compose-path-escape/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/compose-path-escape/tests/test.sh b/tests/integrations/harbor/cases/findings/compose-path-escape/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-path-escape/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/docker.json b/tests/integrations/harbor/cases/findings/docker.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/docker.json @@ -0,0 +1 @@ +{} diff --git a/tests/integrations/harbor/cases/findings/expected.json b/tests/integrations/harbor/cases/findings/expected.json new file mode 100644 index 000000000..310585c52 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/expected.json @@ -0,0 +1,331 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /supported/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [ + [ + "artifact-absolute-destination", + "harbor.invalid.task_config", + "invalid", + "artifacts.0.destination: Value error, artifact destination must be a relative path" + ], + [ + "artifact-backslash-destination", + "harbor.invalid.task_config", + "invalid", + "artifacts.0.destination: Value error, artifact destination must use forward slashes" + ], + [ + "artifact-dotdot", + "harbor.invalid.task_config", + "invalid", + "artifacts.0.source: Value error, artifact source must name a path beneath /" + ], + [ + "artifact-dotdot-destination", + "harbor.invalid.task_config", + "invalid", + "artifacts.0.destination: Value error, artifact destination must be a relative path" + ], + [ + "artifact-double-slash", + "harbor.invalid.task_config", + "invalid", + "artifacts.0.source: Value error, artifact source must name a path beneath /" + ], + [ + "artifact-manifest-destination", + "harbor.invalid.task_config", + "invalid", + "artifacts.0.destination: Value error, artifact destination 'manifest.json' is reserved" + ], + [ + "artifact-root", + "harbor.invalid.task_config", + "invalid", + "artifacts.0.source: Value error, artifact source must name a path beneath /" + ], + [ + "bad-config", + "harbor.invalid.task_config", + "invalid", + "environment.cpus: Input should be greater than 0" + ], + [ + "bad-config", + "harbor.invalid.task_config", + "invalid", + "environment.network_mode: Input should be 'public', 'no-network' or 'allowlist'" + ], + [ + "bad-toml", + "harbor.invalid.task_config_toml", + "invalid", + "Invalid value (at line 1, column 8)" + ], + [ + "compose-not-mapping", + "harbor.invalid.compose", + "invalid", + "docker-compose.yaml is not a Compose document" + ], + [ + "compose-path-escape", + "harbor.invalid.compose_project_path", + "invalid", + "Compose path '../../secret.env' escapes its project directory" + ], + [ + "healthcheck-duration", + "harbor.invalid.healthcheck", + "invalid", + "invalid Compose healthcheck duration '2 seconds'" + ], + [ + "healthcheck-test", + "harbor.invalid.healthcheck", + "invalid", + "Compose main healthcheck test must be CMD or CMD-SHELL" + ], + [ + "host-variable", + "harbor.unsupported.host_compose_variable", + "contract", + "Compose variable 'HARBOR_JUDGE_KEY' is not set by the project .env" + ], + [ + "main-build-escape", + "harbor.invalid.compose_project_path", + "invalid", + "Compose path '../..' escapes its project directory" + ], + [ + "main-build-escape", + "harbor.invalid.compose_main_build_escape", + "invalid", + "Compose main build escapes environment" + ], + [ + "main-build-escape", + "harbor.invalid.missing_compose_main_dockerfile", + "invalid", + "Compose main Dockerfile does not exist" + ], + [ + "main-build-path-type", + "harbor.invalid.compose_main_build_path", + "invalid", + "Compose main build paths must be strings" + ], + [ + "mcp-servers", + "harbor.invalid.duplicate_mcp_name", + "invalid", + "MCP server names must be unique" + ], + [ + "mcp-servers", + "harbor.invalid.reserved_mcp_name", + "invalid", + "MCP server name 'filetracking' is reserved by the workspace" + ], + [ + "mcp-servers", + "harbor.invalid.reserved_mcp_name", + "invalid", + "MCP server name 'shell' is reserved by the workspace" + ], + [ + "mcp-servers", + "harbor.invalid.mcp_url", + "invalid", + "MCP server 'tools' requires a URL" + ], + [ + "missing-instruction", + "harbor.invalid.missing_instruction", + "invalid", + "missing-instruction has no instruction.md" + ], + [ + "missing-main-dockerfile", + "harbor.invalid.missing_compose_main_dockerfile", + "invalid", + "Compose main Dockerfile does not exist" + ], + [ + "missing-tests", + "harbor.invalid.missing_tests", + "invalid", + "task requires a tests directory" + ], + [ + "missing-verifier-dockerfile", + "harbor.invalid.missing_verifier_dockerfile", + "invalid", + "separate verifier requires tests/Dockerfile" + ], + [ + "multi-step", + "harbor.unsupported.multi_step", + "contract", + "multi-step tasks are not supported" + ], + [ + "no-recipe", + "harbor.invalid.environment_recipe", + "invalid", + "main has no environment/Dockerfile, docker_image, or Compose image or build" + ], + [ + "reserved-ports", + "harbor.invalid.reserved_main_port", + "invalid", + "Harbor main service port 3128 conflicts with a HUD reserved port" + ], + [ + "reserved-ports", + "harbor.invalid.reserved_main_port", + "invalid", + "Harbor main service port 3129 conflicts with a HUD reserved port" + ], + [ + "reserved-ports", + "harbor.invalid.reserved_main_port", + "invalid", + "Harbor main service port 8765 conflicts with a HUD reserved port" + ], + [ + "reserved-services", + "harbor.invalid.reserved_compose_service", + "invalid", + "Compose service names 'hud-base' and 'hud-verifier' are reserved" + ], + [ + "sidecar-without-recipe", + "harbor.invalid.sidecar_recipe", + "invalid", + "Compose service 'worker' has neither image nor build" + ] + ], + "projects": { + "findings-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:findings-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "supported", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "findings-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "supported" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/supported/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/Dockerfile b/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/docker-compose.yaml new file mode 100644 index 000000000..b89312b7f --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-duration/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: + healthcheck: + test: [CMD, 'true'] + interval: 2 seconds diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/instruction.md b/tests/integrations/harbor/cases/findings/healthcheck-duration/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-duration/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/task.toml b/tests/integrations/harbor/cases/findings/healthcheck-duration/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/healthcheck-duration/tests/test.sh b/tests/integrations/harbor/cases/findings/healthcheck-duration/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-duration/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/environment/Dockerfile b/tests/integrations/harbor/cases/findings/healthcheck-test/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-test/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/healthcheck-test/environment/docker-compose.yaml new file mode 100644 index 000000000..3d5410ce8 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-test/environment/docker-compose.yaml @@ -0,0 +1,4 @@ +services: + main: + healthcheck: + test: [CMD-SHELL, a, b] diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/instruction.md b/tests/integrations/harbor/cases/findings/healthcheck-test/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-test/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/task.toml b/tests/integrations/harbor/cases/findings/healthcheck-test/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/healthcheck-test/tests/test.sh b/tests/integrations/harbor/cases/findings/healthcheck-test/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/healthcheck-test/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/host-variable/environment/Dockerfile b/tests/integrations/harbor/cases/findings/host-variable/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/host-variable/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/host-variable/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/host-variable/environment/docker-compose.yaml new file mode 100644 index 000000000..33bdab834 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/host-variable/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + image: ${HARBOR_JUDGE_KEY} diff --git a/tests/integrations/harbor/cases/findings/host-variable/instruction.md b/tests/integrations/harbor/cases/findings/host-variable/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/host-variable/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/host-variable/task.toml b/tests/integrations/harbor/cases/findings/host-variable/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/host-variable/tests/test.sh b/tests/integrations/harbor/cases/findings/host-variable/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/host-variable/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/environment/Dockerfile b/tests/integrations/harbor/cases/findings/main-build-escape/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-escape/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/main-build-escape/environment/docker-compose.yaml new file mode 100644 index 000000000..56c44581f --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-escape/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + build: ../.. diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/instruction.md b/tests/integrations/harbor/cases/findings/main-build-escape/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-escape/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/task.toml b/tests/integrations/harbor/cases/findings/main-build-escape/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/main-build-escape/tests/test.sh b/tests/integrations/harbor/cases/findings/main-build-escape/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-escape/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/environment/Dockerfile b/tests/integrations/harbor/cases/findings/main-build-path-type/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-path-type/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/main-build-path-type/environment/docker-compose.yaml new file mode 100644 index 000000000..90223d5ea --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-path-type/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: + build: + context: . + dockerfile: [Dockerfile] diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/instruction.md b/tests/integrations/harbor/cases/findings/main-build-path-type/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-path-type/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/task.toml b/tests/integrations/harbor/cases/findings/main-build-path-type/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/main-build-path-type/tests/test.sh b/tests/integrations/harbor/cases/findings/main-build-path-type/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/main-build-path-type/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/mcp-servers/environment/Dockerfile b/tests/integrations/harbor/cases/findings/mcp-servers/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/mcp-servers/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/mcp-servers/instruction.md b/tests/integrations/harbor/cases/findings/mcp-servers/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/mcp-servers/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/mcp-servers/task.toml b/tests/integrations/harbor/cases/findings/mcp-servers/task.toml new file mode 100644 index 000000000..a72ac7bd5 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/mcp-servers/task.toml @@ -0,0 +1,18 @@ +[[environment.mcp_servers]] +name = "tools" +transport = "sse" +url = "http://tools:8000/sse" + +[[environment.mcp_servers]] +name = "tools" +transport = "streamable-http" + +[[environment.mcp_servers]] +name = "shell" +transport = "stdio" +command = "shell-mcp" + +[[environment.mcp_servers]] +name = "filetracking" +transport = "stdio" +command = "ft-mcp" diff --git a/tests/integrations/harbor/cases/findings/mcp-servers/tests/test.sh b/tests/integrations/harbor/cases/findings/mcp-servers/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/mcp-servers/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/missing-instruction/environment/Dockerfile b/tests/integrations/harbor/cases/findings/missing-instruction/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-instruction/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/missing-instruction/task.toml b/tests/integrations/harbor/cases/findings/missing-instruction/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/missing-instruction/tests/test.sh b/tests/integrations/harbor/cases/findings/missing-instruction/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-instruction/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/missing-main-dockerfile/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/environment/docker-compose.yaml new file mode 100644 index 000000000..31b70f11b --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: + build: + context: . + dockerfile: Missing diff --git a/tests/integrations/harbor/cases/findings/missing-main-dockerfile/instruction.md b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/missing-main-dockerfile/task.toml b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/missing-main-dockerfile/tests/test.sh b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-main-dockerfile/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/missing-tests/environment/Dockerfile b/tests/integrations/harbor/cases/findings/missing-tests/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-tests/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/missing-tests/instruction.md b/tests/integrations/harbor/cases/findings/missing-tests/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-tests/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/missing-tests/task.toml b/tests/integrations/harbor/cases/findings/missing-tests/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/environment/Dockerfile b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/instruction.md b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/task.toml b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/task.toml new file mode 100644 index 000000000..08c3f86d5 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/task.toml @@ -0,0 +1,2 @@ +[verifier] +environment_mode = "separate" diff --git a/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/tests/test.sh b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/missing-verifier-dockerfile/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/multi-step/environment/Dockerfile b/tests/integrations/harbor/cases/findings/multi-step/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/multi-step/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/multi-step/steps/plan/instruction.md b/tests/integrations/harbor/cases/findings/multi-step/steps/plan/instruction.md new file mode 100644 index 000000000..a12d2e375 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/multi-step/steps/plan/instruction.md @@ -0,0 +1 @@ +# plan diff --git a/tests/integrations/harbor/cases/findings/multi-step/steps/plan/tests/test.sh b/tests/integrations/harbor/cases/findings/multi-step/steps/plan/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/multi-step/steps/plan/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/multi-step/task.toml b/tests/integrations/harbor/cases/findings/multi-step/task.toml new file mode 100644 index 000000000..cf49ef638 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/multi-step/task.toml @@ -0,0 +1,2 @@ +[[steps]] +name = "plan" diff --git a/tests/integrations/harbor/cases/findings/no-recipe/instruction.md b/tests/integrations/harbor/cases/findings/no-recipe/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/no-recipe/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/no-recipe/task.toml b/tests/integrations/harbor/cases/findings/no-recipe/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/no-recipe/tests/test.sh b/tests/integrations/harbor/cases/findings/no-recipe/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/no-recipe/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/environment/Dockerfile b/tests/integrations/harbor/cases/findings/reserved-ports/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-ports/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/reserved-ports/environment/docker-compose.yaml new file mode 100644 index 000000000..8c58c8e16 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-ports/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: + expose: [3128, 3129, 8765, 8080] diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/instruction.md b/tests/integrations/harbor/cases/findings/reserved-ports/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-ports/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/task.toml b/tests/integrations/harbor/cases/findings/reserved-ports/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/reserved-ports/tests/test.sh b/tests/integrations/harbor/cases/findings/reserved-ports/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-ports/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/reserved-services/environment/Dockerfile b/tests/integrations/harbor/cases/findings/reserved-services/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-services/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/reserved-services/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/reserved-services/environment/docker-compose.yaml new file mode 100644 index 000000000..d555138bb --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-services/environment/docker-compose.yaml @@ -0,0 +1,6 @@ +services: + main: {} + hud-base: + image: busybox + hud-verifier: + image: busybox diff --git a/tests/integrations/harbor/cases/findings/reserved-services/instruction.md b/tests/integrations/harbor/cases/findings/reserved-services/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-services/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/reserved-services/task.toml b/tests/integrations/harbor/cases/findings/reserved-services/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/reserved-services/tests/test.sh b/tests/integrations/harbor/cases/findings/reserved-services/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/reserved-services/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/Dockerfile b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/docker-compose.yaml new file mode 100644 index 000000000..3cb0b7224 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + main: {} + worker: {} diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/instruction.md b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/task.toml b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/sidecar-without-recipe/tests/test.sh b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/sidecar-without-recipe/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/supported/environment/Dockerfile b/tests/integrations/harbor/cases/findings/supported/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/supported/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/supported/instruction.md b/tests/integrations/harbor/cases/findings/supported/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/supported/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/supported/task.toml b/tests/integrations/harbor/cases/findings/supported/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/supported/tests/test.sh b/tests/integrations/harbor/cases/findings/supported/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/supported/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/image-failures/broken/environment/Dockerfile b/tests/integrations/harbor/cases/image-failures/broken/environment/Dockerfile new file mode 100644 index 000000000..7f48f97f2 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/broken/environment/Dockerfile @@ -0,0 +1 @@ +FROM registry.invalid/missing diff --git a/tests/integrations/harbor/cases/image-failures/broken/instruction.md b/tests/integrations/harbor/cases/image-failures/broken/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/broken/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/image-failures/broken/task.toml b/tests/integrations/harbor/cases/image-failures/broken/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/image-failures/broken/tests/test.sh b/tests/integrations/harbor/cases/image-failures/broken/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/broken/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/image-failures/docker.json b/tests/integrations/harbor/cases/image-failures/docker.json new file mode 100644 index 000000000..79844e4be --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/docker.json @@ -0,0 +1,18 @@ +{ + "platform": "linux/arm64", + "builds": { + "verifier-on-arm/environment": { + "platform": "linux/amd64" + } + }, + "fail": [ + { + "match": "/broken/environment .* build main$", + "stderr": "failed to resolve source metadata for registry.invalid/missing" + }, + { + "match": "^build ", + "stderr": "no matching distribution for numpy==1.21" + } + ] +} diff --git a/tests/integrations/harbor/cases/image-failures/expected.json b/tests/integrations/harbor/cases/image-failures/expected.json new file mode 100644 index 000000000..2e0418a5a --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/expected.json @@ -0,0 +1,282 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /missing-sidecar-image/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "cache": { + "environment": {}, + "expose": [], + "image": "registry.invalid/cache:1", + "ports": [], + "volumes": [] + }, + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /missing-sidecar-image/environment --file /compose.json --file /override.json pull cache", + "files": { + "compose.json": { + "networks": {}, + "services": { + "cache": { + "environment": {}, + "expose": [], + "image": "registry.invalid/cache:1", + "ports": [], + "volumes": [] + }, + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /verifier-on-arm/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "image inspect --format {{.Os}}/{{.Architecture}} hud-harbor-base:

" + }, + { + "argv": "build --platform linux/amd64 --tag hud-harbor-verifier:image-failures-

-

/verifier-on-arm/tests" + }, + { + "argv": "version --format {{.Server.Os}}/{{.Server.Arch}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /working/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /broken/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:
" + } + } + } + } + } + ], + "failures": [ + [ + "missing-sidecar-image", + "harbor.invalid.image", + "invalid", + "docker compose --project-name hud-adapt-

--project-directory /missing-sidecar-image/environment --file /compose.json --file /override.json pull cache failed: Error response from daemon: pull access denied for registry.invalid/cache:1" + ], + [ + "verifier-on-arm", + "harbor.invalid.image", + "invalid", + "docker build --platform linux/amd64 --tag hud-harbor-verifier:image-failures-

-

/verifier-on-arm/tests failed: no matching distribution for numpy==1.21\nThe docker server builds for linux/arm64. Tasks authored for linux/amd64 often pin dependencies that publish no wheels for other architectures; set DOCKER_DEFAULT_PLATFORM=linux/amd64 to adapt the task for one architecture throughout." + ], + [ + "broken", + "harbor.invalid.image", + "invalid", + "docker compose --project-name hud-adapt-
--project-directory /broken/environment --file /compose.json --file /override.json build main failed: failed to resolve source metadata for registry.invalid/missing" + ] + ], + "projects": { + "image-failures-
": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:image-failures-

-", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "working", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "image-failures-
", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "working" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/working/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/Dockerfile b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/docker-compose.yaml b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/docker-compose.yaml new file mode 100644 index 000000000..83f5ef20f --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +services: + cache: + image: registry.invalid/cache:1 diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/instruction.md b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/task.toml b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/tests/test.sh b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/missing-sidecar-image/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/environment/Dockerfile b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/instruction.md b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/task.toml b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/task.toml new file mode 100644 index 000000000..08c3f86d5 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/task.toml @@ -0,0 +1,2 @@ +[verifier] +environment_mode = "separate" diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/Dockerfile b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/Dockerfile new file mode 100644 index 000000000..2b6adf3c5 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-alpine +COPY . /tests diff --git a/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/test.sh b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/verifier-on-arm/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/image-failures/working/environment/Dockerfile b/tests/integrations/harbor/cases/image-failures/working/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/working/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/image-failures/working/instruction.md b/tests/integrations/harbor/cases/image-failures/working/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/working/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/image-failures/working/task.toml b/tests/integrations/harbor/cases/image-failures/working/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/image-failures/working/tests/test.sh b/tests/integrations/harbor/cases/image-failures/working/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/image-failures/working/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/plain-image/docker.json b/tests/integrations/harbor/cases/plain-image/docker.json new file mode 100644 index 000000000..2192a7eca --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/docker.json @@ -0,0 +1,9 @@ +{ + "builds": { + "task-a/environment": { + "config": { + "WorkingDir": "/workspace" + } + } + } +} diff --git a/tests/integrations/harbor/cases/plain-image/expected.json b/tests/integrations/harbor/cases/plain-image/expected.json new file mode 100644 index 000000000..f6e83d4c2 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/expected.json @@ -0,0 +1,132 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /task-a/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [], + "projects": { + "plain-image-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:plain-image-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "task-a", + "separate_verifier": false, + "verifier_timeout": 120.0 + } + }, + "columns": { + "category": "systems" + }, + "env": "plain-image-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "task-a" + } + ], + "tree": [ + "compose-project/environment/.dockerignore", + "compose-project/environment/Dockerfile", + "compose-project/environment/docker-compose.yml", + "compose-project/environment/ignored.txt", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/task-a/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/plain-image/task-a/environment/.dockerignore b/tests/integrations/harbor/cases/plain-image/task-a/environment/.dockerignore new file mode 100644 index 000000000..cdb20a6c1 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/environment/.dockerignore @@ -0,0 +1,2 @@ +hud/ +Dockerfile diff --git a/tests/integrations/harbor/cases/plain-image/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/plain-image/task-a/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/plain-image/task-a/environment/docker-compose.yml b/tests/integrations/harbor/cases/plain-image/task-a/environment/docker-compose.yml new file mode 100644 index 000000000..b8e29e994 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/environment/docker-compose.yml @@ -0,0 +1 @@ +{"api_gateway": {"interval": "30s"}} diff --git a/tests/integrations/harbor/cases/plain-image/task-a/environment/ignored.txt b/tests/integrations/harbor/cases/plain-image/task-a/environment/ignored.txt new file mode 100644 index 000000000..4eea88a85 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/environment/ignored.txt @@ -0,0 +1 @@ +unchanged diff --git a/tests/integrations/harbor/cases/plain-image/task-a/instruction.md b/tests/integrations/harbor/cases/plain-image/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/plain-image/task-a/task.toml b/tests/integrations/harbor/cases/plain-image/task-a/task.toml new file mode 100644 index 000000000..d1ffb5ceb --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/task.toml @@ -0,0 +1,5 @@ +[metadata] +category = "systems" + +[verifier] +timeout_sec = 120 diff --git a/tests/integrations/harbor/cases/plain-image/task-a/tests/test.sh b/tests/integrations/harbor/cases/plain-image/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/plain-image/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/prebuilt-image/docker.json b/tests/integrations/harbor/cases/prebuilt-image/docker.json new file mode 100644 index 000000000..47067dc59 --- /dev/null +++ b/tests/integrations/harbor/cases/prebuilt-image/docker.json @@ -0,0 +1,9 @@ +{ + "pulls": { + "registry.example/base:latest": { + "config": { + "WorkingDir": "/srv" + } + } + } +} diff --git a/tests/integrations/harbor/cases/prebuilt-image/expected.json b/tests/integrations/harbor/cases/prebuilt-image/expected.json new file mode 100644 index 000000000..cdbf5be74 --- /dev/null +++ b/tests/integrations/harbor/cases/prebuilt-image/expected.json @@ -0,0 +1,107 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-missing --project-directory /task-a/environment --file /compose.json --file /override.json pull main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "image": "registry.example/base:latest" + } + } + } + } + }, + { + "argv": "image inspect registry.example/base:latest" + } + ], + "failures": [], + "projects": { + "prebuilt-image-

": { + "compose.yaml": { + "networks": {}, + "services": { + "main": { + "build": { + "args": { + "BASE_IMAGE": "registry.example/base:latest", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "registry.example/base:latest" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:prebuilt-image-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "task-a", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "prebuilt-image-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "task-a" + } + ], + "tree": [ + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/task-a/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/prebuilt-image/task-a/instruction.md b/tests/integrations/harbor/cases/prebuilt-image/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/prebuilt-image/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/prebuilt-image/task-a/task.toml b/tests/integrations/harbor/cases/prebuilt-image/task-a/task.toml new file mode 100644 index 000000000..ab30acba7 --- /dev/null +++ b/tests/integrations/harbor/cases/prebuilt-image/task-a/task.toml @@ -0,0 +1,2 @@ +[environment] +docker_image = "registry.example/base:latest" diff --git a/tests/integrations/harbor/cases/prebuilt-image/task-a/tests/test.sh b/tests/integrations/harbor/cases/prebuilt-image/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/prebuilt-image/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/resources-and-budgets/budget/environment/Dockerfile b/tests/integrations/harbor/cases/resources-and-budgets/budget/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/budget/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/resources-and-budgets/budget/instruction.md b/tests/integrations/harbor/cases/resources-and-budgets/budget/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/budget/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/resources-and-budgets/budget/task.toml b/tests/integrations/harbor/cases/resources-and-budgets/budget/task.toml new file mode 100644 index 000000000..ce24d8eae --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/budget/task.toml @@ -0,0 +1,5 @@ +[environment] +build_timeout_sec = 900.5 + +[agent] +timeout_sec = 28800 diff --git a/tests/integrations/harbor/cases/resources-and-budgets/budget/tests/test.sh b/tests/integrations/harbor/cases/resources-and-budgets/budget/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/budget/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/environment/Dockerfile b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/instruction.md b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/task.toml b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/task.toml new file mode 100644 index 000000000..e2e50397d --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/task.toml @@ -0,0 +1,2 @@ +[environment] +gpus = 0 diff --git a/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/tests/test.sh b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/cpu-only/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/resources-and-budgets/docker.json b/tests/integrations/harbor/cases/resources-and-budgets/docker.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/docker.json @@ -0,0 +1 @@ +{} diff --git a/tests/integrations/harbor/cases/resources-and-budgets/expected.json b/tests/integrations/harbor/cases/resources-and-budgets/expected.json new file mode 100644 index 000000000..18b683817 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/expected.json @@ -0,0 +1,495 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /budget/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /gpu/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /cpu-only/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /windows-tpu/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [], + "projects": { + "resources-and-budgets-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:resources-and-budgets-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "cpu-only", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "resources-and-budgets-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "cpu-only" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/cpu-only/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + }, + "resources-and-budgets-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:resources-and-budgets-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "windows-tpu", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "resources-and-budgets-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": { + "os": "windows", + "tpu": { + "topology": "2x2", + "type": "v5" + } + } + }, + "slug": "windows-tpu" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/windows-tpu/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + }, + "resources-and-budgets-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:resources-and-budgets-

-", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "agent_config": { + "timeout_seconds": 28800.0 + }, + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "budget", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "resources-and-budgets-
", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": { + "startup_timeout_s": 901 + }, + "resources": null + }, + "slug": "budget" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/budget/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + }, + "resources-and-budgets-": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:resources-and-budgets--", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "gpu", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "resources-and-budgets-", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": { + "cpu": 4.0, + "gpu": { + "count": 2, + "type": [ + "H100" + ] + }, + "memory_mb": 8192, + "storage_mb": 32768 + } + }, + "slug": "gpu" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/gpu/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/resources-and-budgets/gpu/environment/Dockerfile b/tests/integrations/harbor/cases/resources-and-budgets/gpu/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/gpu/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/resources-and-budgets/gpu/instruction.md b/tests/integrations/harbor/cases/resources-and-budgets/gpu/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/gpu/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/resources-and-budgets/gpu/task.toml b/tests/integrations/harbor/cases/resources-and-budgets/gpu/task.toml new file mode 100644 index 000000000..eef4cfae7 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/gpu/task.toml @@ -0,0 +1,6 @@ +[environment] +cpus = 4 +memory_mb = 8192 +storage_mb = 32768 +gpus = 2 +gpu_types = ["H100"] diff --git a/tests/integrations/harbor/cases/resources-and-budgets/gpu/tests/test.sh b/tests/integrations/harbor/cases/resources-and-budgets/gpu/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/gpu/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/environment/Dockerfile b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/instruction.md b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/task.toml b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/task.toml new file mode 100644 index 000000000..682c1503d --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/task.toml @@ -0,0 +1,3 @@ +[environment] +os = "windows" +tpu = { type = "v5", topology = "2x2" } diff --git a/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/tests/test.sh b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/resources-and-budgets/windows-tpu/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/separate-verifier/docker.json b/tests/integrations/harbor/cases/separate-verifier/docker.json new file mode 100644 index 000000000..fae5ccf06 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/docker.json @@ -0,0 +1,10 @@ +{ + "builds": { + "placement/environment": { + "platform": "linux/arm64" + } + }, + "pulls": { + "redis:7-alpine": {} + } +} diff --git a/tests/integrations/harbor/cases/separate-verifier/expected.json b/tests/integrations/harbor/cases/separate-verifier/expected.json new file mode 100644 index 000000000..e603f102c --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/expected.json @@ -0,0 +1,550 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /placement/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "redis": { + "environment": {}, + "expose": [ + "6379" + ], + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "image inspect --format {{.Os}}/{{.Architecture}} hud-harbor-base:

" + }, + { + "argv": "build --platform linux/arm64 --tag hud-harbor-verifier:separate-verifier-

-

/placement/tests" + }, + { + "argv": "image inspect hud-harbor-verifier:separate-verifier-

-

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /plain/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "redis": { + "environment": {}, + "expose": [ + "6379" + ], + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "image inspect --format {{.Os}}/{{.Architecture}} hud-harbor-base:

" + }, + { + "argv": "build --platform linux/amd64 --tag hud-harbor-verifier:separate-verifier-

-

/plain/tests" + }, + { + "argv": "image inspect hud-harbor-verifier:separate-verifier-

-

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /same-placement/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:
" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:
" + }, + { + "argv": "image inspect --format {{.Os}}/{{.Architecture}} hud-harbor-base:
" + }, + { + "argv": "build --platform linux/amd64 --tag hud-harbor-verifier:separate-verifier-
-

/same-placement/tests" + }, + { + "argv": "image inspect hud-harbor-verifier:separate-verifier-
-

" + } + ], + "failures": [], + "projects": { + "separate-verifier-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "hud-verifier": { + "build": { + "context": "./compose-project/verifier" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-verifier:separate-verifier-

-

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base", + "hud-verifier": "service:hud-verifier" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-verifier" + }, + "context": "./compose-project/main", + "target": "verifier" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:separate-verifier-

-", + "ports": [], + "volumes": [] + }, + "redis": { + "environment": {}, + "expose": [ + "6379" + ], + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [ + { + "service": "main", + "source": "/tmp/agent.patch" + } + ], + "collect": [ + { + "command": "redis-cli save", + "service": "redis", + "timeout_sec": 10.0 + } + ], + "description": "", + "id": "placement", + "separate_verifier": true, + "verifier_timeout": 30.0 + } + }, + "env": "separate-verifier-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": true + }, + "limits": { + "startup_timeout_s": 600 + }, + "resources": { + "cpu": 2.0, + "memory_mb": 2048 + } + }, + "slug": "placement", + "verifier": { + "args": { + "task": { + "artifacts": [ + { + "service": "main", + "source": "/tmp/agent.patch" + } + ], + "collect": [ + { + "command": "redis-cli save", + "service": "redis", + "timeout_sec": 10.0 + } + ], + "description": "", + "id": "placement", + "separate_verifier": true, + "verifier_timeout": 30.0 + } + }, + "env": "separate-verifier-

", + "id": "verify", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": "." + }, + "limits": { + "startup_timeout_s": 1200 + }, + "resources": { + "cpu": 4.0, + "gpu": { + "count": 1, + "type": [ + "T4" + ] + }, + "memory_mb": 1024 + } + }, + "slug": "placement:verify" + } + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/environment/docker-compose.yaml", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/verifier/Dockerfile", + "compose-project/verifier/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + }, + "separate-verifier-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "hud-verifier": { + "build": { + "context": "./compose-project/verifier" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-verifier:separate-verifier-

-

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base", + "hud-verifier": "service:hud-verifier" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-verifier" + }, + "context": "./compose-project/main", + "target": "verifier" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:separate-verifier-

-", + "ports": [], + "volumes": [] + }, + "redis": { + "environment": {}, + "expose": [ + "6379" + ], + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "plain", + "separate_verifier": true, + "verifier_timeout": 600.0 + } + }, + "env": "separate-verifier-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "plain", + "verifier": { + "args": { + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "plain", + "separate_verifier": true, + "verifier_timeout": 600.0 + } + }, + "env": "separate-verifier-

", + "id": "verify", + "slug": "plain:verify" + } + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/environment/docker-compose.yaml", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/verifier/Dockerfile", + "compose-project/verifier/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + }, + "separate-verifier-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:
", + "ports": [], + "scale": 0, + "volumes": [] + }, + "hud-verifier": { + "build": { + "context": "./compose-project/verifier" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-verifier:separate-verifier-
-

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base", + "hud-verifier": "service:hud-verifier" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-verifier" + }, + "context": "./compose-project/main", + "target": "verifier" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:separate-verifier-

-", + "ports": [], + "volumes": [] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "same-placement", + "separate_verifier": true, + "verifier_timeout": 600.0 + } + }, + "env": "separate-verifier-
", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": { + "cpu": 4.0, + "memory_mb": 14336 + } + }, + "slug": "same-placement", + "verifier": { + "args": { + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "same-placement", + "separate_verifier": true, + "verifier_timeout": 600.0 + } + }, + "env": "separate-verifier-
", + "id": "verify", + "slug": "same-placement:verify" + } + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/verifier/Dockerfile", + "compose-project/verifier/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/environment/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/placement/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/environment/docker-compose.yaml b/tests/integrations/harbor/cases/separate-verifier/placement/environment/docker-compose.yaml new file mode 100644 index 000000000..086670a2d --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: {} + redis: + image: redis:7-alpine + expose: [6379] diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/instruction.md b/tests/integrations/harbor/cases/separate-verifier/placement/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/task.toml b/tests/integrations/harbor/cases/separate-verifier/placement/task.toml new file mode 100644 index 000000000..6b07a4817 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/task.toml @@ -0,0 +1,22 @@ +artifacts = ["/tmp/agent.patch"] + +[environment] +cpus = 2 +memory_mb = 2048 +build_timeout_sec = 600 + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[verifier.environment] +cpus = 4 +memory_mb = 1024 +build_timeout_sec = 1200 +gpus = 1 +gpu_types = ["T4"] + +[[verifier.collect]] +service = "redis" +command = "redis-cli save" +timeout_sec = 10 diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/tests/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/placement/tests/Dockerfile new file mode 100644 index 000000000..2b6adf3c5 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-alpine +COPY . /tests diff --git a/tests/integrations/harbor/cases/separate-verifier/placement/tests/test.sh b/tests/integrations/harbor/cases/separate-verifier/placement/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/placement/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/environment/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/plain/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/environment/docker-compose.yaml b/tests/integrations/harbor/cases/separate-verifier/plain/environment/docker-compose.yaml new file mode 100644 index 000000000..086670a2d --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: {} + redis: + image: redis:7-alpine + expose: [6379] diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/instruction.md b/tests/integrations/harbor/cases/separate-verifier/plain/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/task.toml b/tests/integrations/harbor/cases/separate-verifier/plain/task.toml new file mode 100644 index 000000000..08c3f86d5 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/task.toml @@ -0,0 +1,2 @@ +[verifier] +environment_mode = "separate" diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/tests/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/plain/tests/Dockerfile new file mode 100644 index 000000000..2b6adf3c5 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-alpine +COPY . /tests diff --git a/tests/integrations/harbor/cases/separate-verifier/plain/tests/test.sh b/tests/integrations/harbor/cases/separate-verifier/plain/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/plain/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/environment/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/same-placement/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/instruction.md b/tests/integrations/harbor/cases/separate-verifier/same-placement/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/task.toml b/tests/integrations/harbor/cases/separate-verifier/same-placement/task.toml new file mode 100644 index 000000000..a2e78698b --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/task.toml @@ -0,0 +1,10 @@ +[environment] +cpus = 4 +memory_mb = 14336 + +[verifier] +environment_mode = "separate" + +[verifier.environment] +cpus = 4 +memory_mb = 14336 diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/Dockerfile b/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/Dockerfile new file mode 100644 index 000000000..2b6adf3c5 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-alpine +COPY . /tests diff --git a/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/test.sh b/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/separate-verifier/same-placement/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/docker.json b/tests/integrations/harbor/cases/shared-collect-sidecar/docker.json new file mode 100644 index 000000000..f3f75b0e3 --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/docker.json @@ -0,0 +1,5 @@ +{ + "pulls": { + "redis:7-alpine": {} + } +} diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/expected.json b/tests/integrations/harbor/cases/shared-collect-sidecar/expected.json new file mode 100644 index 000000000..e36e39272 --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/expected.json @@ -0,0 +1,156 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /task-a/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "redis": { + "environment": {}, + "expose": [ + "6379" + ], + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [], + "projects": { + "shared-collect-sidecar-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "service-access" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:shared-collect-sidecar-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + }, + "redis": { + "environment": {}, + "expose": [ + "6379" + ], + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [ + { + "service": "redis", + "source": "/data/dump.rdb" + } + ], + "collect": [ + { + "command": "redis-cli save", + "service": "redis", + "timeout_sec": 600.0 + } + ], + "description": "", + "id": "task-a", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "shared-collect-sidecar-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": true + }, + "limits": null, + "resources": null + }, + "slug": "task-a" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/environment/docker-compose.yaml", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/task-a/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/docker-compose.yaml b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/docker-compose.yaml new file mode 100644 index 000000000..086670a2d --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/environment/docker-compose.yaml @@ -0,0 +1,5 @@ +services: + main: {} + redis: + image: redis:7-alpine + expose: [6379] diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/instruction.md b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/task.toml b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/task.toml new file mode 100644 index 000000000..becc8f63c --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/task.toml @@ -0,0 +1,5 @@ +artifacts = [{ source = "/data/dump.rdb", service = "redis" }] + +[[verifier.collect]] +service = "redis" +command = "redis-cli save" diff --git a/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/tests/test.sh b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/shared-collect-sidecar/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/sidecars/docker.json b/tests/integrations/harbor/cases/sidecars/docker.json new file mode 100644 index 000000000..3be2a1d67 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/docker.json @@ -0,0 +1,22 @@ +{ + "builds": { + "implicit-main/environment/cache": { + "config": { + "ExposedPorts": { + "11211/tcp": {}, + "11211/udp": {} + } + } + } + }, + "pulls": { + "redis:7-alpine": { + "config": { + "ExposedPorts": { + "6379/tcp": {} + } + } + }, + "busybox:1.37": {} + } +} diff --git a/tests/integrations/harbor/cases/sidecars/expected.json b/tests/integrations/harbor/cases/sidecars/expected.json new file mode 100644 index 000000000..9fa6e0173 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/expected.json @@ -0,0 +1,556 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /main-process/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "entrypoint": [ + "/compose-init" + ], + "environment": { + "FROM_COMPOSE": "yes" + }, + "expose": [ + "8080" + ], + "healthcheck": { + "interval": "2s", + "retries": 5, + "start_period": "1s", + "test": [ + "CMD-SHELL", + "curl -f http://localhost:8080/health" + ], + "timeout": "3s" + }, + "ports": [], + "user": "1001:1002", + "volumes": [], + "working_dir": "/compose-work" + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /implicit-main/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "cache": { + "build": "./cache", + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "redis": { + "command": [ + "redis-server", + "--save", + "" + ], + "depends_on": { + "main": { + "condition": "service_healthy", + "restart": true + }, + "seed": { + "condition": "service_completed_successfully" + } + }, + "environment": { + "SIDE": "car" + }, + "expose": [ + "6379" + ], + "healthcheck": { + "interval": "2s", + "test": [ + "CMD", + "redis-cli", + "ping" + ] + }, + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + }, + "seed": { + "build": "./seed", + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "worker": { + "environment": {}, + "expose": [], + "image": "busybox:1.37", + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "cache": { + "image": "hud-harbor-sidecar:

" + }, + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /implicit-main/environment --file /compose.json --file /override.json build cache", + "files": { + "compose.json": { + "networks": {}, + "services": { + "cache": { + "build": "./cache", + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "redis": { + "command": [ + "redis-server", + "--save", + "" + ], + "depends_on": { + "main": { + "condition": "service_healthy", + "restart": true + }, + "seed": { + "condition": "service_completed_successfully" + } + }, + "environment": { + "SIDE": "car" + }, + "expose": [ + "6379" + ], + "healthcheck": { + "interval": "2s", + "test": [ + "CMD", + "redis-cli", + "ping" + ] + }, + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + }, + "seed": { + "build": "./seed", + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "worker": { + "environment": {}, + "expose": [], + "image": "busybox:1.37", + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "cache": { + "image": "hud-harbor-sidecar:

" + }, + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-sidecar:

" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /implicit-main/environment --file /compose.json --file /override.json pull worker", + "files": { + "compose.json": { + "networks": {}, + "services": { + "cache": { + "build": "./cache", + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "redis": { + "command": [ + "redis-server", + "--save", + "" + ], + "depends_on": { + "main": { + "condition": "service_healthy", + "restart": true + }, + "seed": { + "condition": "service_completed_successfully" + } + }, + "environment": { + "SIDE": "car" + }, + "expose": [ + "6379" + ], + "healthcheck": { + "interval": "2s", + "test": [ + "CMD", + "redis-cli", + "ping" + ] + }, + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + }, + "seed": { + "build": "./seed", + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + }, + "worker": { + "environment": {}, + "expose": [], + "image": "busybox:1.37", + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "cache": { + "image": "hud-harbor-sidecar:

" + }, + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect busybox:1.37" + } + ], + "failures": [], + "projects": { + "sidecars-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": { + "FROM_COMPOSE": "yes" + }, + "expose": [ + "8080" + ], + "image": "hud-harbor:sidecars-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "main-process", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "sidecars-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "main-process" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/environment/docker-compose.yaml", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/main-process/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + }, + "sidecars-

": { + "compose.yaml": { + "networks": {}, + "services": { + "cache": { + "build": "./compose-project/environment/cache", + "environment": {}, + "expose": [], + "image": "hud-harbor-sidecar:

", + "ports": [], + "volumes": [] + }, + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:sidecars-

-", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + }, + "redis": { + "command": [ + "redis-server", + "--save", + "" + ], + "depends_on": { + "main": { + "condition": "service_started", + "restart": true + }, + "seed": { + "condition": "service_completed_successfully" + } + }, + "environment": { + "SIDE": "car" + }, + "expose": [ + "6379" + ], + "healthcheck": { + "interval": "2s", + "test": [ + "CMD", + "redis-cli", + "ping" + ] + }, + "image": "redis:7-alpine", + "ports": [], + "volumes": [] + }, + "seed": { + "build": "./compose-project/environment/seed", + "environment": {}, + "expose": [], + "image": "hud-harbor-sidecar:", + "ports": [], + "volumes": [] + }, + "worker": { + "environment": {}, + "expose": [], + "image": "busybox:1.37", + "ports": [], + "volumes": [] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "implicit-main", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "sidecars-
", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "implicit-main" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/environment/cache/Dockerfile", + "compose-project/environment/docker-compose.yaml", + "compose-project/environment/seed/Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/implicit-main/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/environment/Dockerfile b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/environment/cache/Dockerfile b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/cache/Dockerfile new file mode 100644 index 000000000..4003a95aa --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/cache/Dockerfile @@ -0,0 +1 @@ +FROM memcached:1.6 diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/environment/docker-compose.yaml b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/docker-compose.yaml new file mode 100644 index 000000000..c70230697 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/docker-compose.yaml @@ -0,0 +1,21 @@ +services: + redis: + image: redis:7-alpine + depends_on: + main: + condition: service_healthy + restart: true + seed: + condition: service_completed_successfully + command: [redis-server, --save, ""] + environment: {SIDE: car} + expose: [6379] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + worker: + image: busybox:1.37 + cache: + build: ./cache + seed: + build: ./seed diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/environment/seed/Dockerfile b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/seed/Dockerfile new file mode 100644 index 000000000..4f0cb0660 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/environment/seed/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:1.37 +CMD ["true"] diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/instruction.md b/tests/integrations/harbor/cases/sidecars/implicit-main/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/task.toml b/tests/integrations/harbor/cases/sidecars/implicit-main/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/sidecars/implicit-main/tests/test.sh b/tests/integrations/harbor/cases/sidecars/implicit-main/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/implicit-main/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/sidecars/main-process/environment/Dockerfile b/tests/integrations/harbor/cases/sidecars/main-process/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/main-process/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/sidecars/main-process/environment/docker-compose.yaml b/tests/integrations/harbor/cases/sidecars/main-process/environment/docker-compose.yaml new file mode 100644 index 000000000..000fb6084 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/main-process/environment/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + main: + user: 1001:1002 + entrypoint: [/compose-init] + working_dir: /compose-work + environment: {FROM_COMPOSE: "yes"} + expose: [8080] + healthcheck: + test: [CMD-SHELL, curl -f http://localhost:8080/health] + interval: 2s + timeout: 3s + retries: 5 + start_period: 1s diff --git a/tests/integrations/harbor/cases/sidecars/main-process/instruction.md b/tests/integrations/harbor/cases/sidecars/main-process/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/main-process/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/sidecars/main-process/task.toml b/tests/integrations/harbor/cases/sidecars/main-process/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/sidecars/main-process/tests/test.sh b/tests/integrations/harbor/cases/sidecars/main-process/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/sidecars/main-process/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/docker.json b/tests/integrations/harbor/cases/source-builds-and-volumes/docker.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/docker.json @@ -0,0 +1 @@ +{} diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/expected.json b/tests/integrations/harbor/cases/source-builds-and-volumes/expected.json new file mode 100644 index 000000000..249e62c51 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/expected.json @@ -0,0 +1,187 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /task-a/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "database": { + "build": { + "context": "./database" + }, + "env_file": "./database/db.env", + "environment": {}, + "expose": [ + "5432" + ], + "ports": [], + "volumes": [ + "./database/data:/var/lib/postgresql/data" + ] + }, + "main": { + "env_file": "./main.env", + "environment": {}, + "expose": [], + "ports": [], + "volumes": [ + "./main-data:/var/lib/main", + { + "read_only": true, + "source": "./readonly", + "target": "/etc/authored", + "type": "bind" + }, + "outputs:/app/outputs" + ] + } + }, + "volumes": { + "outputs": {} + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [], + "projects": { + "source-builds-and-volumes-

": { + "compose.yaml": { + "networks": {}, + "services": { + "database": { + "build": { + "context": "./compose-project/environment/database" + }, + "env_file": "./compose-project/environment/database/db.env", + "environment": {}, + "expose": [ + "5432" + ], + "image": "hud-harbor-sidecar:

", + "ports": [], + "volumes": [ + "./compose-project/environment/database/data:/var/lib/postgresql/data" + ] + }, + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "env_file": "./compose-project/environment/main.env", + "environment": {}, + "expose": [], + "image": "hud-harbor:source-builds-and-volumes-

-

", + "ports": [], + "volumes": [ + "./compose-project/environment/main-data:/mounts/0", + { + "read_only": true, + "source": "./compose-project/environment/readonly", + "target": "/mounts/1", + "type": "bind" + }, + "outputs:/mounts/2", + "./compose-project/tests:/controller/tests:ro" + ] + } + }, + "volumes": { + "outputs": {} + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "task-a", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "source-builds-and-volumes-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "task-a" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/environment/database/Dockerfile", + "compose-project/environment/database/data/.keep", + "compose-project/environment/database/db.env", + "compose-project/environment/docker-compose.yaml", + "compose-project/environment/main-data/.keep", + "compose-project/environment/main.env", + "compose-project/environment/readonly/.keep", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/task-a/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/Dockerfile b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/Dockerfile new file mode 100644 index 000000000..9a1a0f96e --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/Dockerfile @@ -0,0 +1 @@ +FROM postgres:16 diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/data/.keep b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/data/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/db.env b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/db.env new file mode 100644 index 000000000..cb5b149b8 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/database/db.env @@ -0,0 +1 @@ +POSTGRES_DB=test diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/docker-compose.yaml b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/docker-compose.yaml new file mode 100644 index 000000000..f2644a3c8 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/docker-compose.yaml @@ -0,0 +1,18 @@ +services: + main: + env_file: ./main.env + volumes: + - ./main-data:/var/lib/main + - type: bind + source: ./readonly + target: /etc/authored + read_only: true + - outputs:/app/outputs + database: + build: + context: ./database + env_file: ./database/db.env + volumes: [./database/data:/var/lib/postgresql/data] + expose: [5432] +volumes: + outputs: {} diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main-data/.keep b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main-data/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main.env b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main.env new file mode 100644 index 000000000..bed5dccd5 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/main.env @@ -0,0 +1 @@ +MAIN=true diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/readonly/.keep b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/environment/readonly/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/instruction.md b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/task.toml b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/tests/test.sh b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/source-builds-and-volumes/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/symlinks/docker.json b/tests/integrations/harbor/cases/symlinks/docker.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/docker.json @@ -0,0 +1 @@ +{} diff --git a/tests/integrations/harbor/cases/symlinks/expected.json b/tests/integrations/harbor/cases/symlinks/expected.json new file mode 100644 index 000000000..3e8e4a96c --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/expected.json @@ -0,0 +1,128 @@ +{ + "docker": [ + { + "argv": "version --format {{.Server.Version}}" + }, + { + "argv": "compose --project-name hud-adapt-

--project-directory /task-a/environment --file /compose.json --file /override.json build main", + "files": { + "compose.json": { + "networks": {}, + "services": { + "main": { + "environment": {}, + "expose": [], + "ports": [], + "volumes": [] + } + } + }, + "override.json": { + "services": { + "main": { + "build": { + "context": ".", + "dockerfile": "Dockerfile" + }, + "image": "hud-harbor-base:

" + } + } + } + } + }, + { + "argv": "image inspect hud-harbor-base:

" + } + ], + "failures": [], + "projects": { + "symlinks-

": { + "compose.yaml": { + "networks": {}, + "services": { + "hud-base": { + "build": { + "context": "./compose-project/environment" + }, + "environment": {}, + "expose": [], + "image": "hud-harbor-base:

", + "ports": [], + "scale": 0, + "volumes": [] + }, + "main": { + "build": { + "additional_contexts": { + "hud-base": "service:hud-base" + }, + "args": { + "BASE_IMAGE": "hud-base", + "HUD_REQUIREMENT": "hud", + "VERIFIER_IMAGE": "hud-harbor-base:

" + }, + "context": "./compose-project/main", + "target": "plain" + }, + "command": [ + "/controller/venv/bin/hud", + "serve", + "hud.integrations.harbor.env:env", + "--host", + "0.0.0.0", + "--port", + "8765" + ], + "entrypoint": [], + "environment": {}, + "expose": [], + "image": "hud-harbor:symlinks-

-

", + "ports": [], + "volumes": [ + "./compose-project/tests:/controller/tests:ro" + ] + } + } + }, + "tasks.json": [ + { + "args": { + "instruction": "Solve the task.\n", + "task": { + "artifacts": [], + "collect": [], + "description": "", + "id": "task-a", + "separate_verifier": false, + "verifier_timeout": 600.0 + } + }, + "env": "symlinks-

", + "id": "run", + "runtime_config": { + "compose": { + "document": "compose.yaml", + "root": ".", + "service_access": null + }, + "limits": null, + "resources": null + }, + "slug": "task-a" + } + ], + "tree": [ + "compose-project/environment/Dockerfile", + "compose-project/environment/Dockerfile.link -> Dockerfile", + "compose-project/main/Dockerfile", + "compose-project/main/config.json", + "compose-project/main/install.sh", + "compose-project/tests/task-a/host-file -> /etc/hostname", + "compose-project/tests/task-a/test.sh", + "compose.yaml", + "env.py", + "tasks.json" + ] + } + } +} \ No newline at end of file diff --git a/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile b/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile.link b/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile.link new file mode 120000 index 000000000..1d1fe94df --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/environment/Dockerfile.link @@ -0,0 +1 @@ +Dockerfile \ No newline at end of file diff --git a/tests/integrations/harbor/cases/symlinks/task-a/instruction.md b/tests/integrations/harbor/cases/symlinks/task-a/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/symlinks/task-a/task.toml b/tests/integrations/harbor/cases/symlinks/task-a/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/symlinks/task-a/tests/host-file b/tests/integrations/harbor/cases/symlinks/task-a/tests/host-file new file mode 120000 index 000000000..48980ad58 --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/tests/host-file @@ -0,0 +1 @@ +/etc/hostname \ No newline at end of file diff --git a/tests/integrations/harbor/cases/symlinks/task-a/tests/test.sh b/tests/integrations/harbor/cases/symlinks/task-a/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/symlinks/task-a/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/test_adapt.py b/tests/integrations/harbor/test_adapt.py new file mode 100644 index 000000000..2e844cd15 --- /dev/null +++ b/tests/integrations/harbor/test_adapt.py @@ -0,0 +1,281 @@ +"""``harbor.adapt`` over case directories of Harbor tasks, against the fake docker. + +Each directory under ``cases/`` is a Harbor dataset plus a ``docker.json`` that +says what the docker daemon holds: the OCI config each build context and pulled +image produces, the server platform, and the commands that fail. The scenario +adapts a copy of the dataset and compares what a user and the runtime observe +(the failures, the generated tree, each project's ``compose.yaml`` and +``tasks.json``, and every docker command) with the case's ``expected.json``. +Record a new case with ``--inline-snapshot=create``. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest +from inline_snapshot import external_file + +from hud.integrations import harbor + +if TYPE_CHECKING: + from collections.abc import Callable + + from tests.harness import FakeDocker, HudEnv + +CASES = Path(__file__).parent / "cases" +SECRET = "sk-live-secret" +HASH = re.compile(r"\b[0-9a-f]{12}(?:[0-9a-f]{4})?\b") +RESOLVE_DIR = re.compile(r"[^\s\"']*/hud-harbor-resolve-[^/\s\"']+") + + +def load_case(case: str, tmp_path: Path, fake_docker: FakeDocker, hud_env: HudEnv) -> Path: + """Copy ``case`` into ``tmp_path`` and load its ``docker.json`` into the fake docker.""" + dataset = tmp_path / case + shutil.copytree( + CASES / case, + dataset, + symlinks=True, + ignore=shutil.ignore_patterns("docker.json", "expected.json"), + ) + spec = json.loads((CASES / case / "docker.json").read_text("utf-8")) + fake_docker.images( + dataset, + builds=spec.get("builds"), + pulls=spec.get("pulls"), + platform=spec.get("platform", "linux/amd64"), + ) + for failure in spec.get("fail", []): + fake_docker.on(failure["match"], stderr=failure["stderr"], exit=1) + # A host value a template names must never reach anything adapt writes. + hud_env.set(HARBOR_JUDGE_KEY=SECRET) + return dataset + + +def mask(document: Any) -> Any: + """Replace each distinct content hash with ````, numbered by first appearance.""" + text = json.dumps(document, sort_keys=True) + labels: dict[str, str] = {} + for token in HASH.findall(text): + labels.setdefault(token, f"") + return json.loads(HASH.sub(lambda match: labels[match.group()], text)) + + +def listing(root: Path) -> list[str]: + entries = [] + for entry in sorted(root.rglob("*")): + relative = entry.relative_to(root).as_posix() + if entry.is_symlink(): + entries.append(f"{relative} -> {os.readlink(entry)}") + elif entry.is_file(): + entries.append(relative) + return entries + + +def file_tree(root: Path) -> dict[str, bytes | str]: + return { + entry.relative_to(root).as_posix(): ( + f"-> {os.readlink(entry)}" if entry.is_symlink() else entry.read_bytes() + ) + for entry in sorted(root.rglob("*")) + if entry.is_symlink() or entry.is_file() + } + + +def assert_stock_compose_complete(compose_path: Path) -> dict[str, Any]: + """Every service has an image, and every path a build reads resolves inside the project.""" + project_root = compose_path.parent.resolve() + project = json.loads(compose_path.read_text("utf-8")) + services = project["services"] + for name, service in services.items(): + assert service.get("image"), name + for volume in service.get("volumes", []): + if isinstance(volume, str) and volume.endswith(":/controller/tests:ro"): + tests = (project_root / volume.partition(":")[0]).resolve() + assert tests.is_relative_to(project_root) and tests.is_dir(), (name, tests) + build = service.get("build") + if build is None: + continue + build = {"context": build} if isinstance(build, str) else build + context = (project_root / build.get("context", ".")).resolve() + assert context.is_relative_to(project_root) and context.is_dir(), (name, context) + assert (context / build.get("dockerfile", "Dockerfile")).is_file(), name + for target in build.get("additional_contexts", {}).values(): + if target.startswith("service:"): + assert target.removeprefix("service:") in services, (name, target) + else: + named = (project_root / target).resolve() + assert named.is_relative_to(project_root) and named.is_dir(), (name, named) + return project + + +def observe(result: harbor.AdaptResult, dataset: Path, fake_docker: FakeDocker) -> Any: + def scrub(text: str) -> str: + return RESOLVE_DIR.sub("", text.replace(str(dataset), "")) + + projects: dict[str, Any] = {} + adapted = dataset / ".hud-adapt" + for context in sorted(adapted.iterdir()) if adapted.is_dir() else []: + projects[context.name] = { + "tree": listing(context), + "compose.yaml": assert_stock_compose_complete(context / "compose.yaml"), + "tasks.json": json.loads((context / "tasks.json").read_text("utf-8")), + } + docker = [ + { + "argv": scrub(call.command), + **( + { + "files": { + Path(path).name: json.loads(content) for path, content in call.files.items() + } + } + if call.files + else {} + ), + } + for call in fake_docker.calls + ] + return mask( + { + "failures": [ + [failure.task, finding.code, finding.kind, scrub(finding.message)] + for failure in result.failures + for finding in failure.findings + ], + "projects": projects, + "docker": docker, + } + ) + + +@pytest.mark.parametrize("case", sorted(path.name for path in CASES.iterdir())) +def test_adapting_a_harbor_dataset_writes_its_projects_and_reports_its_failures( + case: str, tmp_path: Path, fake_docker: FakeDocker, hud_env: HudEnv +) -> None: + dataset = load_case(case, tmp_path, fake_docker, hud_env) + + result = harbor.adapt(dataset, hud_requirement="hud") + + assert observe(result, dataset, fake_docker) == external_file( + CASES / case / "expected.json", format=".json" + ) + rows = list(result.taskset) + assert sorted(row.slug or "" for row in rows) == sorted( + row["slug"] + for context in (dataset / ".hud-adapt").glob("*") + for row in json.loads((context / "tasks.json").read_text("utf-8")) + ) + for row in rows: + assert row.runtime_config is not None and row.runtime_config.compose is not None + root = row.runtime_config.compose.root + assert isinstance(root, Path) + assert file_tree(root / "compose-project" / "environment") == file_tree( + dataset / (row.slug or "") / "environment" + ) + leaked = [ + path + for path in (dataset / ".hud-adapt").rglob("*") + if path.is_file() and not path.is_symlink() and SECRET.encode() in path.read_bytes() + ] + assert leaked == [] + + +def test_adapt_raises_when_the_source_holds_no_harbor_tasks(tmp_path: Path) -> None: + (tmp_path / "empty").mkdir() + + for source in (tmp_path / "empty", tmp_path / "nowhere"): + with pytest.raises(ValueError, match="no Harbor tasks found"): + harbor.adapt(source) + + +@pytest.mark.parametrize( + ("docker_state", "message"), + [ + ("daemon unreachable", "requires a reachable Docker daemon: docker version"), + ("no docker executable", "requires Docker to resolve authored images"), + ], +) +def test_adapt_aborts_without_a_docker_daemon_instead_of_failing_tasks( + docker_state: str, + message: str, + tmp_path: Path, + fake_docker: FakeDocker, + hud_env: HudEnv, +) -> None: + dataset = load_case("plain-image", tmp_path, fake_docker, hud_env) + if docker_state == "daemon unreachable": + fake_docker.on(r"^version", stderr="Cannot connect to the Docker daemon", exit=1) + else: + (tmp_path / "empty-path").mkdir() + hud_env.set(PATH=str(tmp_path / "empty-path")) + + with pytest.raises(RuntimeError, match=message): + harbor.adapt(dataset) + + +def edit(relative: str, content: str) -> Callable[[Path], str]: + def apply(task: Path) -> str: + (task / relative).write_text(content, encoding="utf-8") + return "hud" + + return apply + + +def wheel(task: Path) -> str: + built = task.parent.parent / "hud-0.0.0-py3-none-any.whl" + built.write_bytes(b"wheel") + return str(built) + + +@pytest.mark.parametrize( + ("change", "apply", "rebuilds"), + [ + ("an edited instruction", edit("instruction.md", "Second instruction\n"), False), + ("edited inline tests", edit("tests/test.sh", "#!/bin/sh\nexit 1\n"), False), + ("a new verifier timeout", edit("task.toml", "[verifier]\ntimeout_sec = 5\n"), False), + ("new content behind an environment symlink", edit("../outside.txt", "changed\n"), False), + ("an edited environment file", edit("environment/ignored.txt", "changed\n"), True), + ("another hud release", lambda task: "hud==0.0.1", True), + ("a hud wheel", wheel, True), + ], +) +def test_the_environment_image_identity_follows_only_what_the_image_contains( + change: str, + apply: Callable[[Path], str], + rebuilds: bool, + tmp_path: Path, + fake_docker: FakeDocker, + hud_env: HudEnv, +) -> None: + del change + dataset = load_case("plain-image", tmp_path, fake_docker, hud_env) + task = dataset / "task-a" + (dataset / "outside.txt").write_text("first\n", encoding="utf-8") + (task / "environment" / "outside").symlink_to(dataset / "outside.txt") + + def images(requirement: str) -> tuple[dict[str, Any], Path]: + (row,) = harbor.adapt(dataset, hud_requirement=requirement).taskset + assert row.runtime_config is not None and row.runtime_config.compose is not None + assert isinstance(row.runtime_config.compose.document, Path) + compose = json.loads(row.runtime_config.compose.document.read_text("utf-8")) + return compose["services"]["main"], row.runtime_config.compose.document.parent + + before, _ = images("hud") + after, project = images(apply(task)) + + assert (after["image"] != before["image"]) is rebuilds + requirement = after["build"]["args"]["HUD_REQUIREMENT"] + packages = sorted(path.name for path in (project / "compose-project/main/packages").iterdir()) + if requirement.endswith(".whl"): + assert (requirement, packages) == ( + "/controller/packages/hud-0.0.0-py3-none-any.whl", + ["hud-0.0.0-py3-none-any.whl"], + ) + else: + assert packages == [] diff --git a/tests/integrations/harbor/test_contract.py b/tests/integrations/harbor/test_contract.py deleted file mode 100644 index 32998c7e5..000000000 --- a/tests/integrations/harbor/test_contract.py +++ /dev/null @@ -1,1684 +0,0 @@ -"""Observable contracts for adapting Harbor tasks into HUD images.""" - -from __future__ import annotations - -import importlib -import json -import os -import subprocess -from pathlib import Path -from typing import Any - -import pytest - -from hud.environment import Mount, Peer -from hud.eval import RuntimeGPU, RuntimeLimits, RuntimeResources, RuntimeTPU, Taskset -from hud.eval.runtime.compose import ComposeConfig -from hud.integrations import harbor -from hud.integrations.harbor.config import ControllerConfig, TaskSpec - -from .conftest import make_harbor_task, make_multi_step_task - -adapt_module = importlib.import_module("hud.integrations.harbor.adapt") -build_module = importlib.import_module("hud.integrations.harbor.build") - - -@pytest.fixture(autouse=True) -def resolved_images(monkeypatch: pytest.MonkeyPatch) -> None: - def resolve( - source: Any, - compose_project: Any, - *, - verifier_image: str, - peer_services: set[str], - ) -> Any: - del compose_project, verifier_image - dockerfile = source.dockerfile.read_text("utf-8") if source.dockerfile.is_file() else "" - main: dict[str, Any] = { - "Env": [], - "User": "", - "WorkingDir": "/workspace", - "Entrypoint": [], - "ExposedPorts": {}, - } - if "USER 1000:2000" in dockerfile: - main["User"] = "1000:2000" - if "WORKDIR /workspace" not in dockerfile: - main["WorkingDir"] = "" - if "ENV IMAGE_ONLY=present VALUE_WITH_EQUALS=one=two" in dockerfile: - main["Env"] = ["IMAGE_ONLY=present", "VALUE_WITH_EQUALS=one=two"] - if 'ENTRYPOINT ["/usr/local/bin/start-environment"]' in dockerfile: - main["Entrypoint"] = ["/usr/local/bin/start-environment"] - if "EXPOSE 3128 3129 8765 8080" in dockerfile: - main["ExposedPorts"] = {f"{port}/tcp": {} for port in (3128, 3129, 8765, 8080)} - peers = { - service: { - "Env": [], - "User": "", - "WorkingDir": "", - "Entrypoint": [], - "ExposedPorts": {"6379/tcp": {}}, - } - for service in peer_services - } - return build_module.ResolvedImages(main=main, verifier=main, peers=peers) - - monkeypatch.setattr(adapt_module, "require_docker", lambda: None) - monkeypatch.setattr(adapt_module, "resolve_images", resolve) - - -def _adapt(path: Path, *, hud_requirement: str = "hud") -> Taskset: - result = harbor.adapt(path, hud_requirement=hud_requirement) - assert result.failures == () - return result.taskset - - -def _failure(path: Path) -> harbor.AdaptFailure: - result = harbor.adapt(path) - assert list(result.taskset) == [] - assert len(result.failures) == 1 - return result.failures[0] - - -def _tree_snapshot(root: Path) -> dict[str, tuple[str, bytes | str]]: - snapshot: dict[str, tuple[str, bytes | str]] = {} - for entry in sorted(root.rglob("*")): - relative = entry.relative_to(root).as_posix() - if entry.is_symlink(): - snapshot[relative] = ("symlink", os.readlink(entry)) - elif entry.is_file(): - snapshot[relative] = ("file", entry.read_bytes()) - else: - snapshot[relative] = ("directory", b"") - return snapshot - - -def _assert_stock_compose_complete(compose_path: Path) -> dict[str, Any]: - project = json.loads(compose_path.read_text("utf-8")) - services = project["services"] - assert isinstance(services, dict) - for name, service in services.items(): - assert isinstance(service, dict) - assert service.get("image"), name - for volume in service.get("volumes", []): - if not isinstance(volume, str): - continue - source, separator, target = volume.partition(":") - if separator and target == "/controller/tests:ro": - tests = (compose_path.parent / source).resolve() - tests.relative_to(compose_path.parent.resolve()) - assert tests.is_dir(), (name, tests) - build = service.get("build") - if build is None: - continue - build_config = {"context": build} if isinstance(build, str) else build - assert isinstance(build_config, dict) - context_value = build_config.get("context", ".") - assert isinstance(context_value, str) - context = (compose_path.parent / context_value).resolve() - context.relative_to(compose_path.parent.resolve()) - assert context.is_dir(), (name, context) - dockerfile_value = build_config.get("dockerfile", "Dockerfile") - assert isinstance(dockerfile_value, str) - assert (context / dockerfile_value).is_file(), (name, dockerfile_value) - additional_contexts = build_config.get("additional_contexts", {}) - assert isinstance(additional_contexts, dict) - for target in additional_contexts.values(): - assert isinstance(target, str) - if target.startswith("service:"): - assert target.removeprefix("service:") in services - else: - # Local additional-context paths resolve against the project - # directory (the compose file's directory), not the service - # build context. - named_context = (compose_path.parent / target).resolve() - named_context.relative_to(compose_path.parent.resolve()) - assert named_context.is_dir(), (name, named_context) - return project - - -def _environment_config(context: Path) -> dict[str, Any]: - paths = list((context / "compose-project").glob("*/config.json")) - assert len(paths) == 1 - config = json.loads(paths[0].read_text("utf-8")) - assert isinstance(config, dict) - return config - - -def test_image_resolution_builds_and_inspects_the_authored_image( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - calls: list[tuple[str, ...]] = [] - config = { - "Env": ["FROM_IMAGE=yes"], - "User": "1000:1000", - "WorkingDir": "/workspace", - "Entrypoint": ["/bin/start"], - "ExposedPorts": {"8080/tcp": {}}, - } - - overrides: list[dict[str, Any]] = [] - - def docker(*args: str, **_kwargs: Any) -> str: - calls.append(args) - if args[0] == "compose": - overrides.append(json.loads(Path(args[-3]).read_text("utf-8"))["services"]["main"]) - if args[:2] == ("image", "inspect"): - return json.dumps([{"Config": config}]) - return "" - - monkeypatch.setattr(build_module, "docker", docker) - - resolved = build_module.resolve_images( - source, - source.compose.with_project_directory("./environment"), - verifier_image=source.base_image, - peer_services=set(), - ) - - assert calls[0][0] == "compose" - assert calls[0][-2:] == ("build", "main") - assert overrides == [ - { - "image": source.base_image, - "build": {"context": ".", "dockerfile": "Dockerfile"}, - } - ] - assert calls[1] == ("image", "inspect", source.base_image) - assert resolved.main == config - assert resolved.verifier == config - assert resolved.peers == {} - - -def test_image_resolution_builds_compose_main_from_the_environment_dockerfile( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n image: authored/main:latest\n", - encoding="utf-8", - ) - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - built: list[dict[str, Any]] = [] - inspected: list[str] = [] - - def docker(*args: str, **_kwargs: Any) -> str: - if args[0] == "compose": - *_, override_path, operation, service = args - assert (operation, service) == ("build", "main") - built.append(json.loads(Path(override_path).read_text("utf-8"))["services"]["main"]) - return "" - assert args[:2] == ("image", "inspect") - inspected.append(args[2]) - return json.dumps([{"Config": {"Env": []}}]) - - monkeypatch.setattr(build_module, "docker", docker) - - build_module.resolve_images( - source, - source.compose, - verifier_image=source.base_image, - peer_services=set(), - ) - - assert built == [ - {"image": source.base_image, "build": {"context": ".", "dockerfile": "Dockerfile"}} - ] - assert inspected == [source.base_image] - - -def test_image_resolution_builds_the_interpolated_compose_document( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n build:\n context: ${CONTEXT_DIR}\n" - " image: ${MAIN_IMAGE_NAME}\n", - encoding="utf-8", - ) - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - documents: list[dict[str, Any]] = [] - - def docker(*args: str, **_kwargs: Any) -> str: - if args[0] == "compose": - document = args[args.index("--file") + 1] - documents.append(json.loads(Path(document).read_text("utf-8"))) - return "" - return json.dumps([{"Config": {"Env": []}}]) - - monkeypatch.setattr(build_module, "docker", docker) - - build_module.resolve_images( - source, - source.compose, - verifier_image=source.base_image, - peer_services=set(), - ) - - (document,) = documents - assert document["services"]["main"]["build"] == {"context": "."} - assert document["services"]["main"]["image"] == source.base_image - - -def test_separate_verifier_builds_for_the_task_image_platform( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "separate") - (task / "tests" / "Dockerfile").write_text("FROM python:3.12-slim\n", encoding="utf-8") - (task / "task.toml").write_text('[verifier]\nenvironment_mode = "separate"\n', encoding="utf-8") - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - builds: list[tuple[str, ...]] = [] - - def docker(*args: str, **_kwargs: Any) -> str: - if args[0] == "build": - builds.append(args) - if args[:3] == ("image", "inspect", "--format"): - return "linux/amd64" - if args[:2] == ("image", "inspect"): - return json.dumps([{"Config": {"Env": []}}]) - return "" - - monkeypatch.setattr(build_module, "docker", docker) - - build_module.resolve_images( - source, source.compose, verifier_image="verifier:test", peer_services=set() - ) - - (build,) = builds - assert build[build.index("--platform") + 1] == "linux/amd64" - assert build[build.index("--tag") + 1] == "verifier:test" - - -def test_separate_verifier_build_failure_names_the_platform_override( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "separate") - (task / "tests" / "Dockerfile").write_text("FROM python:3.12-slim\n", encoding="utf-8") - (task / "task.toml").write_text('[verifier]\nenvironment_mode = "separate"\n', encoding="utf-8") - source, findings = adapt_module._inspect_task(task) - assert findings == () - assert source is not None - - def docker(*args: str, **_kwargs: Any) -> str: - if args[0] == "build": - raise build_module.ImageResolutionError("docker build failed: no matching wheel") - if args[0] == "version": - return "linux/arm64" - if args[:3] == ("image", "inspect", "--format"): - return "linux/amd64" - if args[:2] == ("image", "inspect"): - return json.dumps([{"Config": {"Env": []}}]) - return "" - - monkeypatch.setattr(build_module, "docker", docker) - - with pytest.raises(build_module.ImageResolutionError, match="DOCKER_DEFAULT_PLATFORM"): - build_module.resolve_images( - source, source.compose, verifier_image="verifier:test", peer_services=set() - ) - - -def test_adapt_reports_tasks_whose_images_cannot_be_resolved( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - make_harbor_task(tmp_path, "broken", dockerfile="FROM registry.invalid/missing\n") - make_harbor_task(tmp_path, "working") - resolve = adapt_module.resolve_images - - def fail_broken(source: Any, *args: Any, **kwargs: Any) -> Any: - if source.path.name == "broken": - raise build_module.ImageResolutionError("docker build failed: pull access denied") - return resolve(source, *args, **kwargs) - - monkeypatch.setattr(adapt_module, "resolve_images", fail_broken) - - result = harbor.adapt(tmp_path) - - assert [task.slug for task in result.taskset] == ["working"] - (failure,) = result.failures - assert failure.task == "broken" - assert [(finding.code, finding.kind) for finding in failure.findings] == [ - ("harbor.invalid.image", "invalid") - ] - assert "pull access denied" in failure.findings[0].message - - -def test_adapt_runs_sidecars_that_expose_no_ports( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n worker:\n image: busybox\n", - encoding="utf-8", - ) - resolve = adapt_module.resolve_images - - def without_peer_ports(source: Any, *args: Any, **kwargs: Any) -> Any: - resolved = resolve(source, *args, **kwargs) - return build_module.ResolvedImages( - main=resolved.main, - verifier=resolved.verifier, - peers={name: {**config, "ExposedPorts": {}} for name, config in resolved.peers.items()}, - ) - - monkeypatch.setattr(adapt_module, "resolve_images", without_peer_ports) - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - assert isinstance(row.runtime_config.compose.document, Path) - project = json.loads(row.runtime_config.compose.document.read_text("utf-8")) - assert "worker" in project["services"] - (context,) = (tmp_path / ".hud-adapt").iterdir() - assert _environment_config(context)["peers"] == [] - - -def test_adapt_maps_the_agent_budget_to_the_agent_config(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - "[environment]\nbuild_timeout_sec = 900\n\n[agent]\ntimeout_sec = 28800\n", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.agent_config == {"timeout_seconds": 28800.0} - assert row.runtime_config is not None - assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=900) - - -def test_adapt_binds_the_compose_variables_harbor_defines(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - "[environment]\ncpus = 2\nmemory_mb = 4096\n", - encoding="utf-8", - ) - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - build: - context: ${CONTEXT_DIR} - image: ${MAIN_IMAGE_NAME} - deploy: - resources: - limits: - cpus: ${CPUS} - memory: ${MEMORY} -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - assert isinstance(row.runtime_config.compose.document, Path) - services = json.loads(row.runtime_config.compose.document.read_text("utf-8"))["services"] - assert services["main"]["deploy"]["resources"]["limits"] == {"cpus": "2", "memory": "4096M"} - assert services["hud-base"]["build"] == {"context": "./compose-project/environment"} - - -def test_adapt_aborts_when_the_docker_daemon_is_unreachable( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - make_harbor_task(tmp_path / "dataset", "task-a") - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - docker = bin_dir / "docker" - docker.write_text( - "#!/bin/sh\necho 'Cannot connect to the Docker daemon' >&2\nexit 1\n", - encoding="utf-8", - ) - docker.chmod(0o755) - monkeypatch.setenv("PATH", str(bin_dir)) - monkeypatch.setattr(adapt_module, "require_docker", build_module.require_docker) - - with pytest.raises(RuntimeError, match="requires a reachable Docker daemon"): - harbor.adapt(tmp_path / "dataset") - - -def test_adapt_packages_an_image_task_as_a_compose_project(tmp_path: Path) -> None: - task_dir = make_harbor_task(tmp_path, "task-a") - authored_environment = _tree_snapshot(task_dir / "environment") - - taskset = _adapt(tmp_path) - - (task,) = list(taskset) - assert task.id == "run" - assert task.slug == "task-a" - assert task.args["instruction"] == "Solve the task." - assert task.args["task"] == { - "artifacts": [], - "collect": [], - "description": "", - "id": "task-a", - "separate_verifier": False, - "verifier_timeout": 120.0, - } - assert task.runtime_config is not None - assert task.runtime_config.image is None - - (context,) = (tmp_path / ".hud-adapt").iterdir() - assert not (context / "Dockerfile").exists() - assert not any(path.name == ".hud" for path in context.rglob(".hud")) - assert {entry.name for entry in context.iterdir()} == { - "compose-project", - "compose.yaml", - "env.py", - "tasks.json", - } - # `hud deploy` resolves the context's identity from a literal Environment(...). - assert f"Environment({context.name!r})" in (context / "env.py").read_text("utf-8") - project_root = context / "compose-project" - assert _tree_snapshot(project_root / "environment") == authored_environment - payload = project_root / "main" - assert {entry.name for entry in payload.iterdir()} == { - "Dockerfile", - "config.json", - "install.sh", - "packages", - } - config = json.loads((payload / "config.json").read_text("utf-8")) - assert "controller_root" not in config - assert "rootfs" not in config - assert "runtime_root" not in config - assert config["mounts"] == [] - assert not (project_root / "build.sh").exists() - assert (project_root / "tests" / "task-a" / "test.sh").is_file() - assert not any(path.name in {"tasks", "tasks.json"} for path in payload.rglob("*")) - assert not (project_root / "compose.json").exists() - assert task.runtime_config.compose is not None - assert task.runtime_config.compose.document == context / "compose.yaml" - assert task.runtime_config.compose.root == context - project = _assert_stock_compose_complete(context / "compose.yaml") - assert set(project["services"]) == {"main", "hud-base"} - assert project["services"]["hud-base"]["build"] == {"context": "./compose-project/environment"} - assert project["services"]["hud-base"]["scale"] == 0 - main = project["services"]["main"] - assert main["image"].startswith("hud-harbor:") - assert main["build"] == { - "additional_contexts": {"hud-base": "service:hud-base"}, - "args": { - "BASE_IMAGE": "hud-base", - "HUD_REQUIREMENT": "hud", - "VERIFIER_IMAGE": project["services"]["hud-base"]["image"], - }, - "context": "./compose-project/main", - "target": "plain", - } - assert "HUD_RUNTIME_ROOT" not in main.get("environment", {}) - assert main["volumes"] == ["./compose-project/tests:/controller/tests:ro"] - assert not (project_root / "Dockerfile").exists() - - -def test_task_content_changes_do_not_rebuild_the_environment(tmp_path: Path) -> None: - task_dir = make_harbor_task(tmp_path, "task-a", instruction="First instruction") - - (before,) = list(_adapt(tmp_path)) - assert before.runtime_config is not None - assert before.runtime_config.compose is not None - assert isinstance(before.runtime_config.compose.document, Path) - before_compose = json.loads(before.runtime_config.compose.document.read_text("utf-8")) - before_image = before_compose["services"]["main"]["image"] - - (task_dir / "instruction.md").write_text("Second instruction", encoding="utf-8") - (task_dir / "tests" / "test.sh").write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") - (after,) = list(_adapt(tmp_path)) - assert after.runtime_config is not None - assert after.runtime_config.compose is not None - assert isinstance(after.runtime_config.compose.document, Path) - after_compose = json.loads(after.runtime_config.compose.document.read_text("utf-8")) - - assert after_compose["services"]["main"]["image"] == before_image - assert after.args["instruction"] == "Second instruction" - (other_release,) = list(_adapt(tmp_path, hud_requirement="hud==0.0.1")) - assert other_release.runtime_config is not None - assert other_release.runtime_config.compose is not None - assert isinstance(other_release.runtime_config.compose.document, Path) - other_compose = json.loads(other_release.runtime_config.compose.document.read_text("utf-8")) - assert other_compose["services"]["main"]["image"] != before_image - assert ( - after.runtime_config.compose.document.parent - / "compose-project" - / "tests" - / "task-a" - / "test.sh" - ).read_text("utf-8") == "#!/bin/sh\nexit 1\n" - - -def test_image_task_keeps_non_recipe_compose_names_as_context_files(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - content = '{"api_gateway": {"interval": "30s"}}\n' - (task / "environment" / "docker-compose.yml").write_text(content, encoding="utf-8") - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - environment = context / "compose-project" / "environment" - assert (environment / "docker-compose.yml").read_text("utf-8") == content - project = json.loads((context / "compose.yaml").read_text("utf-8")) - assert set(project["services"]) == {"main", "hud-base"} - - -def test_image_task_preserves_environment_ignored_paths_verbatim( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - environment = task / "environment" - (environment / ".dockerignore").write_text("hud/\nDockerfile\n", encoding="utf-8") - (environment / "ignored.txt").write_bytes(b"unchanged\x00payload") - authored = _tree_snapshot(environment) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - project = context / "compose-project" - assert _tree_snapshot(project / "environment") == authored - assert (project / "main" / "Dockerfile").is_file() - - -def test_adapt_honors_compose_main_build_settings( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a", dockerfile=None) - environment = task / "environment" - environment.mkdir() - (environment / "docker-compose.yaml").write_text( - """\ -services: - main: - build: - context: . - dockerfile: Containerfile - args: - FLAVOR: compose -""", - encoding="utf-8", - ) - (environment / "Containerfile").write_text("FROM python:3.12\n", encoding="utf-8") - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - compose_path = row.runtime_config.compose.document - assert isinstance(compose_path, Path) - project = _assert_stock_compose_complete(compose_path) - base = project["services"]["hud-base"] - assert base["build"]["dockerfile"] == "Containerfile" - assert base["build"]["args"] == {"FLAVOR": "compose"} - assert base["scale"] == 0 - - -def test_adapt_emits_compose_project_and_peers( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - environment: {FROM_COMPOSE: "yes"} - expose: [8080] - healthcheck: - test: [CMD-SHELL, curl -f http://localhost:8080/health] - interval: 2s - timeout: 3s - retries: 5 - start_period: 1s - redis: - image: redis:7-alpine - depends_on: - main: - condition: service_healthy - restart: true - command: [redis-server, --save, ""] - environment: {SIDE: car} - expose: [6379] - healthcheck: - test: [CMD, redis-cli, ping] - interval: 2s - timeout: 3s - retries: 5 - start_period: 1s -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.image is None - assert row.runtime_config.compose is not None - assert row.runtime_config.compose.service_access is None - (context,) = (tmp_path / ".hud-adapt").iterdir() - assert row.runtime_config.compose.document == context / "compose.yaml" - assert row.runtime_config.compose.root == context - assert not (context / "compose-project" / "compose.json").exists() - project = _assert_stock_compose_complete(context / "compose.yaml") - assert project["services"]["redis"]["image"] == "redis:7-alpine" - assert "build" not in project["services"]["redis"] - assert project["services"]["main"]["build"]["context"] == "./compose-project/main" - assert project["services"]["main"]["build"]["target"] == "plain" - assert project["services"]["main"]["build"]["additional_contexts"] == { - "hud-base": "service:hud-base" - } - assert project["services"]["main"]["image"].startswith("hud-harbor:") - assert project["services"]["hud-base"]["image"].startswith("hud-harbor-base:") - assert project["services"]["hud-base"]["build"]["context"] == "./compose-project/environment" - assert (context / "compose-project" / "environment" / "Dockerfile").is_file() - assert not any(path.name == ".hud" for path in context.rglob(".hud")) - redis = project["services"]["redis"] - assert redis["image"] == "redis:7-alpine" - assert redis["environment"] == {"SIDE": "car"} - assert redis["command"] == ["redis-server", "--save", ""] - assert redis["depends_on"] == {"main": {"condition": "service_started", "restart": True}} - assert redis["expose"] == ["6379"] - assert redis["healthcheck"]["test"] == ["CMD", "redis-cli", "ping"] - assert "build" not in redis - manifest = _environment_config(context) - assert manifest["environment"]["env"] == {"FROM_COMPOSE": "yes"} - assert manifest["environment"]["healthcheck"] == { - "command": "curl -f http://localhost:8080/health", - "interval_sec": 2.0, - "timeout_sec": 3.0, - "start_period_sec": 1.0, - "start_interval_sec": 5.0, - "retries": 5, - } - assert manifest["local_aliases"] == ["main"] - assert manifest["ports"] == [8080] - assert manifest["capabilities"] == [] - assert ControllerConfig.model_validate(manifest).peers == [ - Peer("redis", 6379, target=("redis", 6379)) - ] - assert ComposeConfig.from_file(context / "compose.yaml").healthchecked_services() == ["redis"] - assert project["services"]["main"]["command"] == [ - "/controller/venv/bin/hud", - "serve", - "hud.integrations.harbor.env:env", - "--host", - "0.0.0.0", - "--port", - "8765", - ] - assert "healthcheck" not in project["services"]["main"] - - -def test_compose_adapt_retains_source_builds_in_the_generated_project( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "Dockerfile").write_text( - 'FROM python:3.12\nWORKDIR /app\nENTRYPOINT ["/app/start"]\n', - encoding="utf-8", - ) - database = task / "environment" / "database" - database.mkdir() - (database / "Dockerfile").write_text("FROM postgres:16\n", encoding="utf-8") - (database / "db.env").write_text("POSTGRES_DB=test\n", encoding="utf-8") - (database / "data").mkdir() - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - env_file: ./main.env - volumes: - - ./main-data:/var/lib/main - - type: bind - source: ./readonly - target: /etc/authored - read_only: true - database: - build: - context: ./database - env_file: ./database/db.env - volumes: [./database/data:/var/lib/postgresql/data] - expose: [5432] - redis: - image: redis:7-alpine - expose: [6379] -""", - encoding="utf-8", - ) - (task / "environment" / "main.env").write_text("MAIN=true\n", encoding="utf-8") - (task / "environment" / "main-data").mkdir() - (task / "environment" / "readonly").mkdir() - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - compose_path = row.runtime_config.compose.document - assert isinstance(compose_path, Path) - project = json.loads(compose_path.read_text("utf-8")) - assert project["services"]["database"]["build"]["context"] == ( - "./compose-project/environment/database" - ) - assert project["services"]["database"]["image"].startswith("hud-harbor-sidecar:") - assert ( - project["services"]["database"]["env_file"] - == "./compose-project/environment/database/db.env" - ) - assert project["services"]["database"]["volumes"] == [ - "./compose-project/environment/database/data:/var/lib/postgresql/data" - ] - assert project["services"]["main"]["env_file"] == "./compose-project/environment/main.env" - assert ( - "./compose-project/environment/main-data:/mounts/0" - in (project["services"]["main"]["volumes"]) - ) - assert { - "type": "bind", - "source": "./compose-project/environment/readonly", - "target": "/mounts/1", - "read_only": True, - } in project["services"]["main"]["volumes"] - context = row.runtime_config.compose.root - assert isinstance(context, Path) - manifest = _environment_config(context) - assert ControllerConfig.model_validate(manifest).mounts == [ - Mount("rw", src="/mounts/0", dst="/var/lib/main"), - Mount("ro", src="/mounts/1", dst="/etc/authored"), - ] - assert project["services"]["redis"]["image"] == "redis:7-alpine" - assert project["services"]["main"]["build"]["additional_contexts"] == { - "hud-base": "service:hud-base" - } - - -def test_adapt_moves_compose_main_process_settings_into_the_workspace( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - user: 1001:1002 - entrypoint: [/compose-init] - working_dir: /compose-work -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - compose = _assert_stock_compose_complete(context / "compose.yaml") - assert manifest["image_user"] == "1001:1002" - assert manifest["entrypoint"] == ["/compose-init"] - assert manifest["workdir"] == "/compose-work" - assert "user" not in compose["services"]["main"] - assert compose["services"]["main"]["entrypoint"] == [] - - -def test_adapt_moves_compose_main_healthcheck_into_the_workspace( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - healthcheck: - test: [CMD-SHELL, curl -f http://localhost:8080/health] - interval: 2s - timeout: 3s - retries: 5 - start_period: 1s -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - compose = json.loads((context / "compose.yaml").read_text("utf-8")) - assert manifest["environment"]["healthcheck"] == { - "command": "curl -f http://localhost:8080/health", - "interval_sec": 2.0, - "timeout_sec": 3.0, - "start_period_sec": 1.0, - "start_interval_sec": 5.0, - "retries": 5, - } - assert "healthcheck" not in compose["services"]["main"] - - -def test_adapt_uses_compose_healthcheck_defaults(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n healthcheck:\n test: [CMD, 'true']\n", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["environment"]["healthcheck"] == { - "command": "true", - "interval_sec": 30.0, - "timeout_sec": 30.0, - "start_period_sec": 0.0, - "start_interval_sec": 5.0, - "retries": 3, - } - - -def test_adapt_merges_implicit_main_into_authored_compose(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n default:\n image: sidecar:latest\n expose: [1234]\n", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - compose = json.loads((context / "compose.yaml").read_text("utf-8")) - assert {"main", "default"} <= compose["services"].keys() - assert ControllerConfig.model_validate(_environment_config(context)).peers == [ - Peer("default", 1234, target=("default", 1234)) - ] - - -def test_only_network_mcp_servers_become_capabilities( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main: {}\n redis:\n image: redis:7-alpine\n expose: [6379]\n", - encoding="utf-8", - ) - (task / "task.toml").write_text( - """ -[environment] -skills_dir = "/skills" - -[[environment.mcp_servers]] -name = "redis-tools" -transport = "streamable-http" -url = "http://redis:6379/mcp" -args = [] - -[[environment.mcp_servers]] -name = "db" -transport = "stdio" -command = "db-mcp" -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["capabilities"] == [ - { - "name": "redis-tools", - "params": {"transport": "streamable-http"}, - "protocol": "mcp/2025-11-25", - "url": "http://redis:6379/mcp", - } - ] - - -@pytest.mark.parametrize("name", ["shell", "filetracking"]) -def test_mcp_server_names_cannot_shadow_workspace_capabilities( - tmp_path: Path, - name: str, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - f"""\ -[[environment.mcp_servers]] -name = "{name}" -transport = "streamable-http" -url = "http://server:8000/mcp" -""", - encoding="utf-8", - ) - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == ["harbor.invalid.reserved_mcp_name"] - assert name in failure.findings[0].message - - -def test_adapt_groups_identical_images_and_keeps_row_metadata( - dataset_same_env: Path, -) -> None: - (dataset_same_env / "build-pmars" / "task.toml").write_text( - """\ -artifacts = ["/tmp/result"] - -[metadata] -category = "systems" -difficulty = "medium" -tags = ["bash", "linux"] - -[agent] -timeout_sec = 45 - -[verifier] -timeout_sec = 30 -""", - encoding="utf-8", - ) - taskset = _adapt(dataset_same_env) - - assert len(taskset) == 3 - assert len(taskset.environment_names()) == 1 - assert all( - task.columns - == { - "category": "systems", - "difficulty": "medium", - "tags": ["bash", "linux"], - } - for task in taskset - ) - assert all(task.runtime_config is not None for task in taskset) - assert all(task.runtime_config.compose is not None for task in taskset if task.runtime_config) - assert taskset["build-pmars"].agent_config == {"timeout_seconds": 45.0} - assert taskset["build-pmars"].args["task"]["verifier_timeout"] == 30.0 - assert taskset["build-pmars"].args["task"]["artifacts"] == [ - {"service": "main", "source": "/tmp/result"} - ] - - -def test_distinct_environments_build_distinct_images( - dataset_multi_env: Path, -) -> None: - taskset = _adapt(dataset_multi_env) - - assert len(taskset.environment_names()) == 2 - assert all(task.runtime_config is not None for task in taskset) - assert all(task.runtime_config.compose is not None for task in taskset if task.runtime_config) - - -def test_adapt_maps_resources_onto_the_compose_runtime(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "gpu") - (task / "task.toml").write_text( - """ -[metadata] -difficulty = "hard" - -[environment] -cpus = 4 -memory_mb = 8192 -storage_mb = 32768 -gpus = 2 -gpu_types = ["H100"] -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.columns == {"difficulty": "hard"} - assert row.runtime_config is not None - assert row.runtime_config.image is None - assert row.runtime_config.compose is not None - assert isinstance(row.runtime_config.compose.document, Path) - assert row.runtime_config.resources is not None - assert row.runtime_config.resources.cpu == 4 - assert row.runtime_config.resources.memory_mb == 8192 - assert row.runtime_config.resources.storage_mb == 32768 - assert row.runtime_config.resources.gpu is not None - assert row.runtime_config.resources.gpu.count == 2 - assert row.runtime_config.resources.gpu.acceptable_types == ["H100"] - - -def test_env_templates_are_persisted_verbatim_not_resolved( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Host values must never be baked into the content-hashed image payload - or the persisted task rows; ``${VAR}`` templates resolve at runtime.""" - monkeypatch.setenv("HARBOR_JUDGE_KEY", "sk-live-secret") - task = make_harbor_task(tmp_path, "templated") - (task / "task.toml").write_text( - """ -[environment.env] -JUDGE_KEY = "${HARBOR_JUDGE_KEY}" -MODEL = "${HARBOR_JUDGE_MODEL:-gpt-4o}" - -[verifier] -timeout_sec = 60 - -[verifier.env] -VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" -""", - encoding="utf-8", - ) - - harbor.adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["environment"]["env"] == { - "JUDGE_KEY": "${HARBOR_JUDGE_KEY}", - "MODEL": "${HARBOR_JUDGE_MODEL:-gpt-4o}", - } - assert manifest["verifier"]["env"] == {"VERIFIER_KEY": "${HARBOR_JUDGE_KEY}"} - for persisted in sorted(path for path in context.rglob("*") if path.is_file()): - assert b"sk-live-secret" not in persisted.read_bytes(), persisted - - -def test_prebuilt_harbor_image_emits_a_conventional_wrapper_build( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "prebuilt", dockerfile=None) - (task / "task.toml").write_text( - '[environment]\ndocker_image = "registry.example/base:latest"\n', - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - project = context / "compose-project" - services = _assert_stock_compose_complete(context / "compose.yaml")["services"] - assert set(services) == {"main"} - assert services["main"]["build"]["args"]["BASE_IMAGE"] == "registry.example/base:latest" - assert "additional_contexts" not in services["main"]["build"] - assert not (project / "build.sh").exists() - - -def test_zero_gpus_is_a_valid_harbor_resource_declaration( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "cpu-only") - (task / "task.toml").write_text("[environment]\ngpus = 0\n", encoding="utf-8") - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.resources is None - - -def test_runtime_configuration_is_data_not_dockerfile_codegen( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - """ -[environment] -workdir = "/app" -network_mode = "allowlist" -allowed_hosts = ["pypi.org"] - -[environment.env] -SHARED = "yes" - -[environment.healthcheck] -command = "curl -f http://localhost:8080/health" -interval_sec = 2 -timeout_sec = 4 -start_period_sec = 6 -start_interval_sec = 1 -retries = 5 - -[agent] -user = "agent" - -[agent.env] -AGENT_ONLY = "yes" - -[verifier] -user = 0 -network_mode = "no-network" - -[verifier.env] -VERIFIER_ONLY = "yes" -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["workdir"] == "/app" - assert manifest["environment"] == { - "env": {"SHARED": "yes"}, - "network": {"enabled": True, "allowed_hosts": ["pypi.org"]}, - "healthcheck": { - "command": "curl -f http://localhost:8080/health", - "interval_sec": 2.0, - "timeout_sec": 4.0, - "start_period_sec": 6.0, - "start_interval_sec": 1.0, - "retries": 5, - }, - } - assert manifest["agent"]["user"] == "agent" - assert manifest["agent"]["env"] == {"AGENT_ONLY": "yes"} - assert manifest["agent"]["network"] == {"enabled": True, "allowed_hosts": ["pypi.org"]} - assert manifest["verifier"]["user"] == 0 - assert manifest["verifier"]["network"] == {"enabled": False, "allowed_hosts": []} - assert manifest["verifier"]["env"] == {"VERIFIER_ONLY": "yes"} - - -def test_image_entrypoint_is_preserved_as_runtime_data( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "Dockerfile").write_text( - """\ -FROM python:3.12 -USER 1000:2000 -WORKDIR /workspace -ENV IMAGE_ONLY=present VALUE_WITH_EQUALS=one=two -ENTRYPOINT ["/usr/local/bin/start-environment"] -CMD ["ignored-by-harbor"] -""", - encoding="utf-8", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["image_env"] == { - "IMAGE_ONLY": "present", - "VALUE_WITH_EQUALS": "one=two", - } - assert manifest["image_user"] == "1000:2000" - assert manifest["agent"]["user"] == "1000:2000" - assert manifest["verifier"]["user"] == "1000:2000" - assert manifest["workdir"] == "/workspace" - assert manifest["entrypoint"] == ["/usr/local/bin/start-environment"] - - -def test_dataset_adaptation_returns_successes_and_all_detectable_findings( - tmp_path: Path, -) -> None: - make_harbor_task(tmp_path, "supported") - unsupported = make_harbor_task(tmp_path, "unsupported") - (unsupported / "instruction.md").unlink() - (unsupported / "task.toml").write_text( - """\ -[environment] -os = "windows" - -[[environment.mcp_servers]] -name = "shell" -transport = "streamable-http" -""", - encoding="utf-8", - ) - (unsupported / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n image: ${HOST_ONLY_IMAGE}\n", - encoding="utf-8", - ) - - result = harbor.adapt(tmp_path) - - assert [task.slug for task in result.taskset] == ["supported"] - assert len(result.failures) == 1 - failure = result.failures[0] - assert failure.task == "unsupported" - assert {finding.code for finding in failure.findings} == { - "harbor.unsupported.host_compose_variable", - "harbor.invalid.reserved_mcp_name", - "harbor.invalid.mcp_url", - "harbor.invalid.missing_instruction", - } - assert {finding.kind for finding in failure.findings} == {"contract", "invalid"} - - -def test_unbound_compose_variables_are_deliberate_contract_refusals(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main:\n image: ${MAIN_IMAGE}\n", - encoding="utf-8", - ) - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == [ - "harbor.unsupported.host_compose_variable" - ] - assert failure.findings[0].kind == "contract" - - -@pytest.mark.parametrize("port", [3128, 3129, 8765]) -def test_adapt_rejects_main_ports_reserved_by_hud( - tmp_path: Path, - port: int, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - f"services:\n main:\n expose: [{port}]\n", - encoding="utf-8", - ) - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == ["harbor.invalid.reserved_main_port"] - assert str(port) in failure.findings[0].message - - -def test_adapt_does_not_forward_reserved_ports_exposed_by_the_image(tmp_path: Path) -> None: - make_harbor_task( - tmp_path, - "task-a", - dockerfile="FROM python:3.11-slim\nWORKDIR /workspace\nEXPOSE 3128 3129 8765 8080\n", - ) - - _adapt(tmp_path) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - assert _environment_config(context)["ports"] == [8080] - - -def test_adapt_accepts_explicit_shared_verifier_mode(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "shared") - (task / "task.toml").write_text( - '[verifier]\nenvironment_mode = "shared"\n', - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.verifier is None - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - assert row.runtime_config.compose.service_access is None - - -def test_adapt_builds_a_separate_verifier_with_its_own_placement( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "separate") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main: {}\n redis:\n image: redis:7-alpine\n expose: [6379]\n", - encoding="utf-8", - ) - (task / "tests" / "Dockerfile").write_text( - "FROM python:3.12-alpine\nCOPY . /tests\n", - encoding="utf-8", - ) - (task / "task.toml").write_text( - """ -artifacts = ["/tmp/agent.patch"] - -[environment] -cpus = 2 -memory_mb = 2048 -build_timeout_sec = 600.5 -gpus = 1 -gpu_types = ["H100", "A100"] -os = "windows" -tpu = {type = "v5", topology = "2x2"} - -[verifier] -environment_mode = "separate" -timeout_sec = 30 - -[verifier.environment] -cpus = 4 -memory_mb = 1024 -build_timeout_sec = 1200 -gpus = 1 -gpu_types = ["T4"] -workdir = "/judge" -network_mode = "allowlist" -allowed_hosts = ["verifier.example"] - -[verifier.environment.env] -NESTED_ONLY = "yes" -SHARED = "nested" - -[verifier.env] -SHARED = "phase" - -[[verifier.collect]] -service = "redis" -command = "redis-cli save" -timeout_sec = 10 -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.id == "run" - assert row.slug == "separate" - assert row.runtime_config is not None - assert row.runtime_config.resources == RuntimeResources( - cpu=2, - memory_mb=2048, - gpu=RuntimeGPU(type=["H100", "A100"]), - os="windows", - tpu=RuntimeTPU(type="v5", topology="2x2"), - ) - assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=601) - assert row.runtime_config.compose is not None - assert row.runtime_config.compose.service_access is True - assert row.verifier is not None - assert row.verifier.runtime_config is not None - assert row.verifier.runtime_config.compose is not None - assert row.verifier.runtime_config.compose.document == row.runtime_config.compose.document - assert row.verifier.runtime_config.compose.root == row.runtime_config.compose.root - assert row.verifier.runtime_config.resources == RuntimeResources( - cpu=4, - memory_mb=1024, - gpu=RuntimeGPU(type=["T4"]), - ) - assert row.verifier.runtime_config.limits == RuntimeLimits(startup_timeout_s=1200) - - (context,) = (tmp_path / ".hud-adapt").iterdir() - manifest = _environment_config(context) - assert manifest["verifier_root"] == "/verifier" - assert manifest["verifier_image"]["workdir"] == "/judge" - assert manifest["verifier"] == { - "user": None, - "network": {"enabled": True, "allowed_hosts": ["verifier.example"]}, - "env": {"NESTED_ONLY": "yes", "SHARED": "phase"}, - } - assert "tasks" not in manifest - assert row.args["task"] == { - "artifacts": [{"service": "main", "source": "/tmp/agent.patch"}], - "collect": [{"command": "redis-cli save", "service": "redis", "timeout_sec": 10.0}], - "description": "", - "id": "separate", - "separate_verifier": True, - "verifier_timeout": 30.0, - } - controller = ControllerConfig.model_validate(manifest) - assert controller.verifier.network.allowed_hosts == ["verifier.example"] - spec = TaskSpec.model_validate(row.args["task"]) - assert [hook.service for hook in spec.collect] == ["redis"] - assert [artifact.source for artifact in spec.artifacts] == ["/tmp/agent.patch"] - assert not (context / "compose-project" / "main" / "tasks").exists() - compose = json.loads((context / "compose.yaml").read_text("utf-8")) - assert compose["services"]["main"].get("volumes", []) == [] - assert compose["services"]["main"]["build"]["target"] == "verifier" - assert compose["services"]["main"]["build"]["additional_contexts"] == { - "hud-base": "service:hud-base", - "hud-verifier": "service:hud-verifier", - } - assert compose["services"]["hud-verifier"]["scale"] == 0 - assert compose["services"]["hud-verifier"]["image"].startswith("hud-harbor-verifier:") - - -def test_separate_verifier_image_task_builds_the_verifier_target( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "separate-image") - (task / "tests" / "Dockerfile").write_text( - "FROM python:3.12-alpine\nCOPY . /tests\n", - encoding="utf-8", - ) - (task / "task.toml").write_text( - """ -[environment] -build_timeout_sec = 300 -cpus = 4 -memory_mb = 14336 -gpus = 1 -gpu_types = ["H100"] - -[verifier] -environment_mode = "separate" - -[verifier.environment] -build_timeout_sec = 300 -cpus = 4 -memory_mb = 14336 -gpus = 1 -gpu_types = ["H100"] -""", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - - assert row.runtime_config is not None - assert row.runtime_config.limits == RuntimeLimits(startup_timeout_s=300) - assert row.runtime_config.compose is not None - assert row.runtime_config.compose.service_access is None - assert row.verifier is not None - assert row.verifier.runtime_config is None - assert isinstance(row.runtime_config.compose.document, Path) - project = _assert_stock_compose_complete(row.runtime_config.compose.document) - assert set(project["services"]) == {"main", "hud-base", "hud-verifier"} - build = project["services"]["main"]["build"] - assert build["target"] == "verifier" - assert build["additional_contexts"] == { - "hud-base": "service:hud-base", - "hud-verifier": "service:hud-verifier", - } - assert project["services"]["hud-base"]["scale"] == 0 - assert project["services"]["hud-verifier"]["scale"] == 0 - - -def test_separate_verifier_groups_have_distinct_environment_names( - tmp_path: Path, -) -> None: - declaration = '[verifier]\nenvironment_mode = "separate"\n' - compose = "services:\n main: {}\n redis:\n image: redis:7-alpine\n expose: [6379]\n" - verifier = "FROM python:3.12-alpine\nCOPY . /tests\n" - for name in ("task-a", "task-b"): - task = make_harbor_task(tmp_path, name) - (task / "task.toml").write_text(declaration, encoding="utf-8") - (task / "environment" / "docker-compose.yaml").write_text(compose, encoding="utf-8") - (task / "tests" / "Dockerfile").write_text(verifier, encoding="utf-8") - - rows = list(_adapt(tmp_path)) - - assert len({row.env for row in rows}) == 2 - compose_paths = { - compose - for row in rows - if row.runtime_config is not None - and row.runtime_config.compose is not None - and isinstance((compose := row.runtime_config.compose.document), Path) - } - assert len(compose_paths) == 2 - assert all(path.is_file() for path in compose_paths) - task_files = [ - json.loads((path.parent / "tasks.json").read_text("utf-8")) for path in compose_paths - ] - assert {rows[0]["slug"] for rows in task_files} == {"task-a", "task-b"} - assert {rows[0]["args"]["task"]["id"] for rows in task_files} == {"task-a", "task-b"} - - -def test_multi_step_tasks_are_refused_directly(tmp_path: Path) -> None: - make_multi_step_task(tmp_path, "multi") - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == ["harbor.unsupported.multi_step"] - - -@pytest.mark.parametrize( - "artifact", - [ - '"/"', - '"//"', - '"/workspace/../secret"', - '{ source = "/output", destination = "/tmp/out" }', - '{ source = "/output", destination = "../out" }', - '{ source = "/output", destination = "a\\\\b" }', - '{ source = "/output", destination = "manifest.json" }', - ], -) -def test_artifact_paths_stay_beneath_their_roots(tmp_path: Path, artifact: str) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - f"artifacts = [{artifact}]\n", - encoding="utf-8", - ) - - failure = _failure(tmp_path) - - assert [finding.code for finding in failure.findings] == ["harbor.invalid.task_config"] - - -def test_artifacts_keep_harbor_destination_and_exclude(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text( - """\ -[[artifacts]] -source = "/app/outputs" -destination = "results/outputs" -exclude = ["*.tmp", "cache"] -""", - encoding="utf-8", - ) - - taskset = _adapt(tmp_path) - - assert taskset["task-a"].args["task"]["artifacts"] == [ - { - "service": "main", - "source": "/app/outputs", - "destination": "results/outputs", - "exclude": ["*.tmp", "cache"], - } - ] - - -def test_agent_timeout_becomes_per_task_agent_policy( - tmp_path: Path, -) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "task.toml").write_text("[agent]\ntimeout_sec = 60\n", encoding="utf-8") - - taskset = _adapt(tmp_path) - - (row,) = list(taskset) - assert row.agent_config == {"timeout_seconds": 60.0} - - -def test_task_symlinks_are_copied_without_reading_host_files( - tmp_path: Path, -) -> None: - outside = tmp_path / "outside.txt" - outside.write_text("host secret", encoding="utf-8") - task = make_harbor_task(tmp_path / "dataset", "task-a") - (task / "tests" / "link").symlink_to(outside) - - _adapt(task.parent) - - (context,) = (task.parent / ".hud-adapt").iterdir() - copied = context / "compose-project" / "tests" / "task-a" / "link" - assert copied.is_symlink() - assert os.readlink(copied) == str(outside) - - -def test_adapt_hashes_links_not_their_targets( - tmp_path: Path, -) -> None: - outside = tmp_path / "outside.txt" - outside.write_text("first", encoding="utf-8") - task = make_harbor_task(tmp_path / "dataset", "task-a") - (task / "environment" / "link").symlink_to(outside) - - (before,) = list(_adapt(task.parent)) - outside.write_text("changed", encoding="utf-8") - (after,) = list(_adapt(task.parent)) - - assert before.runtime_config == after.runtime_config - - -def test_authored_runtime_assets_are_valid_source() -> None: - integration = Path(harbor.__file__).parent - compile((integration / "env.py").read_text("utf-8"), "env.py", "exec") - installer = (integration / "install.sh").read_text("utf-8") - assert "python_version=3.12" in installer - assert "sys.version_info[:2] < (3, 13)" in installer - assert 'uv python install "$python_version"' in installer - assert 'python="$root/bin/python$python_version"' in installer - assert 'uv venv "$root/venv" --python "$python"' in installer - assert "dnf install -y bubblewrap" in installer - result = subprocess.run( - ["sh", "-n", integration / "install.sh"], - check=False, - capture_output=True, - ) - assert result.returncode == 0, result.stderr.decode() - - -def test_public_surface_exposes_results_and_the_two_real_operations() -> None: - assert harbor.__all__ == [ - "AdaptFailure", - "AdaptFinding", - "AdaptResult", - "adapt", - "export", - ] - - -def test_portless_sidecar_ports_are_resolved_from_its_built_image(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n main: {}\n redis:\n image: redis:7-alpine\n", - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - context = row.runtime_config.compose.root - assert isinstance(context, Path) - manifest = _environment_config(context) - assert ControllerConfig.model_validate(manifest).peers == [ - Peer("redis", 6379, target=("redis", 6379)) - ] - assert "peer_image_configs" not in manifest - - -def test_completed_compose_dependencies_are_not_routed_as_peers(tmp_path: Path) -> None: - task = make_harbor_task(tmp_path, "task-a") - (task / "environment" / "docker-compose.yaml").write_text( - "services:\n" - " main:\n" - " depends_on:\n" - " seed:\n" - " condition: service_completed_successfully\n" - " seed:\n" - " build: ./seed\n", - encoding="utf-8", - ) - (task / "environment" / "seed").mkdir() - (task / "environment" / "seed" / "Dockerfile").write_text( - 'FROM busybox:1.37\nCMD ["true"]\n', - encoding="utf-8", - ) - - (row,) = list(_adapt(tmp_path)) - assert row.runtime_config is not None - assert row.runtime_config.compose is not None - context = row.runtime_config.compose.root - assert isinstance(context, Path) - manifest = _environment_config(context) - assert manifest["peers"] == [] - assert "peer_image_configs" not in manifest From 9ca8f25d5cebad137f9491ff023fc06791f7dd30 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:25 -0700 Subject: [PATCH 055/248] fix: report a task.toml that is not UTF-8 as a finding TOML documents are UTF-8. A task.toml with other bytes raised UnicodeDecodeError out of adapt() and aborted the whole dataset instead of failing that one task; it is now harbor.invalid.task_config_toml like any other unparsable task.toml. --- hud/integrations/harbor/adapt.py | 2 +- tests/integrations/harbor/cases/findings/expected.json | 6 ++++++ .../cases/findings/non-utf8-toml/environment/Dockerfile | 2 ++ .../harbor/cases/findings/non-utf8-toml/instruction.md | 1 + .../harbor/cases/findings/non-utf8-toml/task.toml | 1 + .../harbor/cases/findings/non-utf8-toml/tests/test.sh | 2 ++ 6 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/integrations/harbor/cases/findings/non-utf8-toml/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/non-utf8-toml/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/non-utf8-toml/task.toml create mode 100644 tests/integrations/harbor/cases/findings/non-utf8-toml/tests/test.sh diff --git a/hud/integrations/harbor/adapt.py b/hud/integrations/harbor/adapt.py index 2fde36430..7ce65f6fc 100644 --- a/hud/integrations/harbor/adapt.py +++ b/hud/integrations/harbor/adapt.py @@ -241,7 +241,7 @@ def add(code: str, message: str) -> None: return None, ( AdaptFinding(code="harbor.invalid.task_config_io", kind="invalid", message=str(error)), ) - except tomllib.TOMLDecodeError as error: + except (tomllib.TOMLDecodeError, UnicodeDecodeError) as error: return None, ( AdaptFinding( code="harbor.invalid.task_config_toml", kind="invalid", message=str(error) diff --git a/tests/integrations/harbor/cases/findings/expected.json b/tests/integrations/harbor/cases/findings/expected.json index 310585c52..c6e9ee836 100644 --- a/tests/integrations/harbor/cases/findings/expected.json +++ b/tests/integrations/harbor/cases/findings/expected.json @@ -209,6 +209,12 @@ "invalid", "main has no environment/Dockerfile, docker_image, or Compose image or build" ], + [ + "non-utf8-toml", + "harbor.invalid.task_config_toml", + "invalid", + "'utf-8' codec can't decode byte 0xff in position 8: invalid start byte" + ], [ "reserved-ports", "harbor.invalid.reserved_main_port", diff --git a/tests/integrations/harbor/cases/findings/non-utf8-toml/environment/Dockerfile b/tests/integrations/harbor/cases/findings/non-utf8-toml/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/non-utf8-toml/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/non-utf8-toml/instruction.md b/tests/integrations/harbor/cases/findings/non-utf8-toml/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/non-utf8-toml/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/non-utf8-toml/task.toml b/tests/integrations/harbor/cases/findings/non-utf8-toml/task.toml new file mode 100644 index 000000000..234e56f06 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/non-utf8-toml/task.toml @@ -0,0 +1 @@ +name = "" diff --git a/tests/integrations/harbor/cases/findings/non-utf8-toml/tests/test.sh b/tests/integrations/harbor/cases/findings/non-utf8-toml/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/non-utf8-toml/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt From 964745dd2b2c2aef91209b12856a60b5453a6113 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 056/248] fix: classify Compose include and extends as unsupported, as documented The Harbor docs list Compose include and extends among the declarations adaptation refuses by contract, but adapt() reported them as harbor.invalid.compose with kind invalid, as if the task were malformed. Harbor runs such projects; HUD's serialized Compose contract excludes them. ComposeConfig.from_file now raises ComposeUnsupportedError for them, and adapt() reports harbor.unsupported.compose_include_extends with kind contract. The docs name the contract codes. --- docs/v6/experimental/harbor.mdx | 13 ++++++++----- hud/eval/runtime/compose.py | 8 +++++++- hud/integrations/harbor/adapt.py | 4 ++++ .../compose-extends/environment/Dockerfile | 2 ++ .../findings/compose-extends/environment/base.yaml | 3 +++ .../environment/docker-compose.yaml | 4 ++++ .../cases/findings/compose-extends/instruction.md | 1 + .../cases/findings/compose-extends/task.toml | 0 .../cases/findings/compose-extends/tests/test.sh | 2 ++ .../compose-include/environment/Dockerfile | 2 ++ .../environment/docker-compose.yaml | 3 +++ .../compose-include/environment/other.yaml | 1 + .../cases/findings/compose-include/instruction.md | 1 + .../cases/findings/compose-include/task.toml | 0 .../cases/findings/compose-include/tests/test.sh | 2 ++ .../harbor/cases/findings/expected.json | 14 +++++++++++++- 16 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 tests/integrations/harbor/cases/findings/compose-extends/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/compose-extends/environment/base.yaml create mode 100644 tests/integrations/harbor/cases/findings/compose-extends/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/compose-extends/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/compose-extends/task.toml create mode 100644 tests/integrations/harbor/cases/findings/compose-extends/tests/test.sh create mode 100644 tests/integrations/harbor/cases/findings/compose-include/environment/Dockerfile create mode 100644 tests/integrations/harbor/cases/findings/compose-include/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/cases/findings/compose-include/environment/other.yaml create mode 100644 tests/integrations/harbor/cases/findings/compose-include/instruction.md create mode 100644 tests/integrations/harbor/cases/findings/compose-include/task.toml create mode 100644 tests/integrations/harbor/cases/findings/compose-include/tests/test.sh diff --git a/docs/v6/experimental/harbor.mdx b/docs/v6/experimental/harbor.mdx index e5e6230d0..ddff3aee1 100644 --- a/docs/v6/experimental/harbor.mdx +++ b/docs/v6/experimental/harbor.mdx @@ -121,7 +121,7 @@ The adapter handles: that agent service; - the Compose variables Harbor defines (`CONTEXT_DIR`, `MAIN_IMAGE_NAME`, `CPUS`, `MEMORY`, `PREBUILT_IMAGE_NAME`) and literal - `[environment.env]` values, alongside the project `.env`; + `[environment.env]` values, which take precedence over the project `.env`; - Compose sidecars, including ones that expose no TCP ports; - HTTP MCP servers (`sse` and `streamable-http`), healthchecks, network modes, allowlists, phase users, environment variables, and placement requirements; @@ -137,10 +137,13 @@ it can provision those requirements. stdio MCP servers and skills directories configure Harbor's own agents; the adapter accepts them and does not expose them to HUD agents. -Unsupported declarations fail during adaptation. These include multi-step -tasks, Compose variables only the host environment could supply, `include`, -and `extends`. A project also fails if its main service has neither an image -nor a valid build recipe. +Unsupported declarations fail during adaptation with a finding of kind +`contract`: multi-step tasks (`harbor.unsupported.multi_step`), Compose +variables only the host environment could supply +(`harbor.unsupported.host_compose_variable`), and Compose `include` and +`extends` (`harbor.unsupported.compose_include_extends`). Malformed tasks fail +with findings of kind `invalid`, such as a project whose main service has +neither an image nor a valid build recipe. ## Export HUD tasks to Harbor diff --git a/hud/eval/runtime/compose.py b/hud/eval/runtime/compose.py index 331387dd1..44e1e8af7 100644 --- a/hud/eval/runtime/compose.py +++ b/hud/eval/runtime/compose.py @@ -39,6 +39,10 @@ class ComposeUnboundVariableError(ValueError): """A Compose variable depends on values outside the packaged project.""" +class ComposeUnsupportedError(ValueError): + """A Compose document uses a feature the serialized project contract excludes.""" + + def _interpolate_compose_value( value: str, environment: Mapping[str, str], @@ -357,7 +361,9 @@ def from_file(cls, path: Path, *, variables: Mapping[str, str] | None = None) -> isinstance(service, dict) and "extends" in service for service in services.values() ) ): - raise ValueError("remote adaptation does not support Compose include or extends") + raise ComposeUnsupportedError( + "remote adaptation does not support Compose include or extends" + ) return cls.model_validate(document) def with_project_directory(self, directory: str) -> ComposeConfig: diff --git a/hud/integrations/harbor/adapt.py b/hud/integrations/harbor/adapt.py index 7ce65f6fc..c1ed443d3 100644 --- a/hud/integrations/harbor/adapt.py +++ b/hud/integrations/harbor/adapt.py @@ -25,6 +25,7 @@ ComposeProject, ComposeService, ComposeUnboundVariableError, + ComposeUnsupportedError, ) from hud.utils.naming import normalize_environment_name from hud.version import __version__ @@ -297,6 +298,9 @@ def add(code: str, message: str) -> None: except ComposeUnboundVariableError as error: add("harbor.unsupported.host_compose_variable", str(error)) compose = None + except ComposeUnsupportedError as error: + add("harbor.unsupported.compose_include_extends", str(error)) + compose = None except (OSError, ValueError, ValidationError) as error: add("harbor.invalid.compose", str(error)) compose = None diff --git a/tests/integrations/harbor/cases/findings/compose-extends/environment/Dockerfile b/tests/integrations/harbor/cases/findings/compose-extends/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/compose-extends/environment/base.yaml b/tests/integrations/harbor/cases/findings/compose-extends/environment/base.yaml new file mode 100644 index 000000000..2ca784f72 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/environment/base.yaml @@ -0,0 +1,3 @@ +services: + web: + image: nginx diff --git a/tests/integrations/harbor/cases/findings/compose-extends/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/compose-extends/environment/docker-compose.yaml new file mode 100644 index 000000000..782f39ca6 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/environment/docker-compose.yaml @@ -0,0 +1,4 @@ +services: + main: {} + web: + extends: {file: base.yaml, service: web} diff --git a/tests/integrations/harbor/cases/findings/compose-extends/instruction.md b/tests/integrations/harbor/cases/findings/compose-extends/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/compose-extends/task.toml b/tests/integrations/harbor/cases/findings/compose-extends/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/compose-extends/tests/test.sh b/tests/integrations/harbor/cases/findings/compose-extends/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-extends/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/compose-include/environment/Dockerfile b/tests/integrations/harbor/cases/findings/compose-include/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/cases/findings/compose-include/environment/docker-compose.yaml b/tests/integrations/harbor/cases/findings/compose-include/environment/docker-compose.yaml new file mode 100644 index 000000000..01b92e914 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/environment/docker-compose.yaml @@ -0,0 +1,3 @@ +include: [other.yaml] +services: + main: {} diff --git a/tests/integrations/harbor/cases/findings/compose-include/environment/other.yaml b/tests/integrations/harbor/cases/findings/compose-include/environment/other.yaml new file mode 100644 index 000000000..ad189ddbe --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/environment/other.yaml @@ -0,0 +1 @@ +services: {} diff --git a/tests/integrations/harbor/cases/findings/compose-include/instruction.md b/tests/integrations/harbor/cases/findings/compose-include/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/cases/findings/compose-include/task.toml b/tests/integrations/harbor/cases/findings/compose-include/task.toml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integrations/harbor/cases/findings/compose-include/tests/test.sh b/tests/integrations/harbor/cases/findings/compose-include/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/cases/findings/compose-include/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/cases/findings/expected.json b/tests/integrations/harbor/cases/findings/expected.json index c6e9ee836..1ce827c98 100644 --- a/tests/integrations/harbor/cases/findings/expected.json +++ b/tests/integrations/harbor/cases/findings/expected.json @@ -95,6 +95,18 @@ "invalid", "Invalid value (at line 1, column 8)" ], + [ + "compose-extends", + "harbor.unsupported.compose_include_extends", + "contract", + "remote adaptation does not support Compose include or extends" + ], + [ + "compose-include", + "harbor.unsupported.compose_include_extends", + "contract", + "remote adaptation does not support Compose include or extends" + ], [ "compose-not-mapping", "harbor.invalid.compose", @@ -334,4 +346,4 @@ ] } } -} \ No newline at end of file +} From 3cb094c3d203714a14ed4d924827b14d127a8629 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 057/248] test: run the scripts harbor.export generates The export tests grepped the generated scripts for strings such as 'exit 1' and '|| true', so neither the setup-failure path nor the grading path ever ran. The scenario now snapshots the written folder and the parsed task.toml, runs hud_entrypoint.sh with a stub hud on PATH and records the commands it issues, and shows that a failing task start exits 1 without running the agent command. A table covers each source form (Python, renamed binding, JSON, JSONL, Dockerfile.hud, dockerignore, custom answer file) and another the refusals, including that the environments export started are stopped when it fails. The docker lane builds the exported image, parks the task the way Harbor starts it, and grades it with tests/test.sh. --- tests/integrations/harbor/test_export.py | 741 +++++++++++++++++++++++ tests/integrations/harbor/test_harbor.py | 267 -------- 2 files changed, 741 insertions(+), 267 deletions(-) create mode 100644 tests/integrations/harbor/test_export.py delete mode 100644 tests/integrations/harbor/test_harbor.py diff --git a/tests/integrations/harbor/test_export.py b/tests/integrations/harbor/test_export.py new file mode 100644 index 000000000..926dbf036 --- /dev/null +++ b/tests/integrations/harbor/test_export.py @@ -0,0 +1,741 @@ +"""``harbor.export`` writes Harbor task folders whose scripts drive the ``hud`` CLI. + +The default lane snapshots what export writes and runs the generated +entrypoint with a stub ``hud`` on ``PATH``, recording the commands it issues. +The docker lane builds the exported image, starts it the way Harbor does, and +runs ``tests/test.sh`` inside it. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import subprocess +import sys +import textwrap +import tomllib +import uuid +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest +from inline_snapshot import snapshot + +from hud.integrations.harbor import export + +if TYPE_CHECKING: + from collections.abc import Callable + + from tests.harness import HudEnv + +REPO = Path(__file__).resolve().parents[3] + +ENV_PY = """\ +import os +from pathlib import Path + +from hud import Environment + +env = Environment("demo") +markers = os.environ.get("EXPORT_TEST_MARKERS") + + +@env.initialize +async def started() -> None: + if markers: + Path(markers, "initialized").touch() + + +@env.shutdown +async def stopped() -> None: + if markers: + Path(markers, "stopped").touch() + + +@env.template() +async def solve(n: int = 1, prompt: str = ""): + answer = yield f"solve {n}: {prompt}" + if answer == "crash": + raise RuntimeError("the grader is broken") + yield 1.0 if answer == str(n) else 0.25 + + +tasks = [solve(n=2, prompt="what's next?")] +""" + +DOCKERFILE = 'FROM python:3.12-slim\nRUN pip install hud\nCMD ["hud", "serve", "env:env"]\n' + +STUB_HUD = """\ +#!{python} +import json, os, sys + +with open(os.environ["STUB_HUD_LOG"], "a") as log: + log.write(json.dumps(sys.argv[1:]) + "\\n") +if sys.argv[1:3] == ["task", "start"]: + sys.exit(int(os.environ.get("STUB_HUD_START_EXIT", "0"))) +""" + + +def write_source(root: Path, files: dict[str, str]) -> Path: + root.mkdir(parents=True, exist_ok=True) + for relative, content in {"env.py": ENV_PY, "Dockerfile": DOCKERFILE, **files}.items(): + (root / relative).write_text(content, encoding="utf-8") + return root + + +def written(task_dir: Path) -> dict[str, Any]: + files = sorted( + path.relative_to(task_dir).as_posix() for path in task_dir.rglob("*") if path.is_file() + ) + environment = task_dir / "environment" + ignore = environment / ".dockerignore" + return { + "files": files, + "task.toml": tomllib.loads((task_dir / "task.toml").read_text("utf-8")), + "instruction.md": (task_dir / "instruction.md").read_text("utf-8"), + "Dockerfile": (environment / "Dockerfile").read_text("utf-8"), + **({".dockerignore": ignore.read_text("utf-8")} if ignore.is_file() else {}), + } + + +def names(directory: Path) -> list[str]: + return sorted(path.name for path in directory.iterdir()) + + +def run_entrypoint(task_dir: Path, tmp_path: Path, *, start_exit: int = 0) -> dict[str, Any]: + """Run ``hud_entrypoint.sh`` with a stub ``hud`` and a command that leaves a marker.""" + stub_bin = tmp_path / "stub-bin" + stub_bin.mkdir(exist_ok=True) + stub = stub_bin / "hud" + stub.write_text(STUB_HUD.format(python=sys.executable), encoding="utf-8") + stub.chmod(0o755) + log = tmp_path / f"hud-{uuid.uuid4().hex}.jsonl" + marker = tmp_path / "agent-ran" + marker.unlink(missing_ok=True) + completed = subprocess.run( + ["sh", str(task_dir / "environment/hud_entrypoint.sh"), "touch", str(marker)], + env={ + **os.environ, + "PATH": f"{stub_bin}{os.pathsep}{os.environ['PATH']}", + "STUB_HUD_LOG": str(log), + "STUB_HUD_START_EXIT": str(start_exit), + }, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + return { + "exit": completed.returncode, + "stderr": completed.stderr, + "agent ran": marker.exists(), + # ``hud serve`` runs in the background, so the two calls race. + "hud calls": sorted(json.loads(line) for line in log.read_text("utf-8").splitlines()), + } + + +def main_thread_guard(source: Path) -> None: + env_py = source / "env.py" + env_py.write_text( + "import threading\nassert threading.current_thread() is threading.main_thread()\n" + + env_py.read_text("utf-8"), + encoding="utf-8", + ) + + +def rows(*entries: dict[str, Any]) -> Callable[[Path], None]: + def write(source: Path) -> None: + suffix = ".jsonl" if len(entries) > 1 else ".json" + lines = [json.dumps(entry) for entry in entries] + (source / f"tasks{suffix}").write_text( + "\n".join(lines) if suffix == ".jsonl" else json.dumps(list(entries)), + encoding="utf-8", + ) + + return write + + +@pytest.fixture +def markers(tmp_path: Path, hud_env: HudEnv) -> Path: + """Where the source environment's initialize and shutdown hooks leave a file.""" + directory = tmp_path / "markers" + directory.mkdir() + hud_env.set(EXPORT_TEST_MARKERS=str(directory)) + return directory + + +async def test_export_writes_a_harbor_task_whose_entrypoint_parks_the_task( + tmp_path: Path, markers: Path +) -> None: + source = write_source(tmp_path / "source", {}) + + (task_dir,) = await export(str(source / "env.py"), tmp_path / "out") + + assert {**written(task_dir), "entrypoint": run_entrypoint(task_dir, tmp_path)} == snapshot( + { + "files": [ + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "task.toml": { + "version": "1.0", + "name": "solve-4080368a", + "metadata": { + "hud_task": "solve", + "hud_args": '{"n": 2, "prompt": "what\'s next?"}', + }, + "agent": {"timeout_sec": 600.0}, + "verifier": {"timeout_sec": 600.0}, + }, + "instruction.md": """\ +solve 2: what's next? + +--- +When you have finished, write your final answer to `/workspace/answer.txt`. +""", + "Dockerfile": """\ +FROM python:3.12-slim +RUN pip install hud +CMD ["hud", "serve", "env:env"] + +# HUD runtime for Harbor; final startup directives override the source image. +COPY --chmod=0755 hud_entrypoint.sh /hud_entrypoint.sh +ENTRYPOINT ["/hud_entrypoint.sh"] +CMD ["sh", "-c", "sleep infinity"] +""", + "entrypoint": { + "exit": 0, + "stderr": "", + "agent ran": True, + "hud calls": [ + ["serve", "env.py:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 2, "prompt": "what\'s next?"}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + }, + } + ) + assert names(markers) == ["initialized", "stopped"] + + +SOURCES: dict[str, dict[str, Any]] = { + "a python source importing on the main thread": {"source": "env.py", "edit": main_thread_guard}, + "an environment bound under another name": { + "source": "tasks.py", + "files": { + "tasks.py": ENV_PY.replace("env = Environment", "bench = Environment").replace( + "@env.", "@bench." + ) + }, + }, + "a json taskset with a nested slug": { + "source": "tasks.json", + "edit": rows({"env": "demo", "id": "solve", "args": {"n": 3}, "slug": "suite/fix"}), + }, + "a jsonl taskset": { + "source": "tasks.jsonl", + "edit": rows( + {"env": "demo", "id": "solve", "args": {"n": 4}, "slug": "four"}, + {"env": "demo", "id": "solve", "args": {"n": 5}, "slug": "five"}, + ), + }, + "a hud dockerfile beside the plain one": { + "source": "env.py", + "files": {"Dockerfile.hud": "FROM python:3.12-slim AS hud\nRUN pip install hud\n"}, + "show": ["Dockerfile"], + }, + "a restrictive dockerignore and build files": { + "source": "env.py", + "files": {".dockerignore": "*\n", "package.json": '{"name": "app"}\n'}, + "show": [".dockerignore"], + }, + "a non-root multiline image": { + "source": "env.py", + "files": { + "Dockerfile": ( + "FROM python:3.12-slim\nRUN useradd -m app\nUSER app\n" + 'CMD ["python", \\\n "-m", \\\n "app"]\n' + ) + }, + "show": ["Dockerfile"], + }, + "a custom answer file": { + "source": "env.py", + "answer_file": "/app/out.txt", + "show": ["instruction.md"], + }, +} + + +@pytest.mark.parametrize("case", list(SOURCES)) +async def test_export_maps_each_source_form_to_harbor_task_folders( + case: str, tmp_path: Path, markers: Path +) -> None: + spec = SOURCES[case] + source = write_source(tmp_path / "source", spec.get("files", {})) + if edit := spec.get("edit"): + edit(source) + + created = await export( + str(source / spec["source"]), + tmp_path / "out", + answer_file=spec.get("answer_file", "/workspace/answer.txt"), + ) + + observed = {} + for task_dir in created: + files = written(task_dir) + observed[task_dir.name] = { + "files": files["files"], + "hud calls": run_entrypoint(task_dir, tmp_path)["hud calls"], + **{key: files[key] for key in spec.get("show", [])}, + } + assert ( + observed + == snapshot( + { + "a python source importing on the main thread": { + "solve-4080368a": { + "files": [ + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "hud calls": [ + ["serve", "env.py:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 2, "prompt": "what\'s next?"}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + } + }, + "an environment bound under another name": { + "solve-4080368a": { + "files": [ + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "environment/tasks.py", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "hud calls": [ + ["serve", "tasks.py:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 2, "prompt": "what\'s next?"}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + } + }, + "a json taskset with a nested slug": { + "suite-fix": { + "files": [ + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "hud calls": [ + ["serve", ".:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 3}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + } + }, + "a jsonl taskset": { + "four": { + "files": [ + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "hud calls": [ + ["serve", ".:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 4}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + }, + "five": { + "files": [ + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "hud calls": [ + ["serve", ".:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 5}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + }, + }, + "a hud dockerfile beside the plain one": { + "solve-4080368a": { + "files": [ + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "hud calls": [ + ["serve", "env.py:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 2, "prompt": "what\'s next?"}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + "Dockerfile": """\ +FROM python:3.12-slim AS hud +RUN pip install hud + +# HUD runtime for Harbor; final startup directives override the source image. +COPY --chmod=0755 hud_entrypoint.sh /hud_entrypoint.sh +ENTRYPOINT ["/hud_entrypoint.sh"] +CMD ["sh", "-c", "sleep infinity"] +""", + } + }, + "a restrictive dockerignore and build files": { + "solve-4080368a": { + "files": [ + "environment/.dockerignore", + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "environment/package.json", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "hud calls": [ + ["serve", "env.py:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 2, "prompt": "what\'s next?"}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + ".dockerignore": """\ +* + +!hud_entrypoint.sh +""", + } + }, + "a non-root multiline image": { + "solve-4080368a": { + "files": [ + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "hud calls": [ + ["serve", "env.py:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 2, "prompt": "what\'s next?"}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + "Dockerfile": """\ +FROM python:3.12-slim +RUN useradd -m app +USER app +CMD ["python", \\ + "-m", \\ + "app"] + +# HUD runtime for Harbor; final startup directives override the source image. +COPY --chmod=0755 hud_entrypoint.sh /hud_entrypoint.sh +ENTRYPOINT ["/hud_entrypoint.sh"] +CMD ["sh", "-c", "sleep infinity"] +""", + } + }, + "a custom answer file": { + "solve-4080368a": { + "files": [ + "environment/Dockerfile", + "environment/env.py", + "environment/hud_entrypoint.sh", + "instruction.md", + "task.toml", + "tests/test.sh", + ], + "hud calls": [ + ["serve", "env.py:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 2, "prompt": "what\'s next?"}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + "instruction.md": """\ +solve 2: what's next? + +--- +When you have finished, write your final answer to `/app/out.txt`. +""", + } + }, + } + )[case] + ) + assert names(markers) == ["initialized", "stopped"] + + +@pytest.mark.parametrize( + ("case", "edit", "error", "message", "started"), + [ + ( + "no dockerfile", + lambda source: (source / "Dockerfile").unlink(), + FileNotFoundError, + "no Dockerfile", + False, + ), + ( + "a capability harbor cannot carry", + lambda source: (source / "env.py").write_text( + ENV_PY.replace( + 'env = Environment("demo")\n', + 'env = Environment("demo")\n' + "from hud.capabilities import Capability\n" + 'env.add_capability(Capability.cdp(url="ws://127.0.0.1:9222"))\n', + ), + encoding="utf-8", + ), + ValueError, + r"non-Harbor capabilities \['cdp/1.3'\]; only ssh/mcp are convertible", + True, + ), + ( + "a task the environment does not define", + rows({"env": "demo", "id": "missing", "args": {}}), + TypeError, + "needs a local env defining task 'missing'", + True, + ), + ( + "a slug without a usable name", + rows({"env": "demo", "id": "solve", "args": {}, "slug": ".."}), + ValueError, + "does not form a usable directory name", + True, + ), + ( + "two slugs naming one directory", + rows( + {"env": "demo", "id": "solve", "args": {"n": 1}, "slug": "suite/fix"}, + {"env": "demo", "id": "solve", "args": {"n": 2}, "slug": "suite-fix"}, + ), + ValueError, + "task slugs 'suite/fix' and 'suite-fix' both name the export directory 'suite-fix'", + True, + ), + ], +) +async def test_export_refuses_sources_harbor_cannot_run_and_stops_what_it_started( + case: str, + edit: Callable[[Path], None], + error: type[Exception], + message: str, + started: bool, + tmp_path: Path, + markers: Path, +) -> None: + del case + source = write_source(tmp_path / "source", {}) + edit(source) + taskset = next(source.glob("tasks.json*"), source / "env.py") + + with pytest.raises(error, match=message): + await export(str(taskset), tmp_path / "out") + + assert names(markers) == (["initialized", "stopped"] if started else []) + + +async def test_the_entrypoint_refuses_to_run_the_agent_when_task_setup_fails( + tmp_path: Path, +) -> None: + (task_dir,) = await export(str(write_source(tmp_path / "source", {}) / "env.py"), tmp_path) + + assert run_entrypoint(task_dir, tmp_path, start_exit=3) == snapshot( + { + "exit": 1, + "stderr": "hud: task setup failed; refusing to run the agent against an unset task\n", + "agent ran": False, + "hud calls": [ + ["serve", "env.py:demo", "--port", "8765"], + [ + "task", + "start", + "solve", + "--args", + '{"n": 2, "prompt": "what\'s next?"}', + "--url", + "tcp://127.0.0.1:8765", + ], + ], + } + ) + + +@pytest.fixture(scope="module") +def wheel(tmp_path_factory: pytest.TempPathFactory) -> Path: + wheels = tmp_path_factory.mktemp("wheels") + subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(wheels)], + cwd=REPO, + check=True, + capture_output=True, + ) + return next(wheels.glob("*.whl")) + + +def docker(*args: str, timeout: float = 600) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", *args], capture_output=True, text=True, timeout=timeout, check=False + ) + + +async def wait_for_process(container: str, command: str, deadline: float = 240) -> bool: + """Whether ``command`` runs in ``container`` before ``deadline`` seconds pass.""" + loop = asyncio.get_running_loop() + end = loop.time() + deadline + while loop.time() < end: + if command in docker("top", container).stdout: + return True + await asyncio.sleep(1) + return False + + +@pytest.mark.e2e +@pytest.mark.docker +@pytest.mark.parametrize( + ("answer", "exit_code", "reward"), + [("2", 0, "1.0"), ("wrong", 0, "0.25"), ("crash", 1, None)], +) +async def test_an_exported_task_grades_its_parked_run_inside_the_built_image( + answer: str, + exit_code: int, + reward: str | None, + wheel: Path, + tmp_path: Path, +) -> None: + source = write_source( + tmp_path / "source", + { + "Dockerfile": textwrap.dedent(f"""\ + FROM python:3.12-slim + COPY {wheel.name} /tmp/ + RUN pip install --no-cache-dir /tmp/{wheel.name} + WORKDIR /workspace + COPY env.py ./ + """), + }, + ) + shutil.copy2(wheel, source / wheel.name) + (task_dir,) = await export(str(source / "env.py"), tmp_path / "out") + image = f"hud-export-test:{uuid.uuid4().hex[:12]}" + built = docker("build", "--tag", image, str(task_dir / "environment")) + assert built.returncode == 0, built.stderr[-4000:] + started = docker("run", "--detach", "--rm", image) + assert started.returncode == 0, started.stderr + container = started.stdout.strip() + try: + parked = await wait_for_process(container, "sleep infinity") + assert parked, docker("logs", container).stderr[-4000:] + assert docker("cp", str(task_dir / "tests"), f"{container}:/tests").returncode == 0 + written_answer = docker( + "exec", container, "sh", "-c", f"printf %s {answer} > /workspace/answer.txt" + ) + assert written_answer.returncode == 0, written_answer.stderr + + graded = docker("exec", container, "sh", "/tests/test.sh", timeout=120) + read = docker("exec", container, "cat", "/logs/verifier/reward.txt") + finally: + docker("rm", "--force", container) + docker("image", "rm", "--force", image) + + assert graded.returncode == exit_code, graded.stderr + assert (read.stdout.strip() if read.returncode == 0 else None) == reward diff --git a/tests/integrations/harbor/test_harbor.py b/tests/integrations/harbor/test_harbor.py deleted file mode 100644 index 229d282d8..000000000 --- a/tests/integrations/harbor/test_harbor.py +++ /dev/null @@ -1,267 +0,0 @@ -"""``hud.integrations.harbor`` task adaptation and export.""" - -from __future__ import annotations - -import asyncio -import json -import textwrap -from typing import TYPE_CHECKING, Any - -import pytest - -from hud.integrations.harbor import export - -if TYPE_CHECKING: - from pathlib import Path - -# ─── export: HUD tasks -> Harbor task folders ─────────────────────────── - -_ENV_PY = """\ -from hud import Environment - -env = Environment("demo") - - -@env.template() -async def solve(n: int = 1): - yield f"solve {n}" - yield 1.0 - - -tasks = [solve(n=2)] -""" - -_DOCKERFILE = """\ -FROM python:3.11-slim -RUN pip install hud -COPY env.py ./ -CMD ["hud", "serve", "env:env"] -""" - - -def _write_env( - tmp_path: Path, *, dockerfile: bool = True, args: dict[str, Any] | None = None -) -> Path: - src = tmp_path / "env.py" - body = textwrap.dedent(_ENV_PY) - if args is not None: - body = body.replace( - "async def solve(n: int = 1):", "async def solve(n: int = 1, prompt: str = ''):" - ).replace("tasks = [solve(n=2)]", f"tasks = [solve(**{args!r})]") - src.write_text(body, encoding="utf-8") - if dockerfile: - (tmp_path / "Dockerfile").write_text(_DOCKERFILE, encoding="utf-8") - return src - - -async def test_export_writes_task_folder(tmp_path: Path) -> None: - src = _write_env(tmp_path) - out = tmp_path / "out" - - created = await export(str(src), out) - - assert len(created) == 1 - task_dir = created[0] - assert (task_dir / "task.toml").exists() - assert (task_dir / "instruction.md").exists() - assert (task_dir / "tests" / "test.sh").exists() - assert (task_dir / "environment" / "Dockerfile").exists() - assert (task_dir / "environment" / "hud_entrypoint.sh").exists() - - -async def test_export_loads_python_source_on_main_thread(tmp_path: Path) -> None: - src = _write_env(tmp_path) - source = src.read_text(encoding="utf-8") - src.write_text( - "import threading\nassert threading.current_thread() is threading.main_thread()\n" + source, - encoding="utf-8", - ) - - created = await export(str(src), tmp_path / "out") - - assert len(created) == 1 - - -async def test_requires_dockerfile(tmp_path: Path) -> None: - _write_env(tmp_path, dockerfile=False) - with pytest.raises(FileNotFoundError, match="Dockerfile"): - await export(str(tmp_path / "env.py"), tmp_path / "out") - - -async def test_instruction_has_prompt_and_answer_convention(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - instruction = (created[0] / "instruction.md").read_text(encoding="utf-8") - assert instruction.startswith("solve 2") # the materialized prompt - assert "/workspace/answer.txt" in instruction # the answer convention - - -async def test_task_toml_is_harbor_native(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - toml = (created[0] / "task.toml").read_text(encoding="utf-8") - assert 'version = "1.0"' in toml - assert "name = " in toml - assert "[verifier]" in toml and "[agent]" in toml - assert "timeout_sec" in toml - # HUD task/args preserved as metadata for the record. - assert "hud_task" in toml and "hud_args" in toml - - -async def test_scripts_drive_hud_task_lifecycle(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - boot = (created[0] / "environment" / "hud_entrypoint.sh").read_text(encoding="utf-8") - test_sh = (created[0] / "tests" / "test.sh").read_text(encoding="utf-8") - - # Boot serves the channel, parks the run via setup, then hands off. - assert "hud serve env.py:demo" in boot - assert "hud task start solve" in boot - assert "python3 -c" not in boot - assert 'exec "$@"' in boot - # Verifier grades the parked run and writes the Harbor reward. - assert "hud task grade solve" in test_sh - assert "--answer-file" in test_sh - assert "/logs/verifier/reward.txt" in test_sh - - -async def test_scripts_survive_arguments_that_need_quoting(tmp_path: Path) -> None: - # Task ids and args are data: an apostrophe must not end the shell string. - _write_env(tmp_path, args={"prompt": "what's next?"}) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - for script in ("environment/hud_entrypoint.sh", "tests/test.sh"): - text = (created[0] / script).read_text(encoding="utf-8") - proc = await asyncio.create_subprocess_exec( - "sh", "-n", "-", stdin=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE - ) - _out, err = await proc.communicate(text.encode()) - assert proc.returncode == 0, f"{script} is not valid shell: {err.decode()}" - - -async def test_environment_context_keeps_its_own_json_files(tmp_path: Path) -> None: - # Only the copied taskset source is dropped; a build context may need its - # own JSON (package.json and friends). - _write_env(tmp_path) - (tmp_path / "package.json").write_text('{"name": "app"}', encoding="utf-8") - - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - assert (created[0] / "environment" / "package.json").is_file() - - -async def test_dockerfile_appends_the_harbor_runtime(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - dockerfile = (created[0] / "environment" / "Dockerfile").read_text(encoding="utf-8") - assert "COPY --chmod=0755 hud_entrypoint.sh /hud_entrypoint.sh" in dockerfile - assert 'ENTRYPOINT ["/hud_entrypoint.sh"]' in dockerfile - assert dockerfile.rindex('CMD ["sh", "-c", "sleep infinity"]') > dockerfile.index( - 'CMD ["hud", "serve", "env:env"]' - ) - # The env build context is copied so the image can be rebuilt under Harbor. - assert (created[0] / "environment" / "env.py").exists() - - -async def test_custom_answer_file(tmp_path: Path) -> None: - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out", answer_file="/app/out.txt") - assert "/app/out.txt" in (created[0] / "instruction.md").read_text(encoding="utf-8") - assert "/app/out.txt" in (created[0] / "tests" / "test.sh").read_text(encoding="utf-8") - - -async def test_export_fails_loudly_when_setup_cannot_run(tmp_path: Path) -> None: - # An unstarted task must not reach the verifier as a score of 0: broken - # infrastructure and a failed agent are different outcomes. - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - boot = (created[0] / "environment" / "hud_entrypoint.sh").read_text(encoding="utf-8") - - assert "|| true" not in boot - assert "exit 1" in boot - - -async def test_export_survives_a_restrictive_dockerignore(tmp_path: Path) -> None: - # Same class as the adaptation layer's: re-including a generated file - # needs its own rule, or COPY fails and no exported image builds. - _write_env(tmp_path) - (tmp_path / ".dockerignore").write_text("*\n", encoding="utf-8") - - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - ignore = (created[0] / "environment" / ".dockerignore").read_text(encoding="utf-8") - assert "!hud_entrypoint.sh" in ignore - - -async def test_export_overrides_multiline_startup_directives(tmp_path: Path) -> None: - _write_env(tmp_path) - (tmp_path / "Dockerfile").write_text( - 'FROM python:3.12-slim\nCMD ["python", \\\n "-m", \\\n "app"]\n', encoding="utf-8" - ) - - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - dockerfile = (created[0] / "environment" / "Dockerfile").read_text(encoding="utf-8") - assert dockerfile.rindex('CMD ["sh", "-c", "sleep infinity"]') > dockerfile.index('"app"]') - - -async def test_export_preserves_a_non_root_runtime_user(tmp_path: Path) -> None: - _write_env(tmp_path) - (tmp_path / "Dockerfile").write_text( - "FROM python:3.12-slim\nRUN useradd -m app\nUSER app\n", encoding="utf-8" - ) - - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - dockerfile = (created[0] / "environment" / "Dockerfile").read_text(encoding="utf-8") - assert dockerfile.count("USER ") == 1 - assert "COPY --chmod=0755" in dockerfile - - -async def test_export_serves_the_resolved_environment(tmp_path: Path) -> None: - # The environment is bound to ``bench`` in tasks.py, while its public name - # remains ``demo``. The generated target uses the canonical loader syntax. - (tmp_path / "tasks.py").write_text( - textwrap.dedent(_ENV_PY) - .replace("env = Environment", "bench = Environment") - .replace("@env.template", "@bench.template"), - encoding="utf-8", - ) - (tmp_path / "Dockerfile").write_text(_DOCKERFILE, encoding="utf-8") - - created = await export(str(tmp_path / "tasks.py"), tmp_path / "out") - - boot = (created[0] / "environment" / "hud_entrypoint.sh").read_text(encoding="utf-8") - assert "hud serve tasks.py:demo" in boot - - -async def test_export_slugs_stay_inside_the_output_directory(tmp_path: Path) -> None: - _write_env(tmp_path) - taskset = tmp_path / "tasks.json" - taskset.write_text( - json.dumps([{"env": "demo", "id": "solve", "args": {"n": 2}, "slug": "suite/fix"}]), - encoding="utf-8", - ) - - created = await export(str(taskset), tmp_path / "out") - - assert created[0].name == "suite-fix" - taskset.write_text( - json.dumps([{"env": "demo", "id": "solve", "args": {"n": 2}, "slug": ".."}]), - encoding="utf-8", - ) - with pytest.raises(ValueError, match="usable directory name"): - await export(str(taskset), tmp_path / "invalid") - - -async def test_export_never_scores_a_broken_grader_as_zero(tmp_path: Path) -> None: - # Startup and grading are the same rule: infrastructure that cannot run is - # an error, not an agent that failed the task. - _write_env(tmp_path) - created = await export(str(tmp_path / "env.py"), tmp_path / "out") - - test_sh = (created[0] / "tests" / "test.sh").read_text(encoding="utf-8") - - assert "echo 0 > /logs/verifier/reward.txt" not in test_sh - assert "exit 1" in test_sh From 05e5efcb05278ee3dc39e2130e5348bed8ac41db Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 058/248] test: add a flaky TCP relay and an eventually() poller to the harness relay(host, port) serves a loopback relay in front of a real server that can sever every connection, refuse the next ones, or stall new ones until released, so reconnecting clients can be driven through real network failures instead of patched internals. eventually() polls a condition with a deadline for effects that have no event to wait on. --- tests/harness/__init__.py | 6 +- tests/harness/relay.py | 142 ++++++++++++++++++++++++++++++++++++++ tests/harness/scenario.py | 9 +++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 tests/harness/relay.py diff --git a/tests/harness/__init__.py b/tests/harness/__init__.py index b1f222517..295fcf462 100644 --- a/tests/harness/__init__.py +++ b/tests/harness/__init__.py @@ -23,8 +23,9 @@ shell_call, stream_error, ) +from .relay import FlakyRelay, relay from .rfb import FakeScreen, KeyEvent, PointerEvent, fake_screen -from .scenario import RecordingProvider, ScriptedAgent, served, task_row +from .scenario import RecordingProvider, ScriptedAgent, eventually, served, task_row from .services import FakeServices, Reply, Request from .spans import ROBOT_STEP_SCHEMA, spans, steps @@ -33,6 +34,7 @@ "FakeDocker", "FakeScreen", "FakeServices", + "FlakyRelay", "Hud", "HudEnv", "KeyEvent", @@ -49,9 +51,11 @@ "call", "calls", "computer_call", + "eventually", "fail", "fake_screen", "interrupted", + "relay", "say", "scrub", "served", diff --git a/tests/harness/relay.py b/tests/harness/relay.py new file mode 100644 index 000000000..413b9e03d --- /dev/null +++ b/tests/harness/relay.py @@ -0,0 +1,142 @@ +"""A TCP relay in front of one upstream that can sever, refuse and stall connections. + +Clients that reconnect (``SSHClient``, the control-channel tunnels) are driven +through :func:`relay` to fake an unreliable network between them and a real +server: :meth:`FlakyRelay.sever` drops every relayed connection, +:meth:`FlakyRelay.refuse` closes the next connections as soon as they are +accepted, and :meth:`FlakyRelay.stall` holds new connections without +forwarding them until :meth:`FlakyRelay.release`. +""" + +from __future__ import annotations + +import asyncio +import contextlib +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + +class FlakyRelay: + """The relay's controls. Use :func:`relay`.""" + + def __init__(self, upstream: tuple[str, int]) -> None: + self.upstream = upstream + self.port = 0 + #: Connections accepted so far, refused and stalled ones included. + self.accepted = 0 + self._refusals = 0 + self._released = asyncio.Event() + self._released.set() + self._changed = asyncio.Condition() + self._writers: set[asyncio.StreamWriter] = set() + self._handlers: set[asyncio.Task[None]] = set() + self._stalled = 0 + + @property + def open(self) -> int: + """Client connections the relay currently holds open.""" + return len(self._writers) + + @property + def stalled(self) -> int: + """Connections waiting for :meth:`release`.""" + return self._stalled + + def sever(self) -> None: + """Drop every connection the relay holds, as a network failure would.""" + for writer in list(self._writers): + writer.transport.abort() + + def refuse(self, count: int) -> None: + """Close each of the next ``count`` connections as soon as it is accepted.""" + self._refusals = count + + def stall(self) -> None: + """Hold new connections open without forwarding them until :meth:`release`.""" + self._released.clear() + + def release(self) -> None: + self._released.set() + + async def wait_until(self, *, accepted: int | None = None, stalled: int | None = None) -> None: + """Wait until at least ``accepted`` connections were accepted or ``stalled`` wait.""" + async with self._changed: + await asyncio.wait_for( + self._changed.wait_for( + lambda: ( + (accepted is None or self.accepted >= accepted) + and (stalled is None or self._stalled >= stalled) + ) + ), + timeout=10, + ) + + async def _notify(self) -> None: + async with self._changed: + self._changed.notify_all() + + async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + task = asyncio.current_task() + assert task is not None + self._handlers.add(task) + self._writers.add(writer) + self.accepted += 1 + try: + if self._refusals: + self._refusals -= 1 + await self._notify() + return + if not self._released.is_set(): + self._stalled += 1 + await self._notify() + try: + await self._released.wait() + finally: + self._stalled -= 1 + else: + await self._notify() + up_reader, up_writer = await asyncio.open_connection(*self.upstream) + try: + await asyncio.gather(_pipe(reader, up_writer), _pipe(up_reader, writer)) + finally: + up_writer.transport.abort() + except OSError: + pass + finally: + self._writers.discard(writer) + self._handlers.discard(task) + writer.transport.abort() + await self._notify() + + +async def _pipe(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + except OSError: + pass + finally: + writer.transport.abort() + + +@asynccontextmanager +async def relay(host: str, port: int) -> AsyncIterator[FlakyRelay]: + """Serve a :class:`FlakyRelay` to ``host:port`` on a loopback port for the block.""" + controls = FlakyRelay((host, port)) + server = await asyncio.start_server(controls._handle, "127.0.0.1", 0) + controls.port = server.sockets[0].getsockname()[1] + try: + yield controls + finally: + server.close() + controls.release() + controls.sever() + for handler in list(controls._handlers): + handler.cancel() + with contextlib.suppress(asyncio.CancelledError): + await asyncio.gather(*controls._handlers, return_exceptions=True) + await server.wait_closed() diff --git a/tests/harness/scenario.py b/tests/harness/scenario.py index ac9794387..84bdcec19 100644 --- a/tests/harness/scenario.py +++ b/tests/harness/scenario.py @@ -78,6 +78,15 @@ async def served(env: Environment, *, row: Task | None = None) -> AsyncIterator[ yield client +async def eventually(condition: Callable[[], bool], *, within: float = 10.0) -> None: + """Poll until ``condition()`` holds; raise ``TimeoutError`` after ``within`` seconds.""" + async with asyncio.timeout(within): + while True: + if condition(): + return + await asyncio.sleep(0.01) + + def task_row(env: Environment, template: str, **args: Any) -> Task: """The data row for ``template`` on ``env``, as a taskset file would carry it.""" return Task(env=env.name, id=template, args=args) From 214d9bd314b99782a2a85ab34be361c3de884c92 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 059/248] test: drive SSHClient through a relay in front of a served workspace The SSH tests patched SSHClient._connect, asyncio.sleep and the client's own run, and asserted the kwargs of a fake process. They now connect to a real env.workspace through the flaky relay: files round-trip (empty and binary content, quoted names), a timed-out or cancelled command's remote process is gone, a command lost in flight is not replayed and the next one reconnects, three refused reconnects give up, a timeout covers a stalled reconnect, closing during a reconnect discards the new connection, and a dropped handshake is a connection error. A stub powershell on the workspace PATH shows the Windows file helpers' encoded scripts and base64 transfer. --- tests/capabilities/test_ssh.py | 614 ++++++++++++--------------------- 1 file changed, 222 insertions(+), 392 deletions(-) diff --git a/tests/capabilities/test_ssh.py b/tests/capabilities/test_ssh.py index 0c619dc28..94aa1d93d 100644 --- a/tests/capabilities/test_ssh.py +++ b/tests/capabilities/test_ssh.py @@ -1,437 +1,267 @@ +"""``SSHClient`` against a served workspace, through a relay that fails on demand. + +Every test serves a real ``env.workspace`` on loopback and connects to its +``ssh`` binding through :func:`tests.harness.relay`, which can sever the +connection mid-command, refuse reconnects, or stall a handshake. +""" + from __future__ import annotations import asyncio -from typing import TYPE_CHECKING, Any, cast -from unittest.mock import AsyncMock +import os +import sys +from contextlib import asynccontextmanager +from dataclasses import replace +from pathlib import Path +from typing import TYPE_CHECKING +from urllib.parse import urlsplit -import asyncssh import pytest -from hud.capabilities.base import Capability -from hud.capabilities.connection import Connection -from hud.capabilities.ssh import PROCESS_CONNECTIONS_REQUEST, SSHClient, SSHConnectionError +from hud import Environment +from hud.capabilities import Connection, SSHClient +from tests.harness import FlakyRelay, eventually, relay, served if TYPE_CHECKING: - from collections.abc import Callable - - -class _Completed: - stdout = "ok" - stderr = "" - exit_status = 0 - returncode: int | None = 0 - - -class _Process: - def __init__( - self, - *, - completed: _Completed | None = None, - wait_error: BaseException | None = None, - block: bool = False, - ) -> None: - self.completed = completed or _Completed() - self.wait_error = wait_error - self.block = block - self.on_wait: Callable[[], None] | None = None - self.started = asyncio.Event() - self.closed = False - self.terminated = False - self.waited_closed = False - - async def wait(self, *, check: bool, **kwargs: Any) -> _Completed: - assert kwargs == {"timeout": None} - del check - self.started.set() - if self.on_wait is not None: - self.on_wait() - if self.block: - await asyncio.Event().wait() - if self.wait_error is not None: - raise self.wait_error - return self.completed - - def close(self) -> None: - self.closed = True - - def terminate(self) -> None: - self.terminated = True - - async def wait_closed(self) -> None: - self.waited_closed = True - - -class _Connection: - def __init__( - self, - *, - closed: bool = False, - run_error: Exception | None = None, - process: _Process | None = None, - stall_open: bool = False, - open_error: Exception | None = None, - ) -> None: - self.closed = closed - self.run_error = run_error - self.process = process or _Process() - self.stall_open = stall_open - self.open_error = open_error - self.open_cancelled = False - self.commands: list[str] = [] - self.process_kwargs: dict[str, Any] = {} - - def is_closed(self) -> bool: - return self.closed - - def close(self) -> None: - self.closed = True - - async def wait_closed(self) -> None: - pass - - async def run(self, command: str, **kwargs: Any) -> _Completed: - del kwargs - self.commands.append(command) - if self.run_error is not None: - if isinstance(self.run_error, asyncssh.ConnectionLost): - self.closed = True - raise self.run_error - return _Completed() - - async def create_process(self, *args: object, **kwargs: Any) -> _Process: - self.process_kwargs = kwargs - self.commands.append(str(args[0])) - if self.stall_open: + from collections.abc import AsyncIterator + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="the workspace shell is POSIX") + +POWERSHELL = """\ +#!{python} +import base64, pathlib, re, sys + +script = base64.b64decode(sys.argv[-1]).decode("utf-16-le") +with open({log!r}, "a") as log: + log.write(script + "\\n") +literals = [text.replace("''", "'") for text in re.findall(r"'((?:[^']|'')*)'", script)] +path = literals[-1] +if "ReadAllBytes" in script: + print(base64.b64encode(pathlib.Path(path).read_bytes()).decode()) +elif "Get-ChildItem" in script: + print("\\n".join(sorted(entry.name for entry in pathlib.Path(path).iterdir()))) +elif "WriteAllBytes" in script: + pathlib.Path(path).write_bytes(b"") +else: + with open(path, "ab") as file: + file.write(base64.b64decode(literals[0])) +""" + + +@asynccontextmanager +async def shell( + tmp_path: Path, *, env: dict[str, str] | None = None +) -> AsyncIterator[tuple[SSHClient, FlakyRelay, Path]]: + """A client connected to a served workspace's ``ssh`` binding through a relay.""" + root = tmp_path / "workspace" + root.mkdir() + environment = Environment("ssh-faults") + environment.workspace(root, track_files=False, env=env) + async with served(environment) as control: + binding = control.binding("shell") + parts = urlsplit(binding.url) + assert parts.hostname is not None and parts.port is not None + async with relay(parts.hostname, parts.port) as flaky: + capability = replace(binding, url=f"ssh://{parts.username}@127.0.0.1:{flaky.port}") + client = await SSHClient.connect(capability) try: - await asyncio.Event().wait() - except asyncio.CancelledError: - self.open_cancelled = True - raise - if self.open_error is not None: - raise self.open_error - if self.run_error is not None: - if isinstance(self.run_error, asyncssh.ConnectionLost): - self.closed = True - raise self.run_error - return self.process - - -def _capability() -> Capability: - return Capability( - name="shell", - protocol="ssh/2", - url="ssh://workspace.example:2222", - params={"user": "agent", "client_key_path": "/tmp/key"}, - ) + yield client, flaky, root + finally: + await client.close() -def _client(connection: object) -> SSHClient: - return SSHClient(_capability(), cast("asyncssh.SSHClientConnection", connection)) +def written(path: Path) -> bool: + return path.is_file() and bool(path.read_text().strip()) -async def test_connect_keeps_tunneled_connection_active(monkeypatch: pytest.MonkeyPatch) -> None: - connection = cast("asyncssh.SSHClientConnection", object()) - connect = AsyncMock(return_value=connection) - monkeypatch.setattr(asyncssh, "connect", connect) - - client = await SSHClient.connect( - Capability(name="shell", protocol="ssh/2", url="ssh://sandbox.example:8765") - ) +def running(pid: int) -> bool: + """Whether ``pid`` is a live process (an unreaped zombie counts as gone).""" + stat = Path(f"/proc/{pid}/stat") + try: + return stat.read_text().split(") ", 1)[1][0] != "Z" + except FileNotFoundError: + return False - assert client.conn is connection - connect.assert_awaited_once_with( - host="sandbox.example", - port=8765, - username="agent", - client_keys=None, - known_hosts=None, - errors="replace", - keepalive_interval=15, - keepalive_count_max=4, - ) - - -async def test_connect_classifies_a_lost_handshake_as_retryable( - monkeypatch: pytest.MonkeyPatch, -) -> None: - connect = AsyncMock(side_effect=asyncssh.ConnectionLost("dropped")) - monkeypatch.setattr(SSHClient, "_connect", connect) - with pytest.raises(SSHConnectionError, match="failed during handshake"): - await SSHClient.connect(_capability()) +async def remote_pid(root: Path) -> int: + """The pid a command wrote to ``pid`` in the workspace.""" + await eventually(lambda: written(root / "pid")) + return int((root / "pid").read_text()) -async def test_run_does_not_replay_a_command_lost_in_flight( - monkeypatch: pytest.MonkeyPatch, -) -> None: - dropped = _Connection(run_error=asyncssh.ConnectionLost("dropped")) - replacement = _Connection() - client = _client(dropped) - reconnect = AsyncMock(return_value=replacement) - monkeypatch.setattr(client, "_connect", reconnect) +async def drop(client: SSHClient, flaky: FlakyRelay) -> None: + """Sever the relayed connection and wait until the client has seen it close.""" + flaky.sever() + await eventually(client.conn.is_closed) - with pytest.raises(SSHConnectionError, match="lost during operation"): - await client.run("apply-side-effect") - reconnect.assert_not_awaited() - assert dropped.commands == ["apply-side-effect"] +async def test_files_round_trip_through_the_exec_channel(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, _, root): + await client.write_text("empty.txt", "") + await client.write_text("it's here.txt", "héllo\nwörld\n") + (root / "raw.bin").write_bytes(b"\x00\xff\x10") - await client.run("next-command") - reconnect.assert_awaited_once_with(client.capability) - assert replacement.commands == ["next-command"] + assert (root / "empty.txt").read_text() == "" + assert await client.read_text("empty.txt") == "" + assert await client.read_text(str(root / "it's here.txt")) == "héllo\nwörld\n" + assert await client.read_bytes("raw.bin") == b"\x00\xff\x10" + assert await client.listdir(".") == ["empty.txt", "it's here.txt", "raw.bin"] -async def test_create_process_reconnects_before_opening_channel( - monkeypatch: pytest.MonkeyPatch, +async def test_windows_shells_move_files_as_base64_through_encoded_powershell( + tmp_path: Path, ) -> None: - dropped = _Connection(closed=True) - replacement = _Connection() - client = _client(dropped) - reconnect = AsyncMock(return_value=replacement) - monkeypatch.setattr(client, "_connect", reconnect) - - process = await client.create_process("bridge") - - assert process is replacement.process - reconnect.assert_awaited_once_with(client.capability) - assert dropped.commands == [] - assert replacement.commands == ["bridge"] - - -async def test_create_process_sends_only_connection_names() -> None: - transport = _Connection() - capability = _capability() - capability.params["process_connections"] = True - client = SSHClient(capability, cast("Any", transport)) - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-secret"}, - ) - - await client.create_process("agent", connections=(connection,)) - - assert transport.process_kwargs["env"] == {PROCESS_CONNECTIONS_REQUEST: '["inference"]'} - assert "scoped-secret" not in str(transport.process_kwargs) - - -async def test_create_process_rejects_connections_the_server_did_not_advertise() -> None: - client = _client(_Connection()) - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-secret"}, - ) - - with pytest.raises(ValueError, match="does not support"): - await client.create_process("agent", connections=(connection,)) - - -async def test_create_process_rejects_connections_for_another_capability() -> None: - transport = _Connection() - capability = _capability() - capability.params["process_connections"] = True - client = SSHClient(capability, cast("Any", transport)) - connection = Connection( - name="inference", - capability="other-workspace", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-secret"}, - ) - - with pytest.raises(ValueError, match="do not belong"): - await client.create_process("agent", connections=(connection,)) - - -async def test_create_process_preserves_reconnect_failure( - monkeypatch: pytest.MonkeyPatch, + tools = tmp_path / "tools" + tools.mkdir() + log = tmp_path / "powershell.log" + stub = tools / "powershell" + stub.write_text(POWERSHELL.format(python=sys.executable, log=str(log)), encoding="utf-8") + stub.chmod(0o755) + async with shell(tmp_path, env={"PATH": f"{tools}{os.pathsep}{os.environ['PATH']}"}) as ( + client, + _, + root, + ): + windows_client = SSHClient( + replace(client.capability, params={**client.capability.params, "shell": "powershell"}), + client.conn, + ) + content = "x" * 7000 + "é" + target = str(root / "it's.txt") + + await windows_client.write_text(target, content) + read = await windows_client.read_text(target) + listing = await windows_client.listdir(str(root)) + + assert (read, listing) == (content, ["it's.txt"]) + scripts = log.read_text("utf-8").splitlines() + assert [script.split("(", 1)[0] for script in scripts] == [ + "[IO.File]::WriteAllBytes", + "$b=[Convert]::FromBase64String", + "$b=[Convert]::FromBase64String", + "[Convert]::ToBase64String", + "Get-ChildItem -Force -Name -LiteralPath '" + str(root) + "'", + ] + assert f"'{root}/it''s.txt'" in scripts[0] + + +async def test_a_timed_out_command_is_killed_and_the_next_one_runs(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, _, root): + with pytest.raises(TimeoutError): + await client.run("echo $$ > pid; exec sleep 30", timeout=0.5) + pid = await remote_pid(root) + await eventually(lambda: not running(pid)) + + result = await client.run("echo next", timeout=10) + + assert result.stdout == "next\n" + + +async def test_a_cancelled_command_kills_its_remote_process(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, _, root): + command = asyncio.create_task(client.run("echo $$ > pid; exec sleep 30")) + pid = await remote_pid(root) + + command.cancel() + with pytest.raises(asyncio.CancelledError): + await command + + await eventually(lambda: not running(pid)) + + +async def test_a_command_lost_in_flight_is_not_replayed_and_the_next_one_reconnects( + tmp_path: Path, ) -> None: - client = _client(_Connection(closed=True)) - reconnect = AsyncMock(side_effect=OSError("unreachable")) - monkeypatch.setattr(client, "_connect", reconnect) - monkeypatch.setattr("hud.capabilities.ssh.asyncio.sleep", AsyncMock()) + async with shell(tmp_path) as (client, flaky, root): + command = asyncio.create_task(client.run("echo ran >> marker; exec sleep 30")) + await eventually(lambda: written(root / "marker")) - with pytest.raises(SSHConnectionError, match="reconnect failed after 3 attempts"): - await client.create_process("bridge") + flaky.sever() + with pytest.raises(ConnectionError, match="SSH command ended without an exit status"): + await command + result = await client.run("cat marker", timeout=10) + process = await client.create_process("echo opened") + opened = await process.wait() + assert result.stdout == "ran\n" + assert opened.stdout == b"opened\n" + assert flaky.accepted == 2 -async def test_run_classifies_rejected_session_as_connection_error() -> None: - connection = _Connection( - open_error=asyncssh.ChannelOpenError(asyncssh.OPEN_RESOURCE_SHORTAGE, "busy") - ) - client = _client(connection) - with pytest.raises(SSHConnectionError, match="rejected the session"): - await client.run("echo never", timeout=1) +async def test_reconnecting_gives_up_after_three_refused_attempts(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, flaky, _): + flaky.refuse(3) + await drop(client, flaky) - assert connection.closed is False + with pytest.raises(ConnectionError, match="SSH reconnect failed after 3 attempts"): + await client.run("true") + assert flaky.accepted == 4 -async def test_run_timeout_includes_opening_the_process() -> None: - connection = _Connection(stall_open=True) - client = _client(connection) - with pytest.raises(TimeoutError): - await client.run("echo never", timeout=0.01) +async def test_a_command_timeout_covers_a_stalled_reconnect(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, flaky, _): + flaky.stall() + await drop(client, flaky) - assert connection.open_cancelled is True + with pytest.raises(TimeoutError): + await client.run("true", timeout=0.5) + flaky.release() + result = await client.run("echo back", timeout=10) + assert result.stdout == "back\n" -async def test_run_timeout_includes_reconnecting( - monkeypatch: pytest.MonkeyPatch, -) -> None: - connection = _Connection(closed=True) - client = _client(connection) - cancelled = False - async def reconnect(capability: Capability) -> asyncssh.SSHClientConnection: - nonlocal cancelled - assert capability is client.capability - try: - await asyncio.Event().wait() - except asyncio.CancelledError: - cancelled = True - raise - raise AssertionError("reconnect unexpectedly completed") +async def test_closing_during_a_reconnect_discards_the_new_connection(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, flaky, _): + flaky.stall() + await drop(client, flaky) + command = asyncio.create_task(client.run("true")) + await flaky.wait_until(stalled=1) - monkeypatch.setattr(client, "_connect", reconnect) + closing = asyncio.create_task(client.close()) + await asyncio.sleep(0) + flaky.release() + with pytest.raises(ConnectionError, match="SSH client is closed"): + await command + await closing - with pytest.raises(TimeoutError): - await client.run("echo never", timeout=0.01) + await flaky.wait_until(accepted=2) + await eventually(lambda: flaky.open == 0) - assert cancelled is True +async def test_a_handshake_the_server_drops_is_a_connection_error(tmp_path: Path) -> None: + async with shell(tmp_path) as (client, flaky, _): + flaky.refuse(1) -async def test_run_preserves_timeout_when_the_connection_closes() -> None: - process = _Process(wait_error=TimeoutError()) - connection = _Connection(process=process) - process.on_wait = lambda: setattr(connection, "closed", True) - client = _client(connection) + with pytest.raises(ConnectionError, match="SSH connection failed during handshake"): + await SSHClient.connect(client.capability) - with pytest.raises(TimeoutError): - await client.run("echo never", timeout=1) - -@pytest.mark.parametrize("command_timeout", [None, 300]) -async def test_run_cancellation_terminates_the_remote_process( - command_timeout: float | None, +@pytest.mark.parametrize( + ("advertised", "connection_capability", "message"), + [ + (False, "shell", "SSH capability does not support process-bound connections"), + (True, "browser", "connections do not belong to SSH capability 'shell': inference"), + ], +) +async def test_process_bound_connections_must_be_advertised_and_belong_to_the_shell( + advertised: bool, connection_capability: str, message: str, tmp_path: Path ) -> None: - connection = _Connection(process=_Process(block=True)) - client = _client(connection) - run = asyncio.create_task(client.run("echo never", timeout=command_timeout)) - await connection.process.started.wait() - - run.cancel() - - with pytest.raises(asyncio.CancelledError): - await run - assert connection.process.terminated is True - assert connection.process.closed is False - assert connection.process.waited_closed is True - - -async def test_run_rejects_a_completed_process_without_a_returncode() -> None: - completed = _Completed() - completed.returncode = None - connection = _Connection(process=_Process(completed=completed)) - client = _client(connection) - - with pytest.raises(SSHConnectionError, match="without an exit status"): - await client.run("echo incomplete", timeout=1) - - assert connection.closed is True - - -async def test_windows_write_uses_one_timeout_budget( - monkeypatch: pytest.MonkeyPatch, -) -> None: - client = SSHClient( - Capability( - name="shell", - protocol="ssh/2", - url="ssh://workspace.example:2222", - params={"shell": "powershell"}, - ), - cast("asyncssh.SSHClientConnection", _Connection()), - ) - timeouts: list[float] = [] - - async def run(*args: object, **kwargs: Any) -> _Completed: - del args - timeout = kwargs["timeout"] - assert isinstance(timeout, float) - timeouts.append(timeout) - await asyncio.sleep(0.01) - return _Completed() - - monkeypatch.setattr(client, "run", run) - - await client.write_text("C:\\file.txt", "x" * 7000, timeout_s=1) - - assert len(timeouts) == 3 - assert timeouts[0] > timeouts[1] > timeouts[2] - - -async def test_posix_empty_write_supplies_eof(monkeypatch: pytest.MonkeyPatch) -> None: - client = _client(_Connection()) - run = AsyncMock(return_value=_Completed()) - monkeypatch.setattr(client, "run", run) - - await client.write_text("empty.txt", "") - - run.assert_awaited_once_with( - "cat > empty.txt", - check=True, - timeout=None, - input="", - stdin=asyncssh.DEVNULL, + connection = Connection( + name="inference", + capability=connection_capability, + url="https://inference.example", + headers={"Authorization": "Bearer secret"}, ) - - -async def test_close_during_reconnect_discards_the_replacement( - monkeypatch: pytest.MonkeyPatch, -) -> None: - dropped = _Connection(closed=True) - replacement = _Connection() - reconnect_started = asyncio.Event() - allow_reconnect = asyncio.Event() - - async def reconnect(capability: Capability) -> asyncssh.SSHClientConnection: - assert capability is client.capability - reconnect_started.set() - await allow_reconnect.wait() - return cast("asyncssh.SSHClientConnection", replacement) - - client = _client(dropped) - monkeypatch.setattr(client, "_connect", reconnect) - - run = asyncio.create_task(client.run("echo never")) - await reconnect_started.wait() - close = asyncio.create_task(client.close()) - await asyncio.sleep(0) - allow_reconnect.set() - - with pytest.raises(SSHConnectionError, match="client is closed"): - await run - await close - - assert replacement.closed is True - assert replacement.commands == [] - - -async def test_reconnect_exhaustion_raises_connection_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - client = _client(_Connection(closed=True)) - reconnect = AsyncMock(side_effect=OSError("unreachable")) - sleep = AsyncMock() - monkeypatch.setattr(client, "_connect", reconnect) - monkeypatch.setattr("hud.capabilities.ssh.asyncio.sleep", sleep) - - with pytest.raises(SSHConnectionError, match="failed after 3 attempts"): - await client.run("echo never") - - assert reconnect.await_count == 3 - assert [call.args for call in sleep.await_args_list] == [(0.25,), (0.5,)] + async with shell(tmp_path) as (client, _, _): + bound = SSHClient( + replace( + client.capability, + params={**client.capability.params, "process_connections": advertised}, + ), + client.conn, + ) + + with pytest.raises(ValueError, match=message): + await bound.create_process("true", connections=(connection,)) From 9ee3bfbf8687a6ff586bb820881507b0acc4b21f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 060/248] test: drive RFBClient against the harness VNC server The RFB tests built the client with object.__new__, set its private connection, patched Image.fromarray and asserted the mock's save kwargs. They now connect to fake_screen and decode what comes back: PNG and WebP screenshots carry the framebuffer and the right mime type, a stream that drops mid-screenshot reconnects and still returns an image, and a capability without a port is refused. fake_screen gains drop_on_update to close the connection instead of answering one update. --- tests/capabilities/test_rfb.py | 125 ++++++++++++++------------------- tests/harness/rfb.py | 15 +++- 2 files changed, 67 insertions(+), 73 deletions(-) diff --git a/tests/capabilities/test_rfb.py b/tests/capabilities/test_rfb.py index 72ff77017..e49035613 100644 --- a/tests/capabilities/test_rfb.py +++ b/tests/capabilities/test_rfb.py @@ -1,76 +1,59 @@ +"""``RFBClient`` against the harness VNC server.""" + from __future__ import annotations -from types import SimpleNamespace -from typing import Any -from unittest.mock import ANY, AsyncMock, Mock, patch +import io -import mcp.types as mcp_types +import pytest from PIL import Image -from hud.agents.tools.rfb import RFBTool -from hud.capabilities.rfb import RFBClient, WebPScreenshotEncoding - - -class RecordingRFBTool(RFBTool): - name = "rfb-test" - client: Any - - def __init__(self) -> None: - self.screenshot_encoding = WebPScreenshotEncoding(quality=42) - self.client = SimpleNamespace( - screenshot_png=AsyncMock(return_value=(b"webp", "image/webp")), - ) - - async def execute(self, arguments: dict[str, Any]) -> Any: - del arguments - raise NotImplementedError - - def to_params(self) -> Any: - raise NotImplementedError - - -async def test_screenshot_uses_requested_webp_mime_type() -> None: - client = object.__new__(RFBClient) - object.__setattr__( - client, - "_conn", - SimpleNamespace( - screenshot=AsyncMock(return_value=object()), - ), - ) - - save = Mock(side_effect=lambda buffer, **_kwargs: buffer.write(b"webp")) - with patch("hud.capabilities.rfb.Image.fromarray", return_value=SimpleNamespace(save=save)): - data, mime_type = await client.screenshot_png(WebPScreenshotEncoding(quality=42)) - - assert mime_type == "image/webp" - assert data == b"webp" - save.assert_called_once_with(ANY, format="WEBP", quality=42) - - -async def test_screenshot_uses_requested_png_mime_type() -> None: - client = object.__new__(RFBClient) - object.__setattr__( - client, - "_conn", - SimpleNamespace( - screenshot=AsyncMock(return_value=object()), - ), - ) - - with patch("hud.capabilities.rfb.Image.fromarray", return_value=Image.new("RGB", (8, 8))): - data, mime_type = await client.screenshot_png("image/png") - - assert mime_type == "image/png" - assert data.startswith(b"\x89PNG") - - -async def test_screenshot_reports_encoded_mime_type() -> None: - tool = RecordingRFBTool() - - result = await tool.screenshot() - - image = result.content[0] - assert isinstance(image, mcp_types.ImageContent) - assert image.mimeType == "image/webp" - tool.client.screenshot_png.assert_awaited_once_with(tool.screenshot_encoding) +from hud.capabilities import Capability, RFBClient +from hud.capabilities.rfb import ScreenshotMimeType, WebPScreenshotEncoding +from tests.harness import fake_screen + +COLOR = (10, 200, 30) + + +@pytest.mark.parametrize( + ("encoding", "mime_type", "image_format"), + [ + ("image/png", "image/png", "PNG"), + ("image/webp", "image/webp", "WEBP"), + (WebPScreenshotEncoding(quality=100), "image/webp", "WEBP"), + ], +) +async def test_a_screenshot_encodes_the_framebuffer_as_requested( + encoding: ScreenshotMimeType | WebPScreenshotEncoding, mime_type: str, image_format: str +) -> None: + async with fake_screen(width=8, height=6, color=COLOR) as screen: + client = await RFBClient.connect(Capability.rfb(url=screen.url)) + try: + data, encoded_as = await client.screenshot_png(encoding) + finally: + await client.close() + + image = Image.open(io.BytesIO(data)) + assert (encoded_as, image.format, image.size) == (mime_type, image_format, (8, 6)) + colors = image.convert("RGB").getcolors() + assert colors is not None + ((count, pixel),) = colors + assert count == 48 + assert max(abs(channel - expected) for channel, expected in zip(pixel, COLOR, strict=True)) <= 4 + + +async def test_a_screenshot_reconnects_when_the_stream_drops() -> None: + # The first update is the warm-up on connect; the second is dropped. + async with fake_screen(width=4, height=4, color=COLOR, drop_on_update=2) as screen: + client = await RFBClient.connect(Capability.rfb(url=screen.url)) + try: + data, _ = await client.screenshot_png() + finally: + await client.close() + + assert screen.connections == 2 + assert Image.open(io.BytesIO(data)).convert("RGB").getpixel((0, 0)) == COLOR + + +async def test_connecting_needs_a_host_and_port() -> None: + with pytest.raises(ValueError, match="rfb capability missing host or port"): + await RFBClient.connect(Capability(name="screen", protocol="rfb/3.8", url="rfb://host")) diff --git a/tests/harness/rfb.py b/tests/harness/rfb.py index aa7c0c089..cb86f6424 100644 --- a/tests/harness/rfb.py +++ b/tests/harness/rfb.py @@ -40,6 +40,10 @@ class FakeScreen: events: list[KeyEvent | PointerEvent] = field(default_factory=list) connections: int = 0 port: int = 0 + #: Close the connection instead of answering this framebuffer update request + #: (counted across connections, from 1); 0 never drops. + drop_on_update: int = 0 + updates: int = 0 @property def url(self) -> str: @@ -74,6 +78,9 @@ async def _session(self, reader: asyncio.StreamReader, writer: asyncio.StreamWri await reader.readexactly(4 * count) elif kind == 3: # FramebufferUpdateRequest await reader.readexactly(9) + self.updates += 1 + if self.updates == self.drop_on_update: + return pixel = bytes([*self.color, 255]) writer.write(struct.pack(">BxH", 0, 1)) writer.write(struct.pack(">HHHHi", 0, 0, self.width, self.height, 0)) @@ -98,10 +105,14 @@ async def _session(self, reader: asyncio.StreamReader, writer: asyncio.StreamWri @asynccontextmanager async def fake_screen( - width: int = 64, height: int = 48, color: tuple[int, int, int] = (40, 120, 200) + width: int = 64, + height: int = 48, + color: tuple[int, int, int] = (40, 120, 200), + *, + drop_on_update: int = 0, ) -> AsyncIterator[FakeScreen]: """Serve a :class:`FakeScreen` on loopback for the duration of the block.""" - screen = FakeScreen(width, height, color) + screen = FakeScreen(width, height, color, drop_on_update=drop_on_update) server = await asyncio.start_server(screen._session, "127.0.0.1", 0) screen.port = server.sockets[0].getsockname()[1] async with server: From 7a254b1feeef42610de8def8d8fbeee87ea7712f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 061/248] test: add a fake DevTools endpoint to the harness fake_browser() serves Chrome's remote-debugging surface on one loopback port: target discovery over GET /json and PUT /json/new, and a WebSocket per page that records every command and answers from scripted replies (a result, an error frame, a closed socket, or no answer), preceded by an unsolicited event. --- tests/harness/__init__.py | 4 ++ tests/harness/cdp.py | 127 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 tests/harness/cdp.py diff --git a/tests/harness/__init__.py b/tests/harness/__init__.py index 295fcf462..8cec631c1 100644 --- a/tests/harness/__init__.py +++ b/tests/harness/__init__.py @@ -6,6 +6,7 @@ live in ``tests/conftest.py``. """ +from .cdp import Command, FakeBrowser, fake_browser from .cli import Hud, Result, scrub from .config import HudEnv from .docker import FakeDocker @@ -31,6 +32,8 @@ __all__ = [ "ROBOT_STEP_SCHEMA", + "Command", + "FakeBrowser", "FakeDocker", "FakeScreen", "FakeServices", @@ -53,6 +56,7 @@ "computer_call", "eventually", "fail", + "fake_browser", "fake_screen", "interrupted", "relay", diff --git a/tests/harness/cdp.py b/tests/harness/cdp.py new file mode 100644 index 000000000..2915b162f --- /dev/null +++ b/tests/harness/cdp.py @@ -0,0 +1,127 @@ +"""A fake Chromium DevTools endpoint: target discovery over HTTP and scripted CDP replies. + +:func:`fake_browser` serves, on one loopback port like Chrome's +``--remote-debugging-port``, ``GET /json`` (the page targets plus a service +worker), ``PUT /json/new`` (a new page), and a WebSocket per page at +``/devtools/page/``. Each command frame is recorded; its reply comes from +``replies[method]``: ``{"result": ...}`` or ``{"error": ...}`` is sent back, +``{"close": True}`` closes the socket instead, and ``{"hold": True}`` never +answers. Unscripted methods answer ``{"result": {}}``. Before every reply the +browser sends an unsolicited ``Page.frameNavigated`` event. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +import uvicorn +from starlette.applications import Starlette +from starlette.responses import JSONResponse +from starlette.routing import Route, WebSocketRoute + +from .scenario import eventually + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from starlette.requests import Request + from starlette.websockets import WebSocket + + +@dataclass(frozen=True) +class Command: + target: str + method: str + params: dict[str, Any] + + +@dataclass +class FakeBrowser: + """The browser side. ``http`` and ``commands`` hold what the client sent.""" + + pages: list[str] = field(default_factory=lambda: ["page-1"]) + replies: dict[str, dict[str, Any]] = field(default_factory=dict) + #: Whether ``PUT /json/new`` creates a page; otherwise it answers ``{}``. + can_create: bool = True + http: list[str] = field(default_factory=list) + commands: list[Command] = field(default_factory=list) + port: int = 0 + + @property + def url(self) -> str: + return f"ws://127.0.0.1:{self.port}" + + def target(self, page: str) -> dict[str, str]: + return { + "id": page, + "type": "page", + "webSocketDebuggerUrl": f"{self.url}/devtools/page/{page}", + } + + async def _list(self, request: Request) -> JSONResponse: + self.http.append(f"{request.method} {request.url.path}") + worker = {"id": "worker", "type": "service_worker", "webSocketDebuggerUrl": "ws://x"} + return JSONResponse([worker, *(self.target(page) for page in self.pages)]) + + async def _new(self, request: Request) -> JSONResponse: + self.http.append(f"{request.method} {request.url.path}?{request.url.query}") + if not self.can_create: + return JSONResponse({}) + page = f"new-{len(self.pages) + 1}" + self.pages.append(page) + return JSONResponse(self.target(page)) + + async def _session(self, websocket: WebSocket) -> None: + page = websocket.path_params["page"] + await websocket.accept() + while True: + frame = json.loads(await websocket.receive_text()) + self.commands.append(Command(page, frame["method"], frame.get("params", {}))) + reply = self.replies.get(frame["method"], {"result": {}}) + if reply.get("close"): + await websocket.close() + return + if reply.get("hold"): + continue + event = {"method": "Page.frameNavigated", "params": {"frame": {"id": page}}} + await websocket.send_text(json.dumps(event)) + await websocket.send_text(json.dumps({"id": frame["id"], **reply})) + + +@asynccontextmanager +async def fake_browser( + *, + pages: list[str] | None = None, + replies: dict[str, dict[str, Any]] | None = None, + can_create: bool = True, +) -> AsyncIterator[FakeBrowser]: + """Serve a :class:`FakeBrowser` on a loopback port for the duration of the block.""" + browser = FakeBrowser( + pages=["page-1"] if pages is None else list(pages), + replies=replies or {}, + can_create=can_create, + ) + app = Starlette( + routes=[ + Route("/json", browser._list, methods=["GET"]), + Route("/json/new", browser._new, methods=["PUT"]), + WebSocketRoute("/devtools/page/{page}", browser._session), + ] + ) + server = uvicorn.Server( + uvicorn.Config( + app, host="127.0.0.1", port=0, log_level="error", lifespan="off", ws="websockets-sansio" + ) + ) + serving = asyncio.create_task(server.serve()) + await eventually(lambda: server.started) + browser.port = server.servers[0].sockets[0].getsockname()[1] + try: + yield browser + finally: + server.should_exit = True + await serving From b81d57cbe6f05771f6efdd9443dc8360ea83e98f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 062/248] test: cover CDPClient against the fake DevTools endpoint CDPClient had no tests. The scenario pins how connect picks its page target (the first page from /json, a page created with /json/new, an explicit target id, a verbatim /devtools/ URL), that it enables the Page, Runtime and DOM domains on that target, that it refuses a browser with no page and a capability with no port, that send returns command results past unsolicited events, and that pending commands fail with ConnectionError when the browser closes the socket. --- tests/capabilities/test_cdp.py | 88 ++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/capabilities/test_cdp.py diff --git a/tests/capabilities/test_cdp.py b/tests/capabilities/test_cdp.py new file mode 100644 index 000000000..c23339829 --- /dev/null +++ b/tests/capabilities/test_cdp.py @@ -0,0 +1,88 @@ +"""``CDPClient`` against the harness DevTools endpoint.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from hud.capabilities import Capability, CDPClient +from tests.harness import Command, fake_browser + +ENABLED = ["Page.enable", "Runtime.enable", "DOM.enable"] + + +@pytest.mark.parametrize( + ("pages", "target_id", "devtools_path", "http", "connected_to"), + [ + (["page-1", "page-2"], None, False, ["GET /json"], "page-1"), + ([], None, False, ["GET /json", "PUT /json/new?about:blank"], "new-1"), + (["page-1", "page-2"], "page-2", False, [], "page-2"), + (["page-1", "page-2"], None, True, [], "page-2"), + ], + ids=["first page", "a new page", "an explicit target", "a devtools url"], +) +async def test_connect_attaches_to_a_page_target_and_enables_its_domains( + pages: list[str], + target_id: str | None, + devtools_path: bool, + http: list[str], + connected_to: str, +) -> None: + async with fake_browser(pages=pages) as browser: + url = f"{browser.url}/devtools/page/page-2" if devtools_path else browser.url + client = await CDPClient.connect(Capability.cdp(url=url, target_id=target_id)) + await client.close() + + assert browser.http == http + assert [(command.target, command.method) for command in browser.commands] == [ + (connected_to, method) for method in ENABLED + ] + + +async def test_connect_fails_when_the_browser_offers_no_page_target() -> None: + async with fake_browser(pages=[], can_create=False) as browser: + with pytest.raises(ValueError, match=r"no CDP page target available at 127\.0\.0\.1:"): + await CDPClient.connect(Capability.cdp(url=browser.url)) + + +async def test_connect_needs_a_host_and_port() -> None: + with pytest.raises(ValueError, match="cdp capability missing host or port"): + await CDPClient.connect(Capability(name="browser", protocol="cdp/1.3", url="ws://host")) + + +@pytest.mark.parametrize( + ("reply", "result"), + [({"result": {"frameId": "f-1"}}, {"frameId": "f-1"}), ({}, {})], + ids=["a result", "an empty reply"], +) +async def test_send_returns_the_command_result( + reply: dict[str, Any], result: dict[str, Any] +) -> None: + async with fake_browser(replies={"Page.navigate": reply}) as browser: + client = await CDPClient.connect(Capability.cdp(url=browser.url)) + try: + returned = await client.send("Page.navigate", {"url": "https://example.com"}) + finally: + await client.close() + + assert returned == result + assert browser.commands[-1] == Command( + "page-1", "Page.navigate", {"url": "https://example.com"} + ) + + +async def test_pending_commands_fail_when_the_browser_closes_the_socket() -> None: + replies = {"Runtime.evaluate": {"hold": True}, "Browser.crash": {"close": True}} + async with fake_browser(replies=replies) as browser: + client = await CDPClient.connect(Capability.cdp(url=browser.url)) + try: + waiting = asyncio.create_task(client.send("Runtime.evaluate", {"expression": "1"})) + await asyncio.sleep(0) + with pytest.raises(ConnectionError, match="CDP connection closed"): + await client.send("Browser.crash") + with pytest.raises(ConnectionError, match="CDP connection closed"): + await waiting + finally: + await client.close() From 538521441ea67c57a0538d10ede69e95eda327fe Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 063/248] fix: name the failed command in CDPError CDPError took its method from the reply frame, but Chrome's error replies carry only an id and an error, so every error read "CDP '' failed [...]". The reader now hands send() the whole reply and send() raises CDPError with the method it sent. --- hud/capabilities/cdp.py | 10 +++++----- tests/capabilities/test_cdp.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/hud/capabilities/cdp.py b/hud/capabilities/cdp.py index 592e17b60..1a665cdfe 100644 --- a/hud/capabilities/cdp.py +++ b/hud/capabilities/cdp.py @@ -113,9 +113,12 @@ async def send(self, method: str, params: dict[str, Any] | None = None) -> dict[ self._pending[msg_id] = future await self._ws.send(json.dumps({"id": msg_id, "method": method, "params": params or {}})) try: - return await future + reply = await future finally: self._pending.pop(msg_id, None) + if "error" in reply: + raise CDPError(method, reply["error"]) + return reply.get("result", {}) async def _read_loop(self) -> None: try: @@ -125,10 +128,7 @@ async def _read_loop(self) -> None: future = self._pending.get(msg_id) if msg_id is not None else None if future is None or future.done(): continue # protocol event (no waiter) — ignored for now - if "error" in msg: - future.set_exception(CDPError(str(msg.get("method", "")), msg["error"])) - else: - future.set_result(msg.get("result", {})) + future.set_result(msg) except (ConnectionClosed, OSError) as exc: LOGGER.debug("CDP read loop ended: %s", exc) finally: diff --git a/tests/capabilities/test_cdp.py b/tests/capabilities/test_cdp.py index c23339829..9fa2e768a 100644 --- a/tests/capabilities/test_cdp.py +++ b/tests/capabilities/test_cdp.py @@ -8,6 +8,7 @@ import pytest from hud.capabilities import Capability, CDPClient +from hud.capabilities.cdp import CDPError from tests.harness import Command, fake_browser ENABLED = ["Page.enable", "Runtime.enable", "DOM.enable"] @@ -73,6 +74,23 @@ async def test_send_returns_the_command_result( ) +async def test_an_error_reply_raises_cdp_error_naming_the_failed_command() -> None: + error = {"code": -32000, "message": "Cannot navigate to invalid URL"} + async with fake_browser(replies={"Page.navigate": {"error": error}}) as browser: + client = await CDPClient.connect(Capability.cdp(url=browser.url)) + try: + with pytest.raises(CDPError) as raised: + await client.send("Page.navigate", {"url": "nope"}) + finally: + await client.close() + + assert (raised.value.code, raised.value.message, str(raised.value)) == ( + -32000, + "Cannot navigate to invalid URL", + "CDP 'Page.navigate' failed [-32000]: Cannot navigate to invalid URL", + ) + + async def test_pending_commands_fail_when_the_browser_closes_the_socket() -> None: replies = {"Runtime.evaluate": {"hold": True}, "Browser.crash": {"close": True}} async with fake_browser(replies=replies) as browser: From 695ffda0f535ebaf4c6d97fce4d5a525452cab51 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 064/248] test: pin capability URL normalization and Connection validation Nothing covered the factories' defaults: ssh on :22, cdp on ws and :9222, robot on ws and :9091, rfb on 5900 plus the display unless the URL names a port, mcp on :80 or :443 with the transport its scheme implies, IPv6 hosts bracketed and userinfo kept, and the URLs and transport pairings mcp refuses. Manifest round trips and the default MCP transport for a binding without params join them. Connection validation, the guard on what reaches a bound process, is a table of names, URLs and header values it must refuse, CR/LF injection included, plus the typed wire form and the credential-free client URL. --- tests/capabilities/test_capability.py | 257 ++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 tests/capabilities/test_capability.py diff --git a/tests/capabilities/test_capability.py b/tests/capabilities/test_capability.py new file mode 100644 index 000000000..388519f47 --- /dev/null +++ b/tests/capabilities/test_capability.py @@ -0,0 +1,257 @@ +"""Capability factories, manifest round trips, and ``Connection`` validation.""" + +from __future__ import annotations + +import sys +from typing import Any + +import pytest +from dirty_equals import IsStr + +from hud.capabilities import Capability, Connection + +DEFAULT_SHELL = "cmd" if sys.platform == "win32" else "bash" + + +@pytest.mark.parametrize( + ("built", "url", "params"), + [ + ( + lambda: Capability.ssh(url="workspace", host_pubkey="key"), + "ssh://workspace:22", + {"user": "agent", "host_pubkey": "key", "shell": DEFAULT_SHELL}, + ), + ( + lambda: Capability.ssh( + url="ssh://root@[::1]", + user="root", + host_pubkey="key", + client_key="secret", + shell="powershell", + cwd="/work", + isolation="bwrap", + process_connections=True, + ), + "ssh://root@[::1]:22", + { + "user": "root", + "host_pubkey": "key", + "shell": "powershell", + "client_key": "secret", + "cwd": "/work", + "isolation": "bwrap", + "process_connections": True, + }, + ), + ( + lambda: Capability.ssh(url="host:2222", host_pubkey="key", client_key_path="/k"), + "ssh://host:2222", + { + "user": "agent", + "host_pubkey": "key", + "shell": DEFAULT_SHELL, + "client_key_path": "/k", + }, + ), + (lambda: Capability.cdp(url="localhost"), "ws://localhost:9222", {}), + ( + lambda: Capability.cdp(url="ws://browser:9333/devtools/page/a", target_id="a"), + "ws://browser:9333/devtools/page/a", + {"target_id": "a"}, + ), + ( + lambda: Capability.robot(url="sim", contract={"robot_type": "arm"}), + "ws://sim:9091", + {"contract": {"robot_type": "arm"}}, + ), + (lambda: Capability.rfb(url="desktop"), "rfb://desktop:5900", {"display": 0}), + ( + lambda: Capability.rfb(url="desktop", display=2, password="pw"), + "rfb://desktop:5902", + {"display": 2, "password": "pw"}, + ), + ( + lambda: Capability.rfb(url="rfb://desktop:5999", display=2), + "rfb://desktop:5999", + {"display": 2}, + ), + ( + lambda: Capability.mcp(url="http://tools/mcp"), + "http://tools:80/mcp", + {"transport": "streamable-http"}, + ), + ( + lambda: Capability.mcp(url="https://tools/sse", transport="sse", auth_token="t"), + "https://tools:443/sse", + {"transport": "sse", "auth_token": "t"}, + ), + ( + lambda: Capability.mcp(url="tools"), + "ws://tools:80", + {"transport": "websocket"}, + ), + (lambda: Capability.mcp(url="wss://tools"), "wss://tools:443", {"transport": "websocket"}), + ], + ids=[ + "ssh defaults", + "ssh everything over ipv6", + "ssh explicit port and key path", + "cdp defaults", + "cdp devtools url", + "robot defaults", + "rfb display 0", + "rfb display 2", + "rfb explicit port wins", + "mcp http", + "mcp https sse", + "mcp bare host", + "mcp wss", + ], +) +def test_factories_normalize_urls_and_params(built: Any, url: str, params: dict[str, Any]) -> None: + capability = built() + + assert (capability.url, capability.params) == (url, params) + assert Capability.from_manifest(capability.to_manifest()) == capability + + +@pytest.mark.parametrize( + ("build", "message"), + [ + ( + lambda: Capability.mcp(url="stdio:server"), + "only ws/wss/http/https URLs are supported, got 'stdio'", + ), + ( + lambda: Capability.mcp(url="ftp://tools"), + "only ws/wss/http/https URLs are supported, got 'ftp'", + ), + ( + lambda: Capability.mcp(url="http://tools", transport="websocket"), + "websocket transport requires a ws:// or wss:// URL", + ), + ( + lambda: Capability.mcp(url="ws://tools", transport="sse"), + "sse transport requires an http:// or https:// URL", + ), + (lambda: Capability.ssh(url="ssh://", host_pubkey="key"), "invalid URL \\(no host\\)"), + ], + ids=["mcp stdio", "mcp ftp", "mcp websocket over http", "mcp sse over ws", "no host"], +) +def test_factories_refuse_urls_their_protocol_cannot_reach(build: Any, message: str) -> None: + with pytest.raises(ValueError, match=message): + build() + + +@pytest.mark.parametrize( + ("manifest", "params"), + [ + ( + {"name": "t", "protocol": "mcp/2025-11-25", "url": "ws://t:80"}, + {"transport": "websocket"}, + ), + ( + {"name": "t", "protocol": "mcp/2025-11-25", "url": "https://t:443", "params": None}, + {"transport": "streamable-http"}, + ), + ( + { + "name": "t", + "protocol": "mcp/2025-11-25", + "url": "http://t:80", + "params": {"transport": "sse"}, + }, + {"transport": "sse"}, + ), + ({"name": "s", "protocol": "ssh/2", "url": "ssh://s:22"}, {}), + ], + ids=["mcp over ws", "mcp over https", "mcp with its transport", "not mcp"], +) +def test_a_manifest_binding_without_params_gets_the_default_mcp_transport( + manifest: dict[str, Any], params: dict[str, Any] +) -> None: + assert Capability.from_manifest(manifest).params == params + + +def connection(**overrides: Any) -> Connection: + fields: dict[str, Any] = { + "name": "inference", + "capability": "shell", + "url": "https://inference.example/v1", + "headers": {"Authorization": "Bearer secret"}, + } + return Connection(**{**fields, **overrides}) + + +def test_a_connection_carries_its_headers_only_on_the_wire() -> None: + headers = {"Authorization": "Bearer secret"} + bound = connection(headers=headers) + headers["Authorization"] = "Bearer changed" + + assert bound.to_wire() == { + "name": "inference", + "capability": "shell", + "url": "https://inference.example/v1", + "headers": {"Authorization": "Bearer secret"}, + } + assert Connection.from_wire(bound.to_wire()) == bound + assert bound.client_url == IsStr(regex=r"http://inference-[0-9a-f]{12}\.hud\.invalid/v1") + assert "secret" not in repr(bound) and "secret" not in bound.client_url + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"name": "Inference"}, "connection name must start with a lowercase letter"), + ({"name": "1-inference"}, "connection name must start with a lowercase letter"), + ({"name": "a" * 64}, "connection name must start with a lowercase letter"), + ({"capability": ""}, "connection capability must not be empty or padded"), + ({"capability": " shell"}, "connection capability must not be empty or padded"), + ({"url": "ftp://inference.example"}, "connection url must be an HTTP\\(S\\) URL"), + ({"url": "https://"}, "connection url must be an HTTP\\(S\\) URL"), + ({"url": "https://user:pw@inference.example"}, "must not contain credentials"), + ({"url": "https://inference.example/v1?key=1"}, "must not contain a query or fragment"), + ({"url": "https://inference.example/v1#top"}, "must not contain a query or fragment"), + ({"headers": {}}, "connection headers must not be empty"), + ({"headers": {"Bad Header": "x"}}, "invalid connection header name: 'Bad Header'"), + ({"headers": {"X-Key": ""}}, "invalid connection header value for 'X-Key'"), + ( + {"headers": {"X-Key": "a\r\nInjected: 1"}}, + "invalid connection header value for 'X-Key'", + ), + ({"headers": {"X-Key": "a\nb"}}, "invalid connection header value for 'X-Key'"), + ], +) +def test_a_connection_refuses_names_urls_and_headers_that_could_inject( + overrides: dict[str, Any], message: str +) -> None: + with pytest.raises(ValueError, match=message): + connection(**overrides) + + +@pytest.mark.parametrize( + ("wire", "message"), + [ + (["inference"], "connections must be objects"), + ( + {"name": 1, "capability": "shell", "url": "https://x", "headers": {"A": "b"}}, + "connection name, capability, and url must be strings", + ), + ( + { + "name": "inference", + "capability": "shell", + "url": "https://x", + "headers": [["A", "b"]], + }, + "connection headers must map strings to strings", + ), + ( + {"name": "inference", "capability": "shell", "url": "https://x", "headers": {"A": 1}}, + "connection headers must map strings to strings", + ), + ], +) +def test_a_connection_from_the_wire_must_be_well_typed(wire: object, message: str) -> None: + with pytest.raises(ValueError, match=message): + Connection.from_wire(wire) From 29df8fe4aa1a0099bbabed59014a71a1056c9073 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 065/248] test: cover MCPClient against a FastMCP server MCPClient had no tests. Over streamable HTTP and SSE, a FastMCP server lists its tools, a structured result arrives with isError false and its structured content, and a tool that raises arrives as an error result with its message. serve_asgi() joins the harness to serve an ASGI app in the test's event loop; the fake DevTools endpoint uses it too. --- tests/capabilities/test_mcp.py | 55 ++++++++++++++++++++++++++++++++++ tests/harness/__init__.py | 2 ++ tests/harness/asgi.py | 35 ++++++++++++++++++++++ tests/harness/cdp.py | 18 ++--------- 4 files changed, 95 insertions(+), 15 deletions(-) create mode 100644 tests/capabilities/test_mcp.py create mode 100644 tests/harness/asgi.py diff --git a/tests/capabilities/test_mcp.py b/tests/capabilities/test_mcp.py new file mode 100644 index 000000000..dd3f26d46 --- /dev/null +++ b/tests/capabilities/test_mcp.py @@ -0,0 +1,55 @@ +"""``MCPClient`` against a FastMCP server over each HTTP transport.""" + +from __future__ import annotations + +from typing import Literal + +import pytest +from fastmcp import FastMCP +from fastmcp.exceptions import ToolError + +from hud.capabilities import Capability, MCPClient +from tests.harness import serve_asgi + + +def tools() -> FastMCP: + server = FastMCP("tools") + + @server.tool + def add(a: int, b: int) -> int: + """Add two numbers.""" + return a + b + + @server.tool + def refuse() -> str: + """Always fail.""" + raise ToolError("not today") + + return server + + +@pytest.mark.parametrize(("transport", "path"), [("streamable-http", "/mcp"), ("sse", "/sse")]) +async def test_tool_calls_return_canonical_results_over_each_transport( + transport: Literal["streamable-http", "sse"], path: str +) -> None: + app = tools().http_app(transport=transport, path=path) + async with serve_asgi(app) as port: + client = await MCPClient.connect( + Capability.mcp(url=f"http://127.0.0.1:{port}{path}", transport=transport) + ) + try: + listed = sorted(tool.name for tool in await client.list_tools()) + added = await client.call_tool("add", {"a": 2, "b": 3}) + refused = await client.call_tool("refuse", {}) + finally: + await client.close() + + assert listed == ["add", "refuse"] + assert (added.isError, added.structuredContent) == (False, {"result": 5}) + assert [block.model_dump(include={"type", "text"}) for block in added.content] == [ + {"type": "text", "text": "5"} + ] + assert refused.isError is True + assert [block.model_dump(include={"type", "text"}) for block in refused.content] == [ + {"type": "text", "text": "not today"} + ] diff --git a/tests/harness/__init__.py b/tests/harness/__init__.py index 8cec631c1..f93006963 100644 --- a/tests/harness/__init__.py +++ b/tests/harness/__init__.py @@ -6,6 +6,7 @@ live in ``tests/conftest.py``. """ +from .asgi import serve_asgi from .cdp import Command, FakeBrowser, fake_browser from .cli import Hud, Result, scrub from .config import HudEnv @@ -62,6 +63,7 @@ "relay", "say", "scrub", + "serve_asgi", "served", "shell_call", "spans", diff --git a/tests/harness/asgi.py b/tests/harness/asgi.py new file mode 100644 index 000000000..376441f65 --- /dev/null +++ b/tests/harness/asgi.py @@ -0,0 +1,35 @@ +"""Serve an ASGI app on a loopback port inside the test's event loop.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING + +import uvicorn + +from .scenario import eventually + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from starlette.types import ASGIApp + + +@asynccontextmanager +async def serve_asgi(app: ASGIApp) -> AsyncIterator[int]: + """Serve ``app`` on ``127.0.0.1`` and yield its port; stop it when the block ends.""" + server = uvicorn.Server( + uvicorn.Config( + app, host="127.0.0.1", port=0, log_level="error", lifespan="on", ws="websockets-sansio" + ) + ) + serving = asyncio.create_task(server.serve()) + await eventually(lambda: server.started or serving.done()) + if serving.done(): + serving.result() + try: + yield server.servers[0].sockets[0].getsockname()[1] + finally: + server.should_exit = True + await serving diff --git a/tests/harness/cdp.py b/tests/harness/cdp.py index 2915b162f..82a350183 100644 --- a/tests/harness/cdp.py +++ b/tests/harness/cdp.py @@ -12,18 +12,16 @@ from __future__ import annotations -import asyncio import json from contextlib import asynccontextmanager from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any -import uvicorn from starlette.applications import Starlette from starlette.responses import JSONResponse from starlette.routing import Route, WebSocketRoute -from .scenario import eventually +from .asgi import serve_asgi if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -112,16 +110,6 @@ async def fake_browser( WebSocketRoute("/devtools/page/{page}", browser._session), ] ) - server = uvicorn.Server( - uvicorn.Config( - app, host="127.0.0.1", port=0, log_level="error", lifespan="off", ws="websockets-sansio" - ) - ) - serving = asyncio.create_task(server.serve()) - await eventually(lambda: server.started) - browser.port = server.servers[0].sockets[0].getsockname()[1] - try: + async with serve_asgi(app) as port: + browser.port = port yield browser - finally: - server.should_exit = True - await serving From 6fa6a61d74d2043d452357335172225747e3617a Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 066/248] test: add a scripted control-channel peer to the harness control_peer() is the env side of the HUD wire: it answers each control connection from a script (a result, an error, a reply with another id, a hang-up, no answer), records every frame, and splices tunnel.open streams to the upstream named for each capability, so client tests pin literal frames and tunnel real capability traffic. --- tests/harness/__init__.py | 6 ++ tests/harness/control.py | 197 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 tests/harness/control.py diff --git a/tests/harness/__init__.py b/tests/harness/__init__.py index f93006963..34fb65223 100644 --- a/tests/harness/__init__.py +++ b/tests/harness/__init__.py @@ -10,6 +10,7 @@ from .cdp import Command, FakeBrowser, fake_browser from .cli import Hud, Result, scrub from .config import HudEnv +from .control import ControlPeer, answer, control_peer, hang_up, hold from .docker import FakeDocker from .models import ( ModelRequest, @@ -34,6 +35,7 @@ __all__ = [ "ROBOT_STEP_SCHEMA", "Command", + "ControlPeer", "FakeBrowser", "FakeDocker", "FakeScreen", @@ -52,13 +54,17 @@ "ScriptedAgent", "ToolCall", "Turn", + "answer", "call", "calls", "computer_call", + "control_peer", "eventually", "fail", "fake_browser", "fake_screen", + "hang_up", + "hold", "interrupted", "relay", "say", diff --git a/tests/harness/control.py b/tests/harness/control.py new file mode 100644 index 000000000..3dc58ac4d --- /dev/null +++ b/tests/harness/control.py @@ -0,0 +1,197 @@ +"""A scripted control-channel peer: the env side of the HUD wire protocol. + +:func:`control_peer` listens on loopback and answers each accepted control +connection from the next script in line (the last script repeats). A script is +a list of actions applied to the requests that connection receives, in order: + +- :func:`answer` replies with a result (or ``error=`` an error object, or + ``id=`` a reply id other than the request's); +- :func:`hang_up` closes the connection, before reading anything when + ``immediately`` is set; +- :func:`hold` reads the request and never replies. + +When a script runs out the connection stays open and every further request +is recorded and left unanswered. A connection whose first frame is +``tunnel.open`` is a capability stream instead, as on a served env: the peer +replies and splices the stream to ``upstreams[capability]``, or answers an +error for a capability it has no upstream for. Every frame received is kept in +:attr:`ControlPeer.frames`, in arrival order. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Sequence + +_DEFAULT_ID = object() + + +@dataclass(frozen=True) +class Action: + kind: str + result: Any = None + error: dict[str, Any] | None = None + reply_id: Any = _DEFAULT_ID + immediately: bool = False + + +def answer( + result: Any = None, *, error: dict[str, Any] | None = None, id: Any = _DEFAULT_ID +) -> Action: + return Action("answer", result=result, error=error, reply_id=id) + + +def hang_up(*, immediately: bool = False) -> Action: + return Action("hang_up", immediately=immediately) + + +def hold() -> Action: + return Action("hold") + + +@dataclass(frozen=True) +class Frame: + connection: int + message: dict[str, Any] + + +@dataclass +class ControlPeer: + """The peer's record. Use :func:`control_peer`.""" + + scripts: Sequence[Sequence[Action]] + upstreams: dict[str, tuple[str, int]] + port: int = 0 + #: Control connections accepted, tunnel streams excluded. + accepted: int = 0 + tunnels: int = 0 + frames: list[Frame] = field(default_factory=list) + _server: asyncio.Server | None = None + _handlers: set[asyncio.Task[None]] = field(default_factory=set) + + @property + def url(self) -> str: + return f"tcp://127.0.0.1:{self.port}" + + def requests(self, method: str | None = None) -> list[dict[str, Any]]: + """The request frames received, those calling ``method`` when given.""" + return [ + frame.message + for frame in self.frames + if method is None or frame.message.get("method") == method + ] + + def stop_accepting(self) -> None: + """Close the listening socket; open connections stay up.""" + assert self._server is not None + self._server.close() + + async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + task = asyncio.current_task() + assert task is not None + self._handlers.add(task) + try: + await self._converse(reader, writer) + except (OSError, asyncio.IncompleteReadError): + pass + finally: + self._handlers.discard(task) + writer.close() + + async def _converse(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + script = list(self.scripts[min(self.accepted, len(self.scripts) - 1)]) + if script and script[0].kind == "hang_up" and script[0].immediately: + self.accepted += 1 + return + first = await _read(reader) + if first is None: + self.accepted += 1 + return + if first.get("method") == "tunnel.open": + await self._tunnel(first, reader, writer) + return + connection = self.accepted + self.accepted += 1 + message: dict[str, Any] | None = first + while message is not None: + self.frames.append(Frame(connection, message)) + action = script.pop(0) if script else hold() + if action.kind == "hang_up": + return + if action.kind == "answer": + reply_id = message.get("id") if action.reply_id is _DEFAULT_ID else action.reply_id + body = ( + {"error": action.error} + if action.error is not None + else {"result": action.result} + ) + await _write(writer, {"jsonrpc": "2.0", "id": reply_id, **body}) + message = await _read(reader) + + async def _tunnel( + self, opened: dict[str, Any], reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + self.tunnels += 1 + self.frames.append(Frame(-1, opened)) + name = opened.get("params", {}).get("capability") + upstream = self.upstreams.get(name) + if upstream is None: + error = {"code": -32602, "message": f"unknown capability: {name!r}"} + await _write(writer, {"jsonrpc": "2.0", "id": opened.get("id"), "error": error}) + return + up_reader, up_writer = await asyncio.open_connection(*upstream) + await _write( + writer, {"jsonrpc": "2.0", "id": opened.get("id"), "result": {"capability": name}} + ) + try: + await asyncio.gather(_pipe(reader, up_writer), _pipe(up_reader, writer)) + finally: + up_writer.close() + + +async def _read(reader: asyncio.StreamReader) -> dict[str, Any] | None: + line = await reader.readline() + return json.loads(line) if line else None + + +async def _write(writer: asyncio.StreamWriter, message: dict[str, Any]) -> None: + writer.write(json.dumps(message).encode() + b"\n") + await writer.drain() + + +async def _pipe(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + except OSError: + pass + finally: + writer.close() + + +@asynccontextmanager +async def control_peer( + *scripts: Sequence[Action], upstreams: dict[str, tuple[str, int]] | None = None +) -> AsyncIterator[ControlPeer]: + """Serve a :class:`ControlPeer` answering connections from ``scripts`` for the block.""" + peer = ControlPeer(scripts=scripts or ([],), upstreams=upstreams or {}) + server = await asyncio.start_server(peer._handle, "127.0.0.1", 0) + peer._server = server + peer.port = server.sockets[0].getsockname()[1] + try: + yield peer + finally: + server.close() + for handler in list(peer._handlers): + handler.cancel() + with contextlib.suppress(asyncio.CancelledError): + await asyncio.gather(*peer._handlers, return_exceptions=True) + await server.wait_closed() From 266401406a4a433aca9f59aee489b26e4c0284e9 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 067/248] test: drive the control-channel client against a scripted peer The client tests patched the client registry, the capability retry delay, asyncio.open_connection, the heartbeat constants and _connect_ready, and built expected hello params with the code's own to_wire(). The scenario now pins the hello frame literally (bare, with workspace routes, with controller connections), retries through accept-then-close and EOF until hello answers, fails at once on error frames, mismatched ids and non-object results, honors the runtime's ready_timeout and refuses invalid ones and non-tcp transports, resolves bindings by name and protocol to local tunnels with the ambiguous and unknown errors, opens an SSH client through a tunnel to a served workspace and closes it with the connection, ends local streams the env cannot tunnel, and pins each tasks.* frame and result. The 120 s heartbeat stays uncovered: its interval has no public knob. --- tests/clients/test_connect.py | 702 ++++++++++++++++------------------ 1 file changed, 329 insertions(+), 373 deletions(-) diff --git a/tests/clients/test_connect.py b/tests/clients/test_connect.py index 7aa42455c..ab51b5b8a 100644 --- a/tests/clients/test_connect.py +++ b/tests/clients/test_connect.py @@ -1,427 +1,383 @@ -"""``connect()`` readiness: the handshake retries until the env actually serves. +"""``hud.clients.connect`` and ``HudClient`` against a scripted control-channel peer. -A provisioned substrate can sit behind a proxied port (``docker -p``, a -port-forward) that *accepts* TCP before the env behind it is up — those -connections die with EOF at the handshake. Readiness is therefore -protocol-level: ``connect`` keeps retrying through both refused connects and -handshake EOFs until ``hello`` answers or the deadline passes. +The peer (:func:`tests.harness.control_peer`) is the env side of the wire: +it answers each connection from a script and records every frame, so the +frames the client sends are pinned literally. Capability streams it tunnels +reach real servers: a served workspace's SSH daemon through a flaky relay. + +Not covered: the 120 s control heartbeat. Its interval is a module constant +with no public knob, and waiting it out has no place in the default lane. """ from __future__ import annotations import asyncio import logging -from typing import ClassVar +import math +from typing import TYPE_CHECKING, Any from urllib.parse import urlsplit import pytest +from dirty_equals import IsStr -import hud.clients.client as client_module -from hud.capabilities import Capability, CapabilityClient, Connection -from hud.clients import HudProtocolError, connect +from hud import Environment +from hud.capabilities import Connection, SSHClient +from hud.clients import HudClient, HudProtocolError, connect from hud.environment import WorkspaceRoute -from hud.environment.utils import read_frame, send_frame from hud.eval.runtime import Runtime +from tests.harness import answer, control_peer, eventually, hang_up, relay, served -HELLO_RESULT = {"session_id": "s-1", "env": {"name": "stub", "version": "1.0"}, "bindings": []} - - -def test_workspace_route_from_url_extracts_transport_address() -> None: - assert WorkspaceRoute.from_url("ssh", "https://inference.hud.so/v1") == WorkspaceRoute( - "ssh", - "inference.hud.so", - 443, - ) - assert WorkspaceRoute.from_url("shell", "http://gateway.test:8080") == WorkspaceRoute( - "shell", - "gateway.test", - 8080, - ) - - -async def test_connect_sends_workspace_routes_in_hello() -> None: - requests: list[dict[str, object]] = [] +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + from pathlib import Path - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - requests.append(hello) - await send_frame(writer, {"jsonrpc": "2.0", "id": hello["id"], "result": HELLO_RESULT}) - await read_frame(reader) - finally: - writer.close() - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - runtime = Runtime(f"tcp://127.0.0.1:{port}") - route = WorkspaceRoute("ssh", "inference.hud.so", 443) - try: - async with connect(runtime, workspace_routes=(route,)): - pass - finally: - server.close() - await server.wait_closed() +def hello(*bindings: dict[str, Any]) -> dict[str, Any]: + return { + "session_id": "s-1", + "env": {"name": "stub", "version": "1.0"}, + "bindings": list(bindings), + } - assert [request["method"] for request in requests] == ["hello"] - params = requests[0]["params"] - assert isinstance(params, dict) - assert params == {"workspace_routes": [route.to_wire()]} +BINDINGS = ( + {"name": "shell", "protocol": "ssh/2", "url": "ssh://agent@environment:22"}, + {"name": "screen-0", "protocol": "rfb/3.8", "url": "rfb://environment:5900"}, + {"name": "screen-1", "protocol": "rfb/3.8", "url": "rfb://environment:5901"}, + {"name": "custom", "protocol": "custom/1", "url": "tcp://environment:7000"}, +) + + +@pytest.mark.parametrize( + ("routes", "connections", "params"), + [ + ((), (), {}), + ( + ( + WorkspaceRoute.from_url("ssh", "https://inference.hud.so/v1"), + WorkspaceRoute.from_url("shell", "http://gateway.test:8080"), + ), + (), + { + "workspace_routes": [ + {"capability": "ssh", "host": "inference.hud.so", "port": 443}, + {"capability": "shell", "host": "gateway.test", "port": 8080}, + ] + }, + ), + ( + (), + ( + Connection( + name="inference", + capability="ssh", + url="https://inference.hud.so", + headers={"Authorization": "Bearer secret"}, + ), + ), + { + "connections": [ + { + "name": "inference", + "capability": "ssh", + "url": "https://inference.hud.so", + "headers": {"Authorization": "Bearer secret"}, + } + ] + }, + ), + ], + ids=["bare", "workspace routes", "controller connections"], +) +async def test_hello_carries_the_workspace_routes_and_connections( + routes: tuple[WorkspaceRoute, ...], connections: tuple[Connection, ...], params: dict[str, Any] +) -> None: + async with ( + control_peer([answer(hello())]) as peer, + connect(Runtime(peer.url), workspace_routes=routes, connections=connections) as client, + ): + manifest = client.manifest + + assert peer.requests() == [{"jsonrpc": "2.0", "id": 1, "method": "hello", "params": params}] + assert manifest is not None + assert (manifest.session_id, manifest.server_info.name, manifest.server_info.version) == ( + "s-1", + "stub", + "1.0", + ) -async def test_connect_sends_controller_connections_in_hello() -> None: - requests: list[dict[str, object]] = [] - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - requests.append(hello) - await send_frame(writer, {"jsonrpc": "2.0", "id": hello["id"], "result": HELLO_RESULT}) - await read_frame(reader) - finally: - writer.close() +@pytest.mark.parametrize( + "not_ready", + [hang_up(immediately=True), hang_up()], + ids=["accept then close", "close during hello"], +) +async def test_connect_retries_until_the_env_answers_hello(not_ready: Any) -> None: + async with ( + control_peer([not_ready], [not_ready], [answer(hello())]) as peer, + connect(Runtime(peer.url), ready_timeout=10) as client, + ): + assert client.manifest is not None + + assert peer.accepted == 3 + + +@pytest.mark.parametrize( + ("script", "message"), + [ + ( + [answer(error={"code": -32000, "message": "frame exceeds size limit"}, id=None)], + "hud rpc error -32000: frame exceeds size limit", + ), + ( + [answer(error={"code": -32601, "message": "unknown method"})], + "hud rpc error -32601: unknown method", + ), + ([answer(hello(), id=99)], "'hello': reply id did not match request"), + ([answer(["not", "an", "object"])], "'hello': result was not an object"), + ], + ids=["an error frame", "an error reply", "a mismatched id", "a non-object"], +) +async def test_connect_fails_without_retrying_on_a_handshake_the_env_breaks( + script: list[Any], message: str +) -> None: + async with control_peer(script) as peer: + with pytest.raises(HudProtocolError, match=message): + async with connect(Runtime(peer.url), ready_timeout=10): + pass - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer secret"}, - ) - try: - async with connect( - Runtime(f"tcp://127.0.0.1:{port}"), - connections=(connection,), - ): - pass - finally: - server.close() - await server.wait_closed() + assert peer.accepted == 1 - params = requests[0]["params"] - assert isinstance(params, dict) - assert params == {"connections": [connection.to_wire()]} +async def test_connect_gives_up_at_the_runtime_ready_timeout_on_an_env_that_never_answers() -> None: + loop = asyncio.get_running_loop() + async with control_peer([hang_up()]) as peer: + started = loop.time() + with pytest.raises(EOFError, match="env closed connection during 'hello'"): + async with connect(Runtime(peer.url, params={"ready_timeout": 0.3}), ready_timeout=60): + pass -async def test_open_retries_transient_capability_connection_failures( - monkeypatch: pytest.MonkeyPatch, -) -> None: - class RetryingClient(CapabilityClient): - protocol: ClassVar[str] = "test/1" - attempts = 0 - - @classmethod - async def connect(cls, cap: Capability) -> RetryingClient: - del cap - cls.attempts += 1 - if cls.attempts < 3: - raise ConnectionError("connection reset") - return cls() - - async def close(self) -> None: - pass + assert loop.time() - started < 5 - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - await send_frame( - writer, - { - "jsonrpc": "2.0", - "id": hello["id"], - "result": { - **HELLO_RESULT, - "bindings": [ - { - "name": "test", - "protocol": RetryingClient.protocol, - "url": "tcp://environment:1234", - } - ], - }, - }, - ) - await read_frame(reader) - finally: - writer.close() - monkeypatch.setitem(client_module._CLIENT_REGISTRY, RetryingClient.protocol, RetryingClient) - monkeypatch.setattr(client_module, "_CAPABILITY_CONNECT_BASE_DELAY_SECONDS", 0) - server = await asyncio.start_server(handler, "127.0.0.1", 0) +async def test_connect_gives_up_on_a_refused_port_after_the_ready_timeout() -> None: + server = await asyncio.start_server(lambda reader, writer: None, "127.0.0.1", 0) port = server.sockets[0].getsockname()[1] - try: - async with connect(Runtime(f"tcp://127.0.0.1:{port}")) as client: - opened = await client.open("test") - assert await client.open("test") is opened - finally: - server.close() - await server.wait_closed() - - assert RetryingClient.attempts == 3 + server.close() + await server.wait_closed() + with pytest.raises(OSError): + async with connect(Runtime(f"tcp://127.0.0.1:{port}"), ready_timeout=0.3): + pass -async def test_connect_retries_through_accept_then_eof_until_the_env_serves() -> None: - attempts = 0 - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - nonlocal attempts - attempts += 1 - if attempts <= 2: - # The docker-proxy shape: accept, then hang up without serving. - writer.close() - return - try: - msg = await read_frame(reader) - assert msg is not None - await send_frame(writer, {"jsonrpc": "2.0", "id": msg["id"], "result": HELLO_RESULT}) - await read_frame(reader) # hold the connection until the client closes - finally: - # 3.12's Server.wait_closed() waits on every connection; a handler - # that returns without closing its writer deadlocks teardown. - writer.close() +@pytest.mark.parametrize("value", [True, 0, -1.0, math.nan, math.inf, "5"]) +async def test_an_invalid_runtime_ready_timeout_is_refused(value: object) -> None: + with pytest.raises(ValueError, match="must be a positive finite number"): + async with connect(Runtime("tcp://127.0.0.1:9", params={"ready_timeout": value})): + pass - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - async with connect(Runtime(f"tcp://127.0.0.1:{port}"), ready_timeout=10) as client: - assert client.manifest is not None - assert client.manifest.server_info.name == "stub" - finally: - server.close() - await server.wait_closed() - assert attempts == 3 +async def test_only_tcp_control_transports_are_supported() -> None: + with pytest.raises(NotImplementedError, match="control transport 'unix' not supported"): + async with connect(Runtime("unix:///run/hud.sock")): + pass -async def test_tunnel_connection_failure_warns_with_peer( - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, +@pytest.mark.parametrize( + ("ref", "outcome"), + [ + ("shell", IsStr(regex=r"ssh://agent@127\.0\.0\.1:\d+")), + ("ssh", IsStr(regex=r"ssh://agent@127\.0\.0\.1:\d+")), + ("ssh/2", IsStr(regex=r"ssh://agent@127\.0\.0\.1:\d+")), + ("screen-1", IsStr(regex=r"rfb://127\.0\.0\.1:\d+")), + ( + "rfb", + KeyError("ambiguous capability 'rfb'; matches: screen-0 (rfb/3.8), screen-1 (rfb/3.8)"), + ), + ( + "browser", + KeyError( + "no capability 'browser' (available: shell (ssh/2), screen-0 (rfb/3.8), " + "screen-1 (rfb/3.8), custom (custom/1))" + ), + ), + ], +) +async def test_a_binding_resolves_by_name_or_protocol_to_a_local_tunnel( + ref: str, outcome: Any ) -> None: - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + async with ( + control_peer([answer(hello(*BINDINGS))]) as peer, + connect(Runtime(peer.url)) as client, + ): + if isinstance(outcome, KeyError): + with pytest.raises(KeyError) as raised: + client.binding(ref) + assert raised.value.args == outcome.args + else: + assert client.binding(ref).url == outcome + + +async def test_a_raw_stream_client_needs_hello_and_passes_bindings_through() -> None: + async with control_peer([answer(hello(*BINDINGS))]) as peer: + reader, writer = await asyncio.open_connection("127.0.0.1", peer.port) + client = HudClient(reader, writer) try: - hello = await read_frame(reader) - assert hello is not None - await send_frame( - writer, - { - "jsonrpc": "2.0", - "id": hello["id"], - "result": { - **HELLO_RESULT, - "bindings": [ - { - "name": "shell", - "protocol": "ssh/2", - "url": "ssh://agent@environment:22", - } - ], - }, - }, - ) - await read_frame(reader) + with pytest.raises(RuntimeError, match=r"call hello\(\) before accessing bindings"): + client.binding("shell") + await client.hello() + url = client.binding("shell").url finally: - writer.close() - - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - open_connection = asyncio.open_connection - - async def fail_tunnel_connection(host: str, peer_port: int): - assert (host, peer_port) == ("127.0.0.1", port) - raise OSError("peer unavailable") + await client.close() - try: - async with connect(Runtime(f"tcp://127.0.0.1:{port}")) as client: - route = urlsplit(client.binding("shell").url) - monkeypatch.setattr(client_module.asyncio, "open_connection", fail_tunnel_connection) - caplog.set_level(logging.WARNING, logger="hud.clients") + assert url == "ssh://agent@environment:22" - reader, writer = await open_connection(route.hostname, route.port) - try: - assert await reader.read() == b"" - finally: - writer.close() - await writer.wait_closed() - finally: - server.close() - await server.wait_closed() - assert f"tunnel peer 127.0.0.1:{port} connection failed: peer unavailable" in caplog.messages +async def test_opening_a_protocol_without_a_client_points_at_binding() -> None: + async with ( + control_peer([answer(hello(*BINDINGS))]) as peer, + connect(Runtime(peer.url)) as client, + ): + with pytest.raises(ValueError, match=r"use binding\('custom'\) for raw access"): + await client.open("custom") -async def test_heartbeat_serializes_calls_without_aborting_on_protocol_errors( - monkeypatch: pytest.MonkeyPatch, +async def test_an_opened_ssh_client_tunnels_to_its_upstream_and_closes_with_the_connection( + tmp_path: Path, ) -> None: - requests: list[dict[str, object]] = [] - heartbeat_received = asyncio.Event() - release_heartbeat = asyncio.Event() - - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + root = tmp_path / "workspace" + root.mkdir() + env = Environment("tunneled") + env.workspace(root, track_files=False) + async with served(env) as workspace: + assert workspace.manifest is not None + (shell,) = (cap for cap in workspace.manifest.bindings if cap.name == "shell") + address = urlsplit(shell.url) + assert address.hostname is not None and address.port is not None + async with relay(address.hostname, address.port) as flaky: + binding = {**shell.to_manifest(), "url": "ssh://agent@environment:22"} + async with control_peer( + [answer(hello(binding))], upstreams={"shell": ("127.0.0.1", flaky.port)} + ) as peer: + async with connect(Runtime(peer.url)) as client: + ssh = await client.open("shell") + assert isinstance(ssh, SSHClient) + result = await ssh.run("echo through the tunnel", timeout=10) + again = await client.open("ssh") + + assert again is ssh + assert ssh.conn.is_closed() + await eventually(lambda: flaky.open == 0) + + assert result.stdout == "through the tunnel\n" + assert (flaky.accepted, peer.tunnels) == (1, 1) + + +@pytest.mark.parametrize( + ("break_tunnel", "warning"), + [ + ( + lambda peer: peer.stop_accepting(), + IsStr(regex=r"tunnel peer 127\.0\.0\.1:\d+ connection failed: .*"), + ), + (lambda peer: None, IsStr(regex=r"tunnel\.open 'shell' refused: .*unknown capability.*")), + ], + ids=["the env stops accepting", "the env refuses the stream"], +) +async def test_a_tunnel_the_env_cannot_open_ends_the_local_stream( + break_tunnel: Callable[[Any], None], warning: Any, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level(logging.WARNING, logger="hud.clients") + async with ( + control_peer([answer(hello(BINDINGS[0]))]) as peer, + connect(Runtime(peer.url)) as client, + ): + local = urlsplit(client.binding("shell").url) + break_tunnel(peer) + reader, writer = await asyncio.open_connection(local.hostname, local.port) try: - hello = await read_frame(reader) - assert hello is not None - requests.append(hello) - await send_frame( - writer, - {"jsonrpc": "2.0", "id": hello["id"], "result": HELLO_RESULT}, - ) - - heartbeat = await read_frame(reader) - assert heartbeat is not None - requests.append(heartbeat) - heartbeat_received.set() - await release_heartbeat.wait() - await send_frame( - writer, - { - "jsonrpc": "2.0", - "id": heartbeat["id"], - "error": {"code": -32601, "message": "heartbeat refused"}, - }, - ) - - grade = await read_frame(reader) - assert grade is not None - requests.append(grade) - await send_frame( - writer, - {"jsonrpc": "2.0", "id": grade["id"], "result": {"score": 1.0}}, - ) - await read_frame(reader) + received = await asyncio.wait_for(reader.read(), 10) finally: writer.close() - monkeypatch.setattr(client_module, "_CONTROL_HEARTBEAT_INTERVAL_SECONDS", 0.01) - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - async with connect(Runtime(f"tcp://127.0.0.1:{port}")) as client: - await asyncio.wait_for(heartbeat_received.wait(), 1.0) - grade = asyncio.create_task(client.grade({"answer": "done"})) - await asyncio.sleep(0) - assert [request["method"] for request in requests] == ["hello", "hello"] - release_heartbeat.set() - assert await asyncio.wait_for(grade, 1.0) == {"score": 1.0} - finally: - server.close() - await server.wait_closed() - - assert [request["method"] for request in requests] == ["hello", "hello", "tasks.grade"] - assert requests[1]["params"] == {"session_id": "s-1"} - - -async def test_heartbeat_transport_failure_interrupts_the_connection_body( - monkeypatch: pytest.MonkeyPatch, + assert received == b"" + assert warning in caplog.messages + + +@pytest.mark.parametrize( + ("call", "result", "sent", "returned"), + [ + ( + lambda client: client.start_task("fix", {"bug": 1}), + {"prompt": "Fix it."}, + ("tasks.start", {"id": "fix", "args": {"bug": 1}}), + {"prompt": "Fix it."}, + ), + ( + lambda client: client.start_task("fix"), + {"prompt": "Fix it."}, + ("tasks.start", {"id": "fix", "args": {}}), + {"prompt": "Fix it."}, + ), + ( + lambda client: client.grade({"answer": "done"}), + {"score": 1.0}, + ("tasks.grade", {"answer": "done"}), + {"score": 1.0}, + ), + (lambda client: client.cancel(), {}, ("tasks.cancel", {}), None), + ( + lambda client: client.list_tasks(), + {"tasks": [{"id": "fix", "description": "Fix it."}]}, + ("tasks.list", {}), + [{"id": "fix", "description": "Fix it."}], + ), + ( + lambda client: client.list_tasks(), + {"tasks": "fix"}, + ("tasks.list", {}), + HudProtocolError(-32603, "tasks.list: 'tasks' must be a list"), + ), + ( + lambda client: client.grade({"answer": "done"}), + {"error": {"code": -32600, "message": "no task in progress"}}, + ("tasks.grade", {"answer": "done"}), + HudProtocolError(-32600, "no task in progress"), + ), + ], + ids=["start", "start without args", "grade", "cancel", "list", "list malformed", "grade error"], +) +async def test_task_calls_send_one_request_and_return_its_result( + call: Callable[[HudClient], Awaitable[Any]], + result: dict[str, Any], + sent: tuple[str, dict[str, Any]], + returned: Any, ) -> None: - heartbeat_received = asyncio.Event() - body_interrupted = asyncio.Event() - - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - hello = await read_frame(reader) - assert hello is not None - await send_frame( - writer, - {"jsonrpc": "2.0", "id": hello["id"], "result": HELLO_RESULT}, + reply = answer(error=result["error"]) if "error" in result else answer(result) + async with control_peer([answer(hello()), reply]) as peer, connect(Runtime(peer.url)) as client: + if isinstance(returned, HudProtocolError): + with pytest.raises(HudProtocolError) as raised: + await call(client) + assert (raised.value.code, raised.value.message) == ( + returned.code, + returned.message, ) - heartbeat = await read_frame(reader) - assert heartbeat is not None - heartbeat_received.set() - await reader.read() - finally: - writer.close() + else: + assert await call(client) == returned - monkeypatch.setattr(client_module, "_CONTROL_HEARTBEAT_INTERVAL_SECONDS", 0.01) - monkeypatch.setattr(client_module, "_CONTROL_HEARTBEAT_TIMEOUT_SECONDS", 0.01) - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - with pytest.raises(TimeoutError): - async with connect(Runtime(f"tcp://127.0.0.1:{port}")): - await asyncio.wait_for(heartbeat_received.wait(), 1.0) - try: - await asyncio.Event().wait() - except asyncio.CancelledError: - body_interrupted.set() - raise - finally: - server.close() - await server.wait_closed() - - assert body_interrupted.is_set() - - -async def test_connect_uses_runtime_ready_timeout_param( - monkeypatch: pytest.MonkeyPatch, -) -> None: - seen: dict[str, float | int | str] = {} + method, params = sent + assert peer.requests()[1] == {"jsonrpc": "2.0", "id": 2, "method": method, "params": params} - class _FakeClient: - async def close(self) -> None: - pass - async def fake_connect_ready( - host: str, - port: int, - *, - ready_timeout: float, - workspace_routes: tuple[WorkspaceRoute, ...], - connections: tuple[Connection, ...], - interval: float = 0.5, - ) -> _FakeClient: - seen["host"] = host - seen["port"] = port - seen["ready_timeout"] = ready_timeout - assert workspace_routes == () - assert connections == () - seen["interval"] = interval - return _FakeClient() - - monkeypatch.setattr(client_module, "_connect_ready", fake_connect_ready) - - async with client_module.connect( - Runtime("tcp://127.0.0.1:1234", params={"ready_timeout": 300.0}) +async def test_a_reply_to_another_request_aborts_the_connection() -> None: + async with ( + control_peer([answer(hello()), answer({"score": 1.0}, id=7)]) as peer, + connect(Runtime(peer.url)) as client, ): - pass - - assert seen == { - "host": "127.0.0.1", - "port": 1234, - "ready_timeout": 300.0, - "interval": 0.5, - } - + with pytest.raises(HudProtocolError, match=r"'tasks\.grade': reply id did not match"): + await client.grade({"answer": "done"}) + with pytest.raises((EOFError, OSError)): + await client.cancel() -@pytest.mark.parametrize("protocol_error", [False, True]) -async def test_connect_reports_handshake_failure(protocol_error: bool) -> None: - async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - try: - await read_frame(reader) - if protocol_error: - await send_frame( - writer, - { - "jsonrpc": "2.0", - "id": None, - "error": {"code": -32000, "message": "frame exceeds size limit"}, - }, - ) - finally: - writer.close() - - server = await asyncio.start_server(handler, "127.0.0.1", 0) - port = server.sockets[0].getsockname()[1] - try: - expected = HudProtocolError if protocol_error else EOFError - message = ( - "frame exceeds size limit" if protocol_error else "closed connection during 'hello'" - ) - with pytest.raises(expected, match=message): - async with connect(Runtime(f"tcp://127.0.0.1:{port}"), ready_timeout=1.2): - pass - finally: - server.close() - await server.wait_closed() + assert [request["method"] for request in peer.requests()] == ["hello", "tasks.grade"] From 13935ee73e521dfbe282df7da93036cc2d458c04 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 068/248] fix: close the far side of a spliced stream when one side resets splice() pumps each direction until EOF and swallowed a reset as the end of that pump. A reset carries no EOF, so the other side was never told: a client tunneled through the control channel to a capability whose upstream reset waited on a half-dead stream (an SSH handshake sat until asyncssh's 120 s login timeout) instead of failing and retrying. A pump that ends in a reset now closes the side it writes to, which ends the other pump too. The control-channel scenario opens an SSH client through a tunnel whose upstream resets twice; it retries and connects instead of hanging. --- hud/environment/utils.py | 10 ++++++++-- tests/clients/test_connect.py | 5 +++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hud/environment/utils.py b/hud/environment/utils.py index 235f27f57..038c92261 100644 --- a/hud/environment/utils.py +++ b/hud/environment/utils.py @@ -94,11 +94,17 @@ def error(msg_id: int | str | None, code: int, message: str) -> dict[str, Any]: async def _pump(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: # Resets/aborts are a normal way for tunneled streams to end (an SSH # client hanging up, a container dying); they end the pump, not the world. - with contextlib.suppress(OSError): + # A reset carries no EOF, so close the far side instead: otherwise it + # waits for bytes that will never come. + try: while data := await reader.read(65536): writer.write(data) await writer.drain() - if writer.can_write_eof(): + except OSError: + writer.close() + return + if writer.can_write_eof(): + with contextlib.suppress(OSError): writer.write_eof() diff --git a/tests/clients/test_connect.py b/tests/clients/test_connect.py index ab51b5b8a..3a6e8365f 100644 --- a/tests/clients/test_connect.py +++ b/tests/clients/test_connect.py @@ -243,7 +243,7 @@ async def test_opening_a_protocol_without_a_client_points_at_binding() -> None: await client.open("custom") -async def test_an_opened_ssh_client_tunnels_to_its_upstream_and_closes_with_the_connection( +async def test_an_opened_ssh_client_retries_an_upstream_that_resets_and_closes_with_the_connection( tmp_path: Path, ) -> None: root = tmp_path / "workspace" @@ -256,6 +256,7 @@ async def test_an_opened_ssh_client_tunnels_to_its_upstream_and_closes_with_the_ address = urlsplit(shell.url) assert address.hostname is not None and address.port is not None async with relay(address.hostname, address.port) as flaky: + flaky.refuse(2) binding = {**shell.to_manifest(), "url": "ssh://agent@environment:22"} async with control_peer( [answer(hello(binding))], upstreams={"shell": ("127.0.0.1", flaky.port)} @@ -271,7 +272,7 @@ async def test_an_opened_ssh_client_tunnels_to_its_upstream_and_closes_with_the_ await eventually(lambda: flaky.open == 0) assert result.stdout == "through the tunnel\n" - assert (flaky.accepted, peer.tunnels) == (1, 1) + assert (flaky.accepted, peer.tunnels) == (3, 3) @pytest.mark.parametrize( From adac87387ba287a358a69299876007febdd73eb4 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 069/248] test: narrow the decoded pixel before comparing channels --- tests/capabilities/test_rfb.py | 1 + tests/integrations/harbor/test_integration.py | 936 ------------------ 2 files changed, 1 insertion(+), 936 deletions(-) delete mode 100644 tests/integrations/harbor/test_integration.py diff --git a/tests/capabilities/test_rfb.py b/tests/capabilities/test_rfb.py index e49035613..a1241d9b7 100644 --- a/tests/capabilities/test_rfb.py +++ b/tests/capabilities/test_rfb.py @@ -37,6 +37,7 @@ async def test_a_screenshot_encodes_the_framebuffer_as_requested( colors = image.convert("RGB").getcolors() assert colors is not None ((count, pixel),) = colors + assert isinstance(pixel, tuple) assert count == 48 assert max(abs(channel - expected) for channel, expected in zip(pixel, COLOR, strict=True)) <= 4 diff --git a/tests/integrations/harbor/test_integration.py b/tests/integrations/harbor/test_integration.py deleted file mode 100644 index 22452976b..000000000 --- a/tests/integrations/harbor/test_integration.py +++ /dev/null @@ -1,936 +0,0 @@ -"""Docker coverage for Harbor phase behavior. - -The tests build adapted images and require network access:: - - uv run pytest -m integration -""" - -from __future__ import annotations - -import asyncio -import shutil -import subprocess -import sys -from pathlib import Path -from typing import TYPE_CHECKING, cast - -import pytest - -from hud.agents.base import Agent -from hud.eval import DockerRuntime, Shared, Taskset -from hud.eval.runtime import DockerBindMount -from hud.integrations import harbor - -from .conftest import make_harbor_task - -if TYPE_CHECKING: - from hud.capabilities import MCPClient, SSHClient - from hud.eval.run import Run - -TASKS = Path(__file__).parent / "tasks" -REPO = Path(__file__).resolve().parents[3] - -pytestmark = [ - pytest.mark.e2e, - pytest.mark.docker, - pytest.mark.skipif(sys.platform == "win32", reason="adapted images are Linux containers"), -] - - -def _adapt(path: Path, *, hud_requirement: str = "hud") -> Taskset: - result = harbor.adapt(path, hud_requirement=hud_requirement) - assert result.failures == () - return result.taskset - - -@pytest.fixture(scope="module", autouse=True) -def docker_daemon() -> None: - if shutil.which("docker") is None: - pytest.skip("needs a running Docker daemon") - try: - probe = subprocess.run(["docker", "info"], capture_output=True, check=False, timeout=20) - except subprocess.TimeoutExpired: - pytest.skip("Docker daemon did not answer") - if probe.returncode != 0: - pytest.skip("needs a running Docker daemon") - - -class Oracle(Agent): - """Run each fixture's reference solution.""" - - def __init__(self, solutions: dict[str, str]) -> None: - super().__init__() - self.solutions = solutions - - async def __call__(self, run: Run) -> None: - setup = next( - step.task_call - for step in run.trace.steps - if step.task_call is not None and step.task_call.phase == "setup" - ) - assert isinstance(setup.arguments, dict) - task = setup.arguments.get("task") - assert isinstance(task, dict) - task_id = task.get("id") - assert isinstance(task_id, str) - ssh = cast("SSHClient", await run.client.open("ssh/2")) - if task_id == "agent-lifecycle": - for index in range(40): - result = await ssh.conn.run(f"printf '%s' {index}", check=False) - assert result.exit_status == 0, f"command session {index} failed: {result.stderr!r}" - assert result.stdout == str(index), f"command session {index} lost output" - await ssh.conn.run("cat > session-input", input="written", check=True) - written = await ssh.conn.run("cat session-input", check=True) - assert written.stdout == "written" - if task_id == "hello-mcp": - mcp = cast("MCPClient", await run.client.open("mcp-server")) - assert {tool.name for tool in await mcp.list_tools()} == {"get_secret"} - assert run.client.manifest is not None - capability = next( - cap for cap in run.client.manifest.bindings if cap.name == "mcp-server" - ) - script = f""" -import json -from pathlib import Path -from urllib.request import Request, urlopen - -url = {capability.url!r} -session_id = None - -def post(payload): - global session_id - headers = {{ - "Accept": "application/json, text/event-stream", - "Content-Type": "application/json", - }} - if session_id is not None: - headers["Mcp-Session-Id"] = session_id - request = Request(url, data=json.dumps(payload).encode(), headers=headers) - with urlopen(request) as response: - body = response.read().decode() - returned_session = response.headers.get("Mcp-Session-Id") - if returned_session: - session_id = returned_session - data = [line[6:] for line in body.splitlines() if line.startswith("data: ")] - return json.loads(data[-1] if data else body) if body else None - -post({{ - "jsonrpc": "2.0", - "id": 1, - "method": "initialize", - "params": {{ - "protocolVersion": "2025-06-18", - "capabilities": {{}}, - "clientInfo": {{"name": "hud-test", "version": "1"}}, - }}, -}}) -post({{"jsonrpc": "2.0", "method": "notifications/initialized"}}) -result = post({{ - "jsonrpc": "2.0", - "id": 2, - "method": "tools/call", - "params": {{"name": "get_secret", "arguments": {{}}}}, -}}) -Path("/app/secret.txt").write_text(result["result"]["content"][0]["text"]) -""" - result = await ssh.conn.run("python3 -", input=script, check=True) - if result.exit_status is None: - raise RuntimeError("workspace MCP client closed without an exit status") - run.trace.content = "called get_secret from inside the workspace" - return - result = await ssh.conn.run(self.solutions[task_id], check=False) - if result.exit_status is None: - run.trace.content = ( - "solution SSH channel closed without an exit status:\n" - f"{result.stdout}\n{result.stderr}" - ) - raise RuntimeError("solution SSH channel closed without an exit status") - if result.exit_status != 0: - run.trace.content = f"solution failed:\n{result.stdout}\n{result.stderr}" - raise RuntimeError(f"solution exited with status {result.exit_status}") - run.trace.content = "solution completed" - - -async def _grade_every_task(dataset: Path, wheel: Path) -> dict[str, Run]: - def load_solutions() -> dict[str, str]: - return { - task.name: (task / "solution" / "solve.sh").read_text("utf-8") - for task in sorted(dataset.iterdir()) - if (task / "task.toml").is_file() - } - - solutions = await asyncio.to_thread(load_solutions) - taskset = _adapt(dataset, hud_requirement=str(wheel)) - job = await taskset.run( - Oracle(solutions), - runtime=DockerRuntime(), - max_concurrent=1, - ) - return {cast("str", run.slug): run for run in job.runs} - - -@pytest.fixture(scope="module") -def wheel(tmp_path_factory: pytest.TempPathFactory) -> Path: - workdir = tmp_path_factory.mktemp("harbor") - wheels = workdir / "wheels" - subprocess.run( - ["uv", "build", "--wheel", "--out-dir", str(wheels)], - cwd=REPO, - check=True, - capture_output=True, - ) - return next(iter(wheels.glob("*.whl"))) - - -@pytest.fixture(scope="module") -def graded(tmp_path_factory: pytest.TempPathFactory, wheel: Path) -> dict[str, Run]: - """Adapt and grade the inline-verifier fixtures once for this module.""" - dataset = tmp_path_factory.mktemp("harbor-inline") / "harbor-harness" - shutil.copytree(TASKS, dataset, ignore=shutil.ignore_patterns("sidecar-reachability")) - return asyncio.run(_grade_every_task(dataset, wheel)) - - -@pytest.fixture(scope="module") -def separately_graded(tmp_path_factory: pytest.TempPathFactory, wheel: Path) -> dict[str, Run]: - """Adapt and grade the separate-verifier fixture.""" - dataset = tmp_path_factory.mktemp("harbor-separate") / "harbor-harness" - dataset.mkdir() - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - declaration = task / "task.toml" - declaration.write_text( - declaration.read_text("utf-8") + "\n[verifier.environment]\ncpus = 1\n", - encoding="utf-8", - ) - return asyncio.run(_grade_every_task(dataset, wheel)) - - -@pytest.mark.parametrize( - "task_id", - [ - "phase-boundary", - "agent-lifecycle", - "verifier-lifecycle", - "hello-mcp", - ], -) -def test_harbor_phase_behavior(graded: dict[str, Run], task_id: str) -> None: - run = graded[task_id] - evaluation = run.evaluation - detail = evaluation.get("content") or "" - info = evaluation.get("info") or {} - detail = "\n".join( - filter( - None, - ( - run.trace.content, - run.trace.error, - detail, - info.get("stdout"), - info.get("stderr"), - ), - ) - ) - assert run.reward == 1.0, f"{task_id} scored {run.reward}; the verifier reported:\n{detail}" - - -def test_separate_verifier_phase_behavior(separately_graded: dict[str, Run]) -> None: - test_harbor_phase_behavior(separately_graded, "sidecar-reachability") - - -def test_sidecar_ports_are_discovered_from_the_built_image( - tmp_path_factory: pytest.TempPathFactory, - wheel: Path, -) -> None: - dataset = tmp_path_factory.mktemp("harbor-sidecar-ports") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - compose = task / "environment/docker-compose.yaml" - authored = compose.read_text("utf-8") - inferred = authored.replace( - " image: ${SIDECAR_IMAGE:-python:3.11-alpine}\n", - " build: ./workspace\n", - ).replace(' expose: ["5678", "5679"]\n', "") - assert inferred != authored - compose.write_text(inferred, encoding="utf-8") - sidecar = task / "environment/workspace" - sidecar.mkdir() - (sidecar / "Dockerfile").write_text( - "FROM python:3.11-alpine\nEXPOSE 5678 5679\n", - encoding="utf-8", - ) - - runs = asyncio.run(_grade_every_task(dataset, wheel)) - - test_harbor_phase_behavior(runs, "sidecar-reachability") - - -def test_separate_verifier_artifact_materialization_is_repeatable( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-verifier-artifacts") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - - async def grade_twice() -> list[Run]: - taskset = _adapt(dataset, hud_requirement=str(wheel)) - job = await taskset.run( - Oracle({"sidecar-reachability": (task / "solution/solve.sh").read_text("utf-8")}), - runtime=Shared(DockerRuntime(), width=1), - group=2, - max_concurrent=1, - ) - return job.runs - - runs = asyncio.run(grade_twice()) - - assert len(runs) == 2 - assert all(run.reward == 1.0 for run in runs) - - -def test_separate_verifier_artifacts_are_accessible_to_child_user( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-child-user") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - dockerfile = task / "tests/Dockerfile" - dockerfile.write_text( - dockerfile.read_text("utf-8").replace("USER verifier\n", "USER root\n"), - encoding="utf-8", - ) - (task / "tests/test.sh").write_text( - "#!/bin/sh\n" - "set -eu\n" - "mkdir -p /logs/verifier\n" - "if su verifier -s /bin/sh -c 'test \"$(cat /root/agent-output.txt)\" = private'; then\n" - " echo 1 > /logs/verifier/reward.txt\n" - "else\n" - " echo 0 > /logs/verifier/reward.txt\n" - "fi\n", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 1.0 - - -def test_fedora_environment_bootstrap( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-fedora") / "harbor-harness" - task = dataset / "fedora-bootstrap" - (task / "environment").mkdir(parents=True) - (task / "tests").mkdir() - (task / "solution").mkdir() - (task / "task.toml").write_text( - '[task]\nname = "fedora-bootstrap"\n\n[verifier]\ntimeout_sec = 30\n', - encoding="utf-8", - ) - (task / "instruction.md").write_text("Verify Fedora bootstrap.\n", encoding="utf-8") - (task / "environment/Dockerfile").write_text( - "FROM fedora:42\nWORKDIR /app\n", - encoding="utf-8", - ) - (task / "tests/test.sh").write_text( - "#!/bin/bash\necho 1 > /logs/verifier/reward.txt\n", - encoding="utf-8", - ) - (task / "solution/solve.sh").write_text("true\n", encoding="utf-8") - - run = asyncio.run(_grade_every_task(dataset, wheel))["fedora-bootstrap"] - - assert run.reward == 1.0 - - -def test_separate_verifier_rejects_artifact_symlinks( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-symlink") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - solution = task / "solution" / "solve.sh" - solution.write_text( - solution.read_text("utf-8") - + "\nrm -f /app/main.html\n" - + "ln -s /verifier/tests /app/main.html\n", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 0.0 - assert "artifact /app/main.html is a symbolic link" in (run.trace.error or "") - - -def test_separate_verifier_rejects_sidecar_symlink_artifact_roots( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-sidecar-symlink") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "task.toml").write_text( - """\ -artifacts = [{ source = "/link", service = "workspace", exclude = ["main.html"] }] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 - -[[verifier.collect]] -service = "workspace" -command = "ln -sfn /app /link" -timeout_sec = 10 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text("true\n", encoding="utf-8") - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 0.0 - assert "artifact /link contains a symbolic link" in (run.trace.error or "") - - -def test_separate_verifier_sees_directory_artifacts_without_excluded_entries( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-artifact-exclude") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "task.toml").write_text( - """\ -artifacts = [ - { source = "/app/outputs", destination = "results", exclude = ["*.tmp", "cache"] }, -] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text( - """\ -#!/bin/sh -set -eu -mkdir -p /app/outputs/cache/nested /app/outputs/logs -echo keep > /app/outputs/keep.txt -echo junk > /app/outputs/junk.tmp -echo junk > /app/outputs/logs/nested.tmp -echo junk > /app/outputs/cache/nested/blob -ln -s /etc/passwd /app/outputs/cache/link -ln -s missing /app/outputs/logs/dangling.tmp -""", - encoding="utf-8", - ) - (task / "tests" / "test.sh").write_text( - """\ -#!/bin/sh -set -u -mkdir -p /logs/verifier -if [ "$(cat /app/outputs/keep.txt 2>/dev/null)" = "keep" ] \\ - && [ -d /app/outputs/logs ] \\ - && [ ! -e /app/outputs/junk.tmp ] \\ - && [ ! -e /app/outputs/logs/nested.tmp ] \\ - && [ ! -L /app/outputs/logs/dangling.tmp ] \\ - && [ ! -e /app/outputs/cache ]; then - echo 1 > /logs/verifier/reward.txt -else - echo "excluded artifact entries leaked into the verifier" >&2 - echo 0 > /logs/verifier/reward.txt -fi -""", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 1.0, run.trace.error - - -def test_separate_verifier_collects_artifacts_written_to_authored_volumes( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-volume-artifacts") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "environment" / "docker-compose.yaml").write_text( - """\ -services: - main: - volumes: - - outputs:/app/outputs -volumes: - outputs: {} -""", - encoding="utf-8", - ) - (task / "task.toml").write_text( - """\ -artifacts = [{ source = "/app/outputs/result.txt", service = "main" }] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text( - "#!/bin/sh\nset -eu\necho from-volume > /app/outputs/result.txt\n", - encoding="utf-8", - ) - (task / "tests" / "test.sh").write_text( - """\ -#!/bin/sh -mkdir -p /logs/verifier -if [ "$(cat /app/outputs/result.txt 2>/dev/null)" = "from-volume" ]; then - echo 1 > /logs/verifier/reward.txt -else - echo 0 > /logs/verifier/reward.txt -fi -""", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 1.0, run.trace.error - - -def test_separate_verifier_rejects_artifacts_beneath_symlinks( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-ancestor-symlink") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - config = task / "task.toml" - config.write_text( - """\ -artifacts = [{ source = "/app/linked/tests", service = "main" }] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - solution = task / "solution" / "solve.sh" - solution.write_text( - "#!/bin/sh\nset -eu\nln -s /verifier /app/linked\n", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 0.0 - assert "artifact /app/linked/tests has a symbolic link in its path" in (run.trace.error or "") - - -def test_separate_verifier_restores_image_paths_between_shared_rollouts( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-verifier-restore") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "task.toml").write_text( - """\ -artifacts = [{ source = "/tests/test.sh", service = "main" }] - -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text("true\n", encoding="utf-8") - - async def grade_twice() -> list[Run]: - taskset = _adapt(dataset, hud_requirement=str(wheel)) - job = await taskset.run( - Oracle({"sidecar-reachability": "true"}), - runtime=Shared(DockerRuntime(), width=1), - group=2, - max_concurrent=1, - ) - return job.runs - - runs = asyncio.run(grade_twice()) - - assert len(runs) == 2 - for run in runs: - assert any( - step.task_call is not None - and step.task_call.phase == "evaluate" - and step.task_call.name == "verify" - for step in run.trace.steps - ) - - -def test_separate_verifier_honors_the_test_script_shebang( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - dataset = tmp_path_factory.mktemp("harbor-verifier-shebang") / "harbor-harness" - task = dataset / "sidecar-reachability" - shutil.copytree(TASKS / "sidecar-reachability", task) - (task / "task.toml").write_text( - """\ -[task] -name = "sidecar-reachability" - -[verifier] -environment_mode = "separate" -timeout_sec = 30 -""", - encoding="utf-8", - ) - (task / "solution" / "solve.sh").write_text("true\n", encoding="utf-8") - (task / "tests" / "test.sh").write_text( - """\ -#!/usr/bin/env python3 -from pathlib import Path - -Path("/logs/verifier/reward.txt").write_text("1") -""", - encoding="utf-8", - ) - - run = asyncio.run(_grade_every_task(dataset, wheel))["sidecar-reachability"] - - assert run.reward == 1.0 - - -def test_env_templates_resolve_from_runtime_env_vars( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - """``${VAR}``/``${VAR:-default}`` env values resolve exactly like Harbor's - host-side resolution, sourced from the runtime's ``env_vars``.""" - dataset = tmp_path_factory.mktemp("harbor-env-templates") / "harbor-harness" - task = make_harbor_task( - dataset, - "env-templates", - task_toml="""\ -[metadata] -category = "systems" - -[environment.env] -GREETING = "${HARBOR_GREETING:-hello}" -JUDGE_KEY = "${HARBOR_JUDGE_KEY}" -EMBEDDED = "Bearer ${HARBOR_JUDGE_KEY}" - -[verifier] -timeout_sec = 120 - -[verifier.env] -VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" -EMPTY_DEFAULT = "${HARBOR_UNSET:-}" -""", - ) - (task / "tests" / "test.sh").write_text( - """\ -#!/bin/bash -fail() { echo "unexpected $1"; echo "0.0" > /logs/verifier/reward.txt; exit 0; } -[ "$GREETING" = "hello" ] || fail "GREETING=$GREETING" -[ "$JUDGE_KEY" = "judge-secret" ] || fail "JUDGE_KEY=$JUDGE_KEY" -[ "$EMBEDDED" = 'Bearer ${HARBOR_JUDGE_KEY}' ] || fail "EMBEDDED=$EMBEDDED" -[ "$VERIFIER_KEY" = "judge-secret" ] || fail "VERIFIER_KEY=$VERIFIER_KEY" -[ "${EMPTY_DEFAULT-unset}" = "" ] || fail "EMPTY_DEFAULT=${EMPTY_DEFAULT-unset}" -echo "1.0" > /logs/verifier/reward.txt -""", - encoding="utf-8", - ) - solution = '[ "$JUDGE_KEY" = "judge-secret" ] && [ "$GREETING" = "hello" ]' - - async def grade() -> Run: - taskset = harbor.adapt(dataset, hud_requirement=str(wheel)).taskset - job = await taskset.run( - Oracle({"env-templates": solution}), - runtime=DockerRuntime( - env_vars={"HARBOR_GREETING": "", "HARBOR_JUDGE_KEY": "judge-secret"} - ), - max_concurrent=1, - ) - (run,) = job.runs - return run - - run = asyncio.run(grade()) - - evaluation = run.evaluation - info = evaluation.get("info") or {} - detail = "\n".join( - filter( - None, - ( - run.trace.content, - run.trace.error, - evaluation.get("content") or "", - info.get("stdout"), - info.get("stderr"), - ), - ) - ) - assert run.reward == 1.0, f"env-templates scored {run.reward}; the verifier reported:\n{detail}" - - -def test_missing_env_template_aborts_startup( - tmp_path_factory: pytest.TempPathFactory, wheel: Path -) -> None: - """A required ``${VAR}`` with no default and no runtime value must abort - environment startup with an error naming the variable, like Harbor.""" - dataset = tmp_path_factory.mktemp("harbor-env-missing") / "harbor-harness" - make_harbor_task( - dataset, - "env-missing", - task_toml="""\ -[metadata] -category = "systems" - -[environment.env] -API_KEY = "${HARBOR_MISSING_KEY}" - -[verifier] -timeout_sec = 120 -""", - ) - - taskset = harbor.adapt(dataset, hud_requirement=str(wheel)).taskset - task = next(iter(taskset)) - assert task.runtime_config is not None - assert task.runtime_config.compose is not None - compose = task.runtime_config.compose.document - assert isinstance(compose, Path) - # Providers yield as soon as the published port exists, before the serve - # process proves itself, so an early abort is only observable from the - # adapted artifact: run its main service in the foreground. - subprocess.run( - ["docker", "compose", "--file", str(compose), "build"], - cwd=compose.parent, - check=True, - capture_output=True, - timeout=600, - ) - command = ["docker", "compose", "--file", str(compose), "run", "--rm", "main"] - try: - serve = subprocess.run(command, capture_output=True, text=True, timeout=60) - except subprocess.TimeoutExpired: - pytest.fail("main service kept serving despite an unresolvable env template") - finally: - subprocess.run( - ["docker", "compose", "--file", str(compose), "down", "--volumes", "--remove-orphans"], - capture_output=True, - check=False, - ) - assert serve.returncode != 0 - assert "HARBOR_MISSING_KEY" in serve.stdout + serve.stderr - - -@pytest.mark.parametrize("mode", ["shared", "separate"]) -@pytest.mark.parametrize("credential", ["missing", "provided", "default"]) -def test_verifier_credentials_are_required_only_for_grading( - tmp_path: Path, wheel: Path, mode: str, credential: str -) -> None: - dataset = tmp_path / "harbor-harness" - template = ( - "${HARBOR_VERIFIER_KEY:-judge-value}" - if credential == "default" - else "${HARBOR_VERIFIER_KEY}" - ) - task = make_harbor_task( - dataset, - "verifier-credentials", - task_toml=f"""[verifier] -environment_mode = "{mode}" -timeout_sec = 30 - -[verifier.env] -JUDGE_KEY = "{template}" -""", - dockerfile="FROM python:3.11-slim\nWORKDIR /workspace\n", - ) - (task / "tests/test.sh").write_text( - '#!/bin/sh\nset -eu\n[ "$JUDGE_KEY" = "judge-value" ]\n' - "echo 1 > /logs/verifier/reward.txt\n", - encoding="utf-8", - ) - if mode == "separate": - (task / "tests/Dockerfile").write_text( - "FROM python:3.11-slim\nCOPY . /tests\n", encoding="utf-8" - ) - - async def grade() -> Run: - taskset = _adapt(dataset, hud_requirement=str(wheel)) - job = await taskset.run( - Oracle({"verifier-credentials": '[ "${JUDGE_KEY-unset}" = unset ]'}), - runtime=DockerRuntime( - env_vars={"HARBOR_VERIFIER_KEY": "judge-value"} if credential == "provided" else {} - ), - max_concurrent=1, - ) - (run,) = job.runs - return run - - run = asyncio.run(grade()) - - assert run.trace.content == "solution completed" - if credential == "missing": - assert run.reward != 1.0 - assert "HARBOR_VERIFIER_KEY" in (run.trace.error or str(run.evaluation)) - else: - assert run.reward == 1.0, run.trace.error or run.evaluation - - -def test_main_image_ports_preserve_the_outer_control_channel( - tmp_path_factory: pytest.TempPathFactory, - wheel: Path, -) -> None: - dataset = tmp_path_factory.mktemp("harbor-main-ports") / "harbor-harness" - task = make_harbor_task( - dataset, - "main-ports", - dockerfile="""\ -FROM debian:bookworm-slim -WORKDIR /app -COPY entrypoint.sh /usr/local/bin/start-environment -RUN chmod +x /usr/local/bin/start-environment && printf nested > /app/marker -EXPOSE 3128 3129 8765 8080 -ENTRYPOINT ["/usr/local/bin/start-environment"] -""", - task_toml="""\ -[environment.healthcheck] -command = "curl -fsS http://127.0.0.1:8765/marker && curl -fsS http://127.0.0.1:8080/marker" -interval_sec = 0.1 -timeout_sec = 5 -retries = 50 - -[verifier] -timeout_sec = 30 - -[[verifier.collect]] -service = "observer" -command = ''' -python - <<'PY' -from urllib.request import urlopen -assert urlopen('http://main:8080/marker').read() == b'nested' -PY -''' -timeout_sec = 10 -""", - ) - (task / "environment/entrypoint.sh").write_text( - "#!/bin/sh\nset -eu\n" - "python3 -m http.server 8765 --directory /app >/tmp/inner.log 2>&1 &\n" - "python3 -m http.server 8080 --directory /app >/tmp/app.log 2>&1 &\n" - 'exec "$@"\n', - encoding="utf-8", - ) - (task / "environment/docker-compose.yaml").write_text( - "services:\n main: {}\n observer:\n image: python:3.11-alpine\n" - ' command: ["python", "-m", "http.server", "9000"]\n expose: [9000]\n', - encoding="utf-8", - ) - (task / "solution").mkdir() - (task / "solution/solve.sh").write_text( - "curl -fsS http://127.0.0.1:8765/marker > /app/result\n", - encoding="utf-8", - ) - (task / "tests/test.sh").write_text( - '#!/bin/sh\nset -eu\ntest "$(cat /app/result)" = nested\n' - "echo 1 > /logs/verifier/reward.txt\n", - encoding="utf-8", - ) - - runs = asyncio.run(_grade_every_task(dataset, wheel)) - - test_harbor_phase_behavior(runs, "main-ports") - - -def test_adapter_install_ignores_vendor_uv_configuration( - tmp_path_factory: pytest.TempPathFactory, - wheel: Path, -) -> None: - """The private HUD install does not consume or rewrite the task's uv settings.""" - dataset = tmp_path_factory.mktemp("harbor-vendor-uv") / "harbor-harness" - task = make_harbor_task( - dataset, - "vendor-uv", - dockerfile="FROM python:3.11-slim\nWORKDIR /workspace\nCOPY pyproject.toml .\n", - ) - config = '[tool.uv]\nrequired-version = "==0.0.0"\n[tool.uv.pip]\nno-index = true\n' - config_path = task / "environment/pyproject.toml" - config_path.write_text(config, encoding="utf-8") - (adapted,) = _adapt(dataset, hud_requirement=str(wheel)) - assert adapted.runtime_config is not None - assert adapted.runtime_config.compose is not None - compose = adapted.runtime_config.compose.document - context = adapted.runtime_config.compose.root - assert isinstance(compose, Path) - assert isinstance(context, Path) - - build = subprocess.run( - ["docker", "compose", "--file", str(compose), "build"], - cwd=compose.parent, - capture_output=True, - text=True, - check=False, - timeout=600, - ) - - assert build.returncode == 0, build.stdout[-4000:] + build.stderr[-4000:] - assert config_path.read_text(encoding="utf-8") == config - copied_config = context / "compose-project/environment/pyproject.toml" - assert copied_config.read_text(encoding="utf-8") == config - - -def test_agent_sessions_run_the_managed_cli_bundle( - tmp_path_factory: pytest.TempPathFactory, - wheel: Path, -) -> None: - """A CLI bundle the runtime mounts at /usr/local/lib/agents runs in agent sessions.""" - root = tmp_path_factory.mktemp("harbor-managed-agents") - binary = root / "codex" / "bin" / "codex" - binary.parent.mkdir(parents=True) - binary.write_text("#!/bin/sh\necho managed-codex\n", encoding="utf-8") - binary.chmod(0o755) - dataset = root / "harbor-harness" - task = make_harbor_task( - dataset, - "managed-agents", - dockerfile="FROM python:3.11-slim\nWORKDIR /workspace\n", - ) - (task / "tests/test.sh").write_text( - '#!/bin/sh\nset -eu\ntest "$(cat /workspace/result)" = managed-codex\n' - "echo 1 > /logs/verifier/reward.txt\n", - encoding="utf-8", - ) - - async def grade() -> Run: - job = await _adapt(dataset, hud_requirement=str(wheel)).run( - Oracle({"managed-agents": "/usr/local/lib/agents/codex/bin/codex > result"}), - runtime=DockerRuntime( - bind_mounts=(DockerBindMount(root / "codex", "/usr/local/lib/agents/codex"),), - ), - max_concurrent=1, - ) - (run,) = job.runs - return run - - run = asyncio.run(grade()) - - assert run.trace.content == "solution completed" - assert run.reward == 1.0, run.trace.error or run.evaluation From 13449b6c61d9220902e6dc49560df31723f3ff51 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 070/248] test: run the Harbor controller through fixture tasks with a generic oracle test_integration.py was the only coverage of the in-image controller. Its oracle branched per task to drive a 40-session SSH loop and an MCP call, and its variants were built by rewriting fixtures in Python. Each case is now a fixture directory under tasks/ with an optional expected.toml (reward, error text, runtime env vars and bind mounts, group and shared width, a host-side MCP probe, sessions to repeat, or a startup refusal), and one oracle that only runs solution/solve.sh. The Python-built variants are fixture directories of their own, and new fixtures cover the controller contracts nothing pinned: reward.json over reward.txt and the reward shapes it accepts and refuses, a verifier timeout, missing rewards, unknown agent and verifier users and groups, failing and timed-out collect hooks, an exhausted healthcheck, an entrypoint that exits, and the agent's answer reaching the verifier in both placements. The suite is marked e2e and docker. The wheel fixture moves to the Harbor conftest, shared with the export docker lane, and the builders there go with the tests that used them. --- tests/integrations/harbor/conftest.py | 140 ++--------- .../environment/Dockerfile | 2 + .../agent-answer-separate/instruction.md | 1 + .../agent-answer-separate/solution/solve.sh | 2 + .../tasks/agent-answer-separate/task.toml | 3 + .../agent-answer-separate/tests/Dockerfile | 2 + .../tasks/agent-answer-separate/tests/test.sh | 7 + .../tasks/agent-answer/environment/Dockerfile | 2 + .../harbor/tasks/agent-answer/expected.toml | 1 + .../harbor/tasks/agent-answer/instruction.md | 1 + .../tasks/agent-answer/solution/solve.sh | 2 + .../harbor/tasks/agent-answer/task.toml | 2 + .../harbor/tasks/agent-answer/tests/test.sh | 7 + .../environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 11 + .../artifact-ancestor-symlink/expected.toml | 2 + .../artifact-ancestor-symlink/instruction.md | 1 + .../solution/solve.sh | 3 + .../tasks/artifact-ancestor-symlink/task.toml | 8 + .../tests/Dockerfile | 16 ++ .../artifact-ancestor-symlink/tests/test.sh | 32 +++ .../artifact-exclude/environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 11 + .../tasks/artifact-exclude/instruction.md | 1 + .../tasks/artifact-exclude/solution/solve.sh | 9 + .../harbor/tasks/artifact-exclude/task.toml | 10 + .../tasks/artifact-exclude/tests/Dockerfile | 16 ++ .../tasks/artifact-exclude/tests/test.sh | 14 ++ .../environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 11 + .../expected.toml | 2 + .../instruction.md | 1 + .../solution/solve.sh | 2 + .../artifact-sidecar-symlink-root/task.toml | 13 ++ .../tests/Dockerfile | 16 ++ .../tests/test.sh | 32 +++ .../artifact-symlink/environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 11 + .../tasks/artifact-symlink/expected.toml | 2 + .../tasks/artifact-symlink/instruction.md | 1 + .../tasks/artifact-symlink/solution/solve.sh | 41 ++++ .../harbor/tasks/artifact-symlink/task.toml | 30 +++ .../tasks/artifact-symlink/tests/Dockerfile | 16 ++ .../tasks/artifact-symlink/tests/test.sh | 32 +++ .../artifact-volume/environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 6 + .../tasks/artifact-volume/instruction.md | 1 + .../tasks/artifact-volume/solution/solve.sh | 3 + .../harbor/tasks/artifact-volume/task.toml | 8 + .../tasks/artifact-volume/tests/Dockerfile | 16 ++ .../tasks/artifact-volume/tests/test.sh | 7 + .../bad-agent-group/environment/Dockerfile | 2 + .../tasks/bad-agent-group/expected.toml | 2 + .../tasks/bad-agent-group/instruction.md | 1 + .../tasks/bad-agent-group/solution/solve.sh | 2 + .../harbor/tasks/bad-agent-group/task.toml | 5 + .../tasks/bad-agent-group/tests/test.sh | 2 + .../bad-agent-user/environment/Dockerfile | 2 + .../harbor/tasks/bad-agent-user/expected.toml | 2 + .../tasks/bad-agent-user/instruction.md | 1 + .../tasks/bad-agent-user/solution/solve.sh | 2 + .../harbor/tasks/bad-agent-user/task.toml | 5 + .../harbor/tasks/bad-agent-user/tests/test.sh | 2 + .../bad-verifier-user/environment/Dockerfile | 2 + .../tasks/bad-verifier-user/expected.toml | 2 + .../tasks/bad-verifier-user/instruction.md | 1 + .../tasks/bad-verifier-user/solution/solve.sh | 2 + .../harbor/tasks/bad-verifier-user/task.toml | 3 + .../tasks/bad-verifier-user/tests/test.sh | 2 + .../collect-hook-fails/environment/Dockerfile | 2 + .../tasks/collect-hook-fails/expected.toml | 2 + .../tasks/collect-hook-fails/instruction.md | 1 + .../collect-hook-fails/solution/solve.sh | 2 + .../harbor/tasks/collect-hook-fails/task.toml | 7 + .../tasks/collect-hook-fails/tests/Dockerfile | 2 + .../tasks/collect-hook-fails/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../collect-hook-times-out/expected.toml | 2 + .../collect-hook-times-out/instruction.md | 1 + .../collect-hook-times-out/solution/solve.sh | 2 + .../tasks/collect-hook-times-out/task.toml | 7 + .../collect-hook-times-out/tests/Dockerfile | 2 + .../collect-hook-times-out/tests/test.sh | 2 + .../entrypoint-exits/environment/Dockerfile | 3 + .../tasks/entrypoint-exits/expected.toml | 2 + .../tasks/entrypoint-exits/instruction.md | 1 + .../tasks/entrypoint-exits/solution/solve.sh | 2 + .../harbor/tasks/entrypoint-exits/task.toml | 2 + .../tasks/entrypoint-exits/tests/test.sh | 2 + .../environment/Dockerfile | 2 + .../tasks/env-template-missing/expected.toml | 2 + .../tasks/env-template-missing/instruction.md | 1 + .../env-template-missing/solution/solve.sh | 2 + .../tasks/env-template-missing/task.toml | 5 + .../tasks/env-template-missing/tests/test.sh | 2 + .../env-templates/environment/Dockerfile | 2 + .../harbor/tasks/env-templates/expected.toml | 4 + .../harbor/tasks/env-templates/instruction.md | 1 + .../tasks/env-templates/solution/solve.sh | 2 + .../harbor/tasks/env-templates/task.toml | 11 + .../harbor/tasks/env-templates/tests/test.sh | 8 + .../fedora-bootstrap/environment/Dockerfile | 2 + .../tasks/fedora-bootstrap/instruction.md | 1 + .../tasks/fedora-bootstrap/solution/solve.sh | 2 + .../harbor/tasks/fedora-bootstrap/task.toml | 5 + .../tasks/fedora-bootstrap/tests/test.sh | 2 + .../healthcheck-fails/environment/Dockerfile | 2 + .../tasks/healthcheck-fails/expected.toml | 2 + .../tasks/healthcheck-fails/instruction.md | 1 + .../tasks/healthcheck-fails/solution/solve.sh | 2 + .../harbor/tasks/healthcheck-fails/task.toml | 8 + .../tasks/healthcheck-fails/tests/test.sh | 2 + .../harbor/tasks/hello-mcp/expected.toml | 2 + .../harbor/tasks/hello-mcp/solution/solve.sh | 46 +++- .../tasks/main-ports/environment/Dockerfile | 6 + .../environment/docker-compose.yaml | 6 + .../main-ports/environment/entrypoint.sh | 5 + .../harbor/tasks/main-ports/instruction.md | 1 + .../harbor/tasks/main-ports/solution/solve.sh | 2 + .../harbor/tasks/main-ports/task.toml | 18 ++ .../harbor/tasks/main-ports/tests/test.sh | 4 + .../managed-agents/environment/Dockerfile | 2 + .../harbor/tasks/managed-agents/expected.toml | 4 + .../tasks/managed-agents/instruction.md | 1 + .../managed-agents/managed/codex/bin/codex | 2 + .../tasks/managed-agents/solution/solve.sh | 3 + .../harbor/tasks/managed-agents/task.toml | 2 + .../harbor/tasks/managed-agents/tests/test.sh | 4 + .../tasks/reward-files/environment/Dockerfile | 2 + .../harbor/tasks/reward-files/expected.toml | 56 +++++ .../harbor/tasks/reward-files/instruction.md | 1 + .../tasks/reward-files/solution/solve.sh | 2 + .../harbor/tasks/reward-files/task.toml | 5 + .../harbor/tasks/reward-files/tests/test.sh | 16 ++ .../environment/Dockerfile | 2 + .../tasks/session-reliability/expected.toml | 2 + .../tasks/session-reliability/instruction.md | 1 + .../session-reliability/solution/solve.sh | 3 + .../tasks/session-reliability/task.toml | 2 + .../tasks/session-reliability/tests/test.sh | 7 + .../environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 10 + .../environment/workspace/Dockerfile | 2 + .../tasks/sidecar-built-ports/instruction.md | 1 + .../sidecar-built-ports/solution/solve.sh | 39 ++++ .../tasks/sidecar-built-ports/task.toml | 30 +++ .../sidecar-built-ports/tests/Dockerfile | 16 ++ .../tasks/sidecar-built-ports/tests/test.sh | 32 +++ .../environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 11 + .../instruction.md | 1 + .../solution/solve.sh | 39 ++++ .../sidecar-reachability-placed/task.toml | 33 +++ .../tests/Dockerfile | 16 ++ .../sidecar-reachability-placed/tests/test.sh | 32 +++ .../environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 11 + .../expected.toml | 3 + .../instruction.md | 1 + .../solution/solve.sh | 39 ++++ .../sidecar-reachability-repeated/task.toml | 30 +++ .../tests/Dockerfile | 16 ++ .../tests/test.sh | 32 +++ .../tasks/vendor-uv/environment/Dockerfile | 3 + .../vendor-uv/environment/pyproject.toml | 4 + .../harbor/tasks/vendor-uv/instruction.md | 1 + .../harbor/tasks/vendor-uv/solution/solve.sh | 2 + .../harbor/tasks/vendor-uv/task.toml | 2 + .../harbor/tasks/vendor-uv/tests/test.sh | 7 + .../environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 11 + .../tasks/verifier-child-user/instruction.md | 1 + .../verifier-child-user/solution/solve.sh | 39 ++++ .../tasks/verifier-child-user/task.toml | 30 +++ .../verifier-child-user/tests/Dockerfile | 16 ++ .../tasks/verifier-child-user/tests/test.sh | 8 + .../environment/Dockerfile | 2 + .../expected.toml | 1 + .../instruction.md | 1 + .../solution/solve.sh | 2 + .../task.toml | 6 + .../tests/Dockerfile | 2 + .../tests/test.sh | 4 + .../environment/Dockerfile | 2 + .../expected.toml | 1 + .../instruction.md | 1 + .../solution/solve.sh | 2 + .../task.toml | 6 + .../tests/test.sh | 4 + .../environment/Dockerfile | 2 + .../expected.toml | 9 + .../instruction.md | 1 + .../solution/solve.sh | 2 + .../verifier-credentials-separate/task.toml | 6 + .../tests/Dockerfile | 2 + .../tests/test.sh | 4 + .../environment/Dockerfile | 2 + .../verifier-credentials-shared/expected.toml | 9 + .../instruction.md | 1 + .../solution/solve.sh | 2 + .../verifier-credentials-shared/task.toml | 6 + .../verifier-credentials-shared/tests/test.sh | 4 + .../verifier-restore/environment/Dockerfile | 2 + .../tasks/verifier-restore/expected.toml | 2 + .../tasks/verifier-restore/instruction.md | 1 + .../tasks/verifier-restore/solution/solve.sh | 4 + .../harbor/tasks/verifier-restore/task.toml | 5 + .../tasks/verifier-restore/tests/Dockerfile | 3 + .../tasks/verifier-restore/tests/test.sh | 10 + .../verifier-shebang/environment/Dockerfile | 5 + .../environment/docker-compose.yaml | 11 + .../tasks/verifier-shebang/instruction.md | 1 + .../tasks/verifier-shebang/solution/solve.sh | 2 + .../harbor/tasks/verifier-shebang/task.toml | 6 + .../tasks/verifier-shebang/tests/Dockerfile | 16 ++ .../tasks/verifier-shebang/tests/test.sh | 4 + .../verifier-timeout/environment/Dockerfile | 2 + .../tasks/verifier-timeout/expected.toml | 2 + .../tasks/verifier-timeout/instruction.md | 1 + .../tasks/verifier-timeout/solution/solve.sh | 2 + .../harbor/tasks/verifier-timeout/task.toml | 2 + .../tasks/verifier-timeout/tests/test.sh | 3 + tests/integrations/harbor/test_controller.py | 219 ++++++++++++++++++ tests/integrations/harbor/test_export.py | 16 +- 224 files changed, 1747 insertions(+), 142 deletions(-) create mode 100644 tests/integrations/harbor/tasks/agent-answer-separate/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/agent-answer-separate/instruction.md create mode 100644 tests/integrations/harbor/tasks/agent-answer-separate/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/agent-answer-separate/task.toml create mode 100644 tests/integrations/harbor/tasks/agent-answer-separate/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/agent-answer-separate/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/agent-answer/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/agent-answer/expected.toml create mode 100644 tests/integrations/harbor/tasks/agent-answer/instruction.md create mode 100644 tests/integrations/harbor/tasks/agent-answer/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/agent-answer/task.toml create mode 100644 tests/integrations/harbor/tasks/agent-answer/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/artifact-ancestor-symlink/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-ancestor-symlink/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/artifact-ancestor-symlink/expected.toml create mode 100644 tests/integrations/harbor/tasks/artifact-ancestor-symlink/instruction.md create mode 100644 tests/integrations/harbor/tasks/artifact-ancestor-symlink/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/artifact-ancestor-symlink/task.toml create mode 100644 tests/integrations/harbor/tasks/artifact-ancestor-symlink/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-ancestor-symlink/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/artifact-exclude/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-exclude/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/artifact-exclude/instruction.md create mode 100644 tests/integrations/harbor/tasks/artifact-exclude/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/artifact-exclude/task.toml create mode 100644 tests/integrations/harbor/tasks/artifact-exclude/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-exclude/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/expected.toml create mode 100644 tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/instruction.md create mode 100644 tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/task.toml create mode 100644 tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/artifact-symlink/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-symlink/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/artifact-symlink/expected.toml create mode 100644 tests/integrations/harbor/tasks/artifact-symlink/instruction.md create mode 100644 tests/integrations/harbor/tasks/artifact-symlink/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/artifact-symlink/task.toml create mode 100644 tests/integrations/harbor/tasks/artifact-symlink/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-symlink/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/artifact-volume/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-volume/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/artifact-volume/instruction.md create mode 100644 tests/integrations/harbor/tasks/artifact-volume/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/artifact-volume/task.toml create mode 100644 tests/integrations/harbor/tasks/artifact-volume/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/artifact-volume/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/bad-agent-group/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/bad-agent-group/expected.toml create mode 100644 tests/integrations/harbor/tasks/bad-agent-group/instruction.md create mode 100644 tests/integrations/harbor/tasks/bad-agent-group/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/bad-agent-group/task.toml create mode 100644 tests/integrations/harbor/tasks/bad-agent-group/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/bad-agent-user/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/bad-agent-user/expected.toml create mode 100644 tests/integrations/harbor/tasks/bad-agent-user/instruction.md create mode 100644 tests/integrations/harbor/tasks/bad-agent-user/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/bad-agent-user/task.toml create mode 100644 tests/integrations/harbor/tasks/bad-agent-user/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/bad-verifier-user/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/bad-verifier-user/expected.toml create mode 100644 tests/integrations/harbor/tasks/bad-verifier-user/instruction.md create mode 100644 tests/integrations/harbor/tasks/bad-verifier-user/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/bad-verifier-user/task.toml create mode 100644 tests/integrations/harbor/tasks/bad-verifier-user/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/collect-hook-fails/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/collect-hook-fails/expected.toml create mode 100644 tests/integrations/harbor/tasks/collect-hook-fails/instruction.md create mode 100644 tests/integrations/harbor/tasks/collect-hook-fails/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/collect-hook-fails/task.toml create mode 100644 tests/integrations/harbor/tasks/collect-hook-fails/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/collect-hook-fails/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/collect-hook-times-out/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/collect-hook-times-out/expected.toml create mode 100644 tests/integrations/harbor/tasks/collect-hook-times-out/instruction.md create mode 100644 tests/integrations/harbor/tasks/collect-hook-times-out/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/collect-hook-times-out/task.toml create mode 100644 tests/integrations/harbor/tasks/collect-hook-times-out/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/collect-hook-times-out/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/entrypoint-exits/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/entrypoint-exits/expected.toml create mode 100644 tests/integrations/harbor/tasks/entrypoint-exits/instruction.md create mode 100644 tests/integrations/harbor/tasks/entrypoint-exits/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/entrypoint-exits/task.toml create mode 100644 tests/integrations/harbor/tasks/entrypoint-exits/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/env-template-missing/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/env-template-missing/expected.toml create mode 100644 tests/integrations/harbor/tasks/env-template-missing/instruction.md create mode 100644 tests/integrations/harbor/tasks/env-template-missing/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/env-template-missing/task.toml create mode 100644 tests/integrations/harbor/tasks/env-template-missing/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/env-templates/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/env-templates/expected.toml create mode 100644 tests/integrations/harbor/tasks/env-templates/instruction.md create mode 100644 tests/integrations/harbor/tasks/env-templates/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/env-templates/task.toml create mode 100644 tests/integrations/harbor/tasks/env-templates/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/fedora-bootstrap/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/fedora-bootstrap/instruction.md create mode 100644 tests/integrations/harbor/tasks/fedora-bootstrap/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/fedora-bootstrap/task.toml create mode 100644 tests/integrations/harbor/tasks/fedora-bootstrap/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/healthcheck-fails/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/healthcheck-fails/expected.toml create mode 100644 tests/integrations/harbor/tasks/healthcheck-fails/instruction.md create mode 100644 tests/integrations/harbor/tasks/healthcheck-fails/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/healthcheck-fails/task.toml create mode 100644 tests/integrations/harbor/tasks/healthcheck-fails/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/hello-mcp/expected.toml create mode 100644 tests/integrations/harbor/tasks/main-ports/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/main-ports/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/main-ports/environment/entrypoint.sh create mode 100644 tests/integrations/harbor/tasks/main-ports/instruction.md create mode 100644 tests/integrations/harbor/tasks/main-ports/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/main-ports/task.toml create mode 100644 tests/integrations/harbor/tasks/main-ports/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/managed-agents/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/managed-agents/expected.toml create mode 100644 tests/integrations/harbor/tasks/managed-agents/instruction.md create mode 100755 tests/integrations/harbor/tasks/managed-agents/managed/codex/bin/codex create mode 100644 tests/integrations/harbor/tasks/managed-agents/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/managed-agents/task.toml create mode 100644 tests/integrations/harbor/tasks/managed-agents/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/reward-files/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/reward-files/expected.toml create mode 100644 tests/integrations/harbor/tasks/reward-files/instruction.md create mode 100644 tests/integrations/harbor/tasks/reward-files/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/reward-files/task.toml create mode 100644 tests/integrations/harbor/tasks/reward-files/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/session-reliability/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/session-reliability/expected.toml create mode 100644 tests/integrations/harbor/tasks/session-reliability/instruction.md create mode 100644 tests/integrations/harbor/tasks/session-reliability/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/session-reliability/task.toml create mode 100644 tests/integrations/harbor/tasks/session-reliability/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/sidecar-built-ports/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/sidecar-built-ports/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/sidecar-built-ports/environment/workspace/Dockerfile create mode 100644 tests/integrations/harbor/tasks/sidecar-built-ports/instruction.md create mode 100644 tests/integrations/harbor/tasks/sidecar-built-ports/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/sidecar-built-ports/task.toml create mode 100644 tests/integrations/harbor/tasks/sidecar-built-ports/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/sidecar-built-ports/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-placed/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-placed/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-placed/instruction.md create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-placed/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-placed/task.toml create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-placed/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-placed/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-repeated/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-repeated/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-repeated/expected.toml create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-repeated/instruction.md create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-repeated/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-repeated/task.toml create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-repeated/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/sidecar-reachability-repeated/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/vendor-uv/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/vendor-uv/environment/pyproject.toml create mode 100644 tests/integrations/harbor/tasks/vendor-uv/instruction.md create mode 100644 tests/integrations/harbor/tasks/vendor-uv/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/vendor-uv/task.toml create mode 100644 tests/integrations/harbor/tasks/vendor-uv/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/verifier-child-user/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-child-user/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/verifier-child-user/instruction.md create mode 100644 tests/integrations/harbor/tasks/verifier-child-user/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/verifier-child-user/task.toml create mode 100644 tests/integrations/harbor/tasks/verifier-child-user/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-child-user/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-separate/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-separate/expected.toml create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-separate/instruction.md create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-separate/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-separate/task.toml create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-separate/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-separate/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-shared/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-shared/expected.toml create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-shared/instruction.md create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-shared/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-shared/task.toml create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-default-shared/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-separate/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-separate/expected.toml create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-separate/instruction.md create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-separate/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-separate/task.toml create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-separate/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-separate/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-shared/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-shared/expected.toml create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-shared/instruction.md create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-shared/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-shared/task.toml create mode 100644 tests/integrations/harbor/tasks/verifier-credentials-shared/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/verifier-restore/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-restore/expected.toml create mode 100644 tests/integrations/harbor/tasks/verifier-restore/instruction.md create mode 100644 tests/integrations/harbor/tasks/verifier-restore/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/verifier-restore/task.toml create mode 100644 tests/integrations/harbor/tasks/verifier-restore/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-restore/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/verifier-shebang/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-shebang/environment/docker-compose.yaml create mode 100644 tests/integrations/harbor/tasks/verifier-shebang/instruction.md create mode 100644 tests/integrations/harbor/tasks/verifier-shebang/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/verifier-shebang/task.toml create mode 100644 tests/integrations/harbor/tasks/verifier-shebang/tests/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-shebang/tests/test.sh create mode 100644 tests/integrations/harbor/tasks/verifier-timeout/environment/Dockerfile create mode 100644 tests/integrations/harbor/tasks/verifier-timeout/expected.toml create mode 100644 tests/integrations/harbor/tasks/verifier-timeout/instruction.md create mode 100644 tests/integrations/harbor/tasks/verifier-timeout/solution/solve.sh create mode 100644 tests/integrations/harbor/tasks/verifier-timeout/task.toml create mode 100644 tests/integrations/harbor/tasks/verifier-timeout/tests/test.sh create mode 100644 tests/integrations/harbor/test_controller.py diff --git a/tests/integrations/harbor/conftest.py b/tests/integrations/harbor/conftest.py index d50c86d90..34fd133de 100644 --- a/tests/integrations/harbor/conftest.py +++ b/tests/integrations/harbor/conftest.py @@ -1,135 +1,23 @@ -"""Builders for synthetic Harbor-format task directories (terminal-bench layout): - -task_name/ -├── task.toml -├── instruction.md -├── environment/Dockerfile -└── tests/test.sh -""" +"""Fixtures shared by the Harbor docker-lane scenarios.""" from __future__ import annotations -import textwrap -from pathlib import Path # noqa: TC003 - used at runtime +import subprocess +from pathlib import Path import pytest -_DEFAULT_TASK_TOML = textwrap.dedent("""\ - [metadata] - category = "systems" - difficulty = "medium" - tags = ["bash", "linux"] - - [verifier] - timeout_sec = 120 -""") - -_ML_TASK_TOML = textwrap.dedent("""\ - [metadata] - category = "machine-learning" - difficulty = "hard" - tags = ["python", "ml"] - - [docker] - image = "alexgshaw/caffe-cifar-10:20251031" - - [verifier] - timeout_sec = 300 -""") - -_SIMPLE_DOCKERFILE = textwrap.dedent("""\ - FROM python:3.11-slim - RUN apt-get update && apt-get install -y curl git - WORKDIR /workspace - CMD ["bash"] -""") - -_ML_DOCKERFILE = textwrap.dedent("""\ - FROM nvidia/cuda:12.0-runtime-ubuntu22.04 - RUN apt-get update && apt-get install -y python3 python3-pip - WORKDIR /workspace - ENTRYPOINT ["/bin/bash"] -""") - - -def make_harbor_task( - parent: Path, - name: str, - instruction: str = "Solve the task.", - task_toml: str = _DEFAULT_TASK_TOML, - dockerfile: str | None = _SIMPLE_DOCKERFILE, -) -> Path: - """Create a synthetic Harbor task directory under *parent*; return it.""" - task_dir = parent / name - task_dir.mkdir(parents=True, exist_ok=True) - (task_dir / "instruction.md").write_text(instruction, encoding="utf-8") - (task_dir / "task.toml").write_text(task_toml, encoding="utf-8") - if dockerfile is not None: - env_dir = task_dir / "environment" - env_dir.mkdir(exist_ok=True) - (env_dir / "Dockerfile").write_text(dockerfile, encoding="utf-8") - tests_dir = task_dir / "tests" - tests_dir.mkdir(exist_ok=True) - (tests_dir / "test.sh").write_text( - '#!/bin/bash\necho "1.0" > /logs/verifier/reward.txt\n', encoding="utf-8" - ) - return task_dir - - -def make_multi_step_task( - parent: Path, - name: str, - steps: tuple[str, ...] = ("plan", "implement"), - dockerfile: str | None = _SIMPLE_DOCKERFILE, -) -> Path: - """Create a synthetic multi-step Harbor task: no root instruction.md; each - step under steps// carries its own instruction, declared by a - [[steps]] array in task.toml.""" - task_dir = parent / name - task_dir.mkdir(parents=True, exist_ok=True) - steps_toml = "".join(f'\n[[steps]]\nname = "{step}"\n' for step in steps) - (task_dir / "task.toml").write_text(_DEFAULT_TASK_TOML + steps_toml, encoding="utf-8") - if dockerfile is not None: - env_dir = task_dir / "environment" - env_dir.mkdir(exist_ok=True) - (env_dir / "Dockerfile").write_text(dockerfile, encoding="utf-8") - for step in steps: - step_dir = task_dir / "steps" / step - (step_dir / "tests").mkdir(parents=True, exist_ok=True) - (step_dir / "instruction.md").write_text(f"# {step}\n", encoding="utf-8") - (step_dir / "tests" / "test.sh").write_text( - '#!/bin/bash\necho "1.0" > /logs/verifier/reward.txt\n', encoding="utf-8" - ) - return task_dir +REPO = Path(__file__).resolve().parents[3] -@pytest.fixture() -def single_task(tmp_path: Path) -> Path: - """A single standalone Harbor task directory.""" - return make_harbor_task( - tmp_path, - "cancel-async-tasks", - instruction="# Cancel Async Tasks\n\nCancel 5 asyncio tasks within 2 seconds.\n", +@pytest.fixture(scope="session") +def wheel(tmp_path_factory: pytest.TempPathFactory) -> Path: + """This checkout built as a wheel, for images that install the controller.""" + wheels = tmp_path_factory.mktemp("wheels") + subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(wheels)], + cwd=REPO, + check=True, + capture_output=True, ) - - -@pytest.fixture() -def dataset_same_env(tmp_path: Path) -> Path: - """A dataset directory with 3 tasks sharing the same Dockerfile.""" - dataset = tmp_path / "terminal-bench-sample" - dataset.mkdir() - for name in ("cancel-async-tasks", "build-pmars", "chess-best-move"): - make_harbor_task(dataset, name, instruction=f"# {name}\n\nSolve the {name} task.\n") - return dataset - - -@pytest.fixture() -def dataset_multi_env(tmp_path: Path) -> Path: - """A dataset directory with tasks split across 2 different Dockerfiles.""" - dataset = tmp_path / "Mixed Bench" - dataset.mkdir() - for name in ("cancel-async-tasks", "build-pmars"): - make_harbor_task(dataset, name, dockerfile=_SIMPLE_DOCKERFILE) - for name in ("caffe-cifar-10", "sam-cell-seg"): - make_harbor_task(dataset, name, task_toml=_ML_TASK_TOML, dockerfile=_ML_DOCKERFILE) - return dataset + return next(wheels.glob("*.whl")) diff --git a/tests/integrations/harbor/tasks/agent-answer-separate/environment/Dockerfile b/tests/integrations/harbor/tasks/agent-answer-separate/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer-separate/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/agent-answer-separate/instruction.md b/tests/integrations/harbor/tasks/agent-answer-separate/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer-separate/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/agent-answer-separate/solution/solve.sh b/tests/integrations/harbor/tasks/agent-answer-separate/solution/solve.sh new file mode 100644 index 000000000..420a14799 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer-separate/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo forty-two diff --git a/tests/integrations/harbor/tasks/agent-answer-separate/task.toml b/tests/integrations/harbor/tasks/agent-answer-separate/task.toml new file mode 100644 index 000000000..999372f26 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer-separate/task.toml @@ -0,0 +1,3 @@ +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/agent-answer-separate/tests/Dockerfile b/tests/integrations/harbor/tasks/agent-answer-separate/tests/Dockerfile new file mode 100644 index 000000000..3d6ba54af --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer-separate/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +COPY . /tests diff --git a/tests/integrations/harbor/tasks/agent-answer-separate/tests/test.sh b/tests/integrations/harbor/tasks/agent-answer-separate/tests/test.sh new file mode 100644 index 000000000..551bb4079 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer-separate/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p /logs/verifier +if [ "$(cat /logs/agent_answer.txt)" = "forty-two" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/agent-answer/environment/Dockerfile b/tests/integrations/harbor/tasks/agent-answer/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/agent-answer/expected.toml b/tests/integrations/harbor/tasks/agent-answer/expected.toml new file mode 100644 index 000000000..466d882db --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/expected.toml @@ -0,0 +1 @@ +# The oracle answers with the solution's stdout. diff --git a/tests/integrations/harbor/tasks/agent-answer/instruction.md b/tests/integrations/harbor/tasks/agent-answer/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/agent-answer/solution/solve.sh b/tests/integrations/harbor/tasks/agent-answer/solution/solve.sh new file mode 100644 index 000000000..420a14799 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo forty-two diff --git a/tests/integrations/harbor/tasks/agent-answer/task.toml b/tests/integrations/harbor/tasks/agent-answer/task.toml new file mode 100644 index 000000000..0db3dcc22 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/task.toml @@ -0,0 +1,2 @@ +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/agent-answer/tests/test.sh b/tests/integrations/harbor/tasks/agent-answer/tests/test.sh new file mode 100644 index 000000000..551bb4079 --- /dev/null +++ b/tests/integrations/harbor/tasks/agent-answer/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p /logs/verifier +if [ "$(cat /logs/agent_answer.txt)" = "forty-two" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/environment/Dockerfile b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/environment/docker-compose.yaml new file mode 100644 index 000000000..3829482e6 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/environment/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + image: ${SIDECAR_IMAGE:-python:3.11-alpine} + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] + expose: ["5678", "5679"] diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/expected.toml b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/expected.toml new file mode 100644 index 000000000..1b584d1f3 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/expected.toml @@ -0,0 +1,2 @@ +reward = 0.0 +error = "artifact /app/linked/tests has a symbolic link in its path" diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/instruction.md b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/solution/solve.sh new file mode 100644 index 000000000..a086c3b59 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +ln -s /verifier /app/linked diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/task.toml b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/task.toml new file mode 100644 index 000000000..4030594c3 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/task.toml @@ -0,0 +1,8 @@ +artifacts = [{ source = "/app/linked/tests", service = "main" }] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/tests/Dockerfile b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/tests/Dockerfile new file mode 100644 index 000000000..1db4095e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER verifier +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/artifact-ancestor-symlink/tests/test.sh b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/tests/test.sh new file mode 100644 index 000000000..6dd69d428 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-ancestor-symlink/tests/test.sh @@ -0,0 +1,32 @@ +# generated task metadata +#!/bin/bash +set -uo pipefail +mkdir -p /logs/verifier +mv /app/binary /app/binary.original +printf regenerated > /app/binary +mv /opt/result /opt/result.moved +env -i PATH=/usr/bin:/bin getent hosts pypi.org >/dev/null \ + || { echo "the public verifier could not resolve an external host"; exit 1; } + +if grep -q "Directory listing" /app/sidecar.html 2>/dev/null \ + && grep -q "Directory listing" /app/main.html 2>/dev/null \ + && [ -c /dev/null ] \ + && [ "$(cat /tmp/main.txt 2>/dev/null)" = "collected-from-main" ] \ + && [ "$(cat /tmp/sidecar.txt 2>/dev/null)" = "collected-from-sidecar" ] \ + && [ "$(cat /app/results/keep.txt 2>/dev/null)" = "keep" ] \ + && [ ! -e /app/results/drop.tmp ] \ + && [ "$(cat /app/binary.original 2>/dev/null)" = "original" ] \ + && [ "$(cat /app/binary 2>/dev/null)" = "regenerated" ] \ + && [ "$(cat /opt/result.moved/new.txt 2>/dev/null)" = "replacement" ] \ + && [ ! -e /opt/result.moved/base.txt ] \ + && [ "$(cat /root/agent-output.txt 2>/dev/null)" = "private" ] \ + && [ "$(id -u)" = "1001" ] \ + && [ "$(stat -c %u /home/verifier/owned)" = "1001" ] \ + && [ "$PWD" = "/home/verifier" ] \ + && [ "$HOME" = "/home/verifier/dotfiles" ] \ + && [ "$VERIFIER_PRECEDENCE" = "verifier-image" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo "the agent output or collected sidecar artifact is missing" + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/artifact-exclude/environment/Dockerfile b/tests/integrations/harbor/tasks/artifact-exclude/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/artifact-exclude/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/artifact-exclude/environment/docker-compose.yaml new file mode 100644 index 000000000..3829482e6 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/environment/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + image: ${SIDECAR_IMAGE:-python:3.11-alpine} + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] + expose: ["5678", "5679"] diff --git a/tests/integrations/harbor/tasks/artifact-exclude/instruction.md b/tests/integrations/harbor/tasks/artifact-exclude/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/artifact-exclude/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-exclude/solution/solve.sh new file mode 100644 index 000000000..faa5dddaf --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/solution/solve.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +mkdir -p /app/outputs/cache/nested /app/outputs/logs +echo keep > /app/outputs/keep.txt +echo junk > /app/outputs/junk.tmp +echo junk > /app/outputs/logs/nested.tmp +echo junk > /app/outputs/cache/nested/blob +ln -s /etc/passwd /app/outputs/cache/link +ln -s missing /app/outputs/logs/dangling.tmp diff --git a/tests/integrations/harbor/tasks/artifact-exclude/task.toml b/tests/integrations/harbor/tasks/artifact-exclude/task.toml new file mode 100644 index 000000000..ff6a6b891 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/task.toml @@ -0,0 +1,10 @@ +artifacts = [ + { source = "/app/outputs", destination = "results", exclude = ["*.tmp", "cache"] }, +] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/artifact-exclude/tests/Dockerfile b/tests/integrations/harbor/tasks/artifact-exclude/tests/Dockerfile new file mode 100644 index 000000000..1db4095e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER verifier +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/artifact-exclude/tests/test.sh b/tests/integrations/harbor/tasks/artifact-exclude/tests/test.sh new file mode 100644 index 000000000..334ce2ecc --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-exclude/tests/test.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -u +mkdir -p /logs/verifier +if [ "$(cat /app/outputs/keep.txt 2>/dev/null)" = "keep" ] \ + && [ -d /app/outputs/logs ] \ + && [ ! -e /app/outputs/junk.tmp ] \ + && [ ! -e /app/outputs/logs/nested.tmp ] \ + && [ ! -L /app/outputs/logs/dangling.tmp ] \ + && [ ! -e /app/outputs/cache ]; then + echo 1 > /logs/verifier/reward.txt +else + echo "excluded artifact entries leaked into the verifier" >&2 + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/environment/Dockerfile b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/environment/docker-compose.yaml new file mode 100644 index 000000000..3829482e6 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/environment/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + image: ${SIDECAR_IMAGE:-python:3.11-alpine} + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] + expose: ["5678", "5679"] diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/expected.toml b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/expected.toml new file mode 100644 index 000000000..7a97b24ec --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/expected.toml @@ -0,0 +1,2 @@ +reward = 0.0 +error = "artifact /link contains a symbolic link" diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/instruction.md b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/task.toml b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/task.toml new file mode 100644 index 000000000..106f24e9c --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/task.toml @@ -0,0 +1,13 @@ +artifacts = [{ source = "/link", service = "workspace", exclude = ["main.html"] }] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +service = "workspace" +command = "ln -sfn /app /link" +timeout_sec = 10 diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/tests/Dockerfile b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/tests/Dockerfile new file mode 100644 index 000000000..1db4095e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER verifier +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/tests/test.sh b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/tests/test.sh new file mode 100644 index 000000000..6dd69d428 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-sidecar-symlink-root/tests/test.sh @@ -0,0 +1,32 @@ +# generated task metadata +#!/bin/bash +set -uo pipefail +mkdir -p /logs/verifier +mv /app/binary /app/binary.original +printf regenerated > /app/binary +mv /opt/result /opt/result.moved +env -i PATH=/usr/bin:/bin getent hosts pypi.org >/dev/null \ + || { echo "the public verifier could not resolve an external host"; exit 1; } + +if grep -q "Directory listing" /app/sidecar.html 2>/dev/null \ + && grep -q "Directory listing" /app/main.html 2>/dev/null \ + && [ -c /dev/null ] \ + && [ "$(cat /tmp/main.txt 2>/dev/null)" = "collected-from-main" ] \ + && [ "$(cat /tmp/sidecar.txt 2>/dev/null)" = "collected-from-sidecar" ] \ + && [ "$(cat /app/results/keep.txt 2>/dev/null)" = "keep" ] \ + && [ ! -e /app/results/drop.tmp ] \ + && [ "$(cat /app/binary.original 2>/dev/null)" = "original" ] \ + && [ "$(cat /app/binary 2>/dev/null)" = "regenerated" ] \ + && [ "$(cat /opt/result.moved/new.txt 2>/dev/null)" = "replacement" ] \ + && [ ! -e /opt/result.moved/base.txt ] \ + && [ "$(cat /root/agent-output.txt 2>/dev/null)" = "private" ] \ + && [ "$(id -u)" = "1001" ] \ + && [ "$(stat -c %u /home/verifier/owned)" = "1001" ] \ + && [ "$PWD" = "/home/verifier" ] \ + && [ "$HOME" = "/home/verifier/dotfiles" ] \ + && [ "$VERIFIER_PRECEDENCE" = "verifier-image" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo "the agent output or collected sidecar artifact is missing" + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/artifact-symlink/environment/Dockerfile b/tests/integrations/harbor/tasks/artifact-symlink/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/artifact-symlink/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/artifact-symlink/environment/docker-compose.yaml new file mode 100644 index 000000000..3829482e6 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/environment/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + image: ${SIDECAR_IMAGE:-python:3.11-alpine} + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] + expose: ["5678", "5679"] diff --git a/tests/integrations/harbor/tasks/artifact-symlink/expected.toml b/tests/integrations/harbor/tasks/artifact-symlink/expected.toml new file mode 100644 index 000000000..7b6745e44 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/expected.toml @@ -0,0 +1,2 @@ +reward = 0.0 +error = "artifact /app/main.html is a symbolic link" diff --git a/tests/integrations/harbor/tasks/artifact-symlink/instruction.md b/tests/integrations/harbor/tasks/artifact-symlink/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/artifact-symlink/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-symlink/solution/solve.sh new file mode 100644 index 000000000..046ad2eb8 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/solution/solve.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -eu +[ -c /dev/null ] +printf discarded > /dev/null +curl -fsS --max-time 10 http://workspace:5678/ > /app/sidecar.html +curl -fsS --max-time 10 http://workspace:5679/ >/dev/null +mkdir -p /app/results +printf keep > /app/results/keep.txt +printf drop > /app/results/drop.tmp +printf original > /app/binary +mkdir -p /opt/result +printf replacement > /opt/result/new.txt +printf private > /root/agent-output.txt +case ",${NO_PROXY:-}," in + *,main,*) ;; + *) echo "main is absent from NO_PROXY" >&2; exit 1 ;; +esac +curl -fsS --max-time 10 http://main:8080/ > /app/main.html +protected=/runtime/session-"keys" +[ ! -e "$protected" ] +(while :; do sleep 1; done) >/tmp/actor-background.log 2>&1 & +printf '%s\n' "$!" > /app/actor.pid +entrypoint_visible=false +for pid in $(pgrep -x python3); do + if tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null \ + | grep -F "python3 -m http.server 8080 --directory /app" >/dev/null; then + entrypoint_visible=true + break + fi +done +if [ "$entrypoint_visible" = false ]; then + echo "the environment entrypoint process is absent from the agent process namespace" >&2 + exit 1 +fi +processes=$(ps -ef) +if printf '%s\n' "$processes" | grep -F "$protected"; then + echo "protected bridge path is visible in the process list" >&2 + exit 1 +fi +rm -f /app/main.html +ln -s /verifier/tests /app/main.html diff --git a/tests/integrations/harbor/tasks/artifact-symlink/task.toml b/tests/integrations/harbor/tasks/artifact-symlink/task.toml new file mode 100644 index 000000000..136c73399 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/task.toml @@ -0,0 +1,30 @@ +artifacts = [ + { source = "/app/sidecar.html", service = "main" }, + { source = "/app/main.html", service = "main" }, + { source = "/app/results", destination = "results", exclude = ["*.tmp"] }, + { source = "/app/binary", service = "main" }, + { source = "/opt/result", service = "main" }, + { source = "/root/agent-output.txt", service = "main" }, + { source = "/tmp/main.txt", service = "main" }, + { source = "/tmp/sidecar.txt", service = "workspace" }, +] + +[task] +name = "sidecar-reachability" + +[environment.env] +VERIFIER_PRECEDENCE = "actor" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +service = "main" +command = "test -s /app/actor.pid && ! kill -0 \"$(cat /app/actor.pid)\" 2>/dev/null && curl -fsS --max-time 10 http://127.0.0.1:8080/ >/dev/null && head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" +timeout_sec = 10 + +[[verifier.collect]] +service = "workspace" +command = "echo collected-from-sidecar > /tmp/sidecar.txt" +timeout_sec = 10 diff --git a/tests/integrations/harbor/tasks/artifact-symlink/tests/Dockerfile b/tests/integrations/harbor/tasks/artifact-symlink/tests/Dockerfile new file mode 100644 index 000000000..1db4095e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER verifier +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/artifact-symlink/tests/test.sh b/tests/integrations/harbor/tasks/artifact-symlink/tests/test.sh new file mode 100644 index 000000000..6dd69d428 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-symlink/tests/test.sh @@ -0,0 +1,32 @@ +# generated task metadata +#!/bin/bash +set -uo pipefail +mkdir -p /logs/verifier +mv /app/binary /app/binary.original +printf regenerated > /app/binary +mv /opt/result /opt/result.moved +env -i PATH=/usr/bin:/bin getent hosts pypi.org >/dev/null \ + || { echo "the public verifier could not resolve an external host"; exit 1; } + +if grep -q "Directory listing" /app/sidecar.html 2>/dev/null \ + && grep -q "Directory listing" /app/main.html 2>/dev/null \ + && [ -c /dev/null ] \ + && [ "$(cat /tmp/main.txt 2>/dev/null)" = "collected-from-main" ] \ + && [ "$(cat /tmp/sidecar.txt 2>/dev/null)" = "collected-from-sidecar" ] \ + && [ "$(cat /app/results/keep.txt 2>/dev/null)" = "keep" ] \ + && [ ! -e /app/results/drop.tmp ] \ + && [ "$(cat /app/binary.original 2>/dev/null)" = "original" ] \ + && [ "$(cat /app/binary 2>/dev/null)" = "regenerated" ] \ + && [ "$(cat /opt/result.moved/new.txt 2>/dev/null)" = "replacement" ] \ + && [ ! -e /opt/result.moved/base.txt ] \ + && [ "$(cat /root/agent-output.txt 2>/dev/null)" = "private" ] \ + && [ "$(id -u)" = "1001" ] \ + && [ "$(stat -c %u /home/verifier/owned)" = "1001" ] \ + && [ "$PWD" = "/home/verifier" ] \ + && [ "$HOME" = "/home/verifier/dotfiles" ] \ + && [ "$VERIFIER_PRECEDENCE" = "verifier-image" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo "the agent output or collected sidecar artifact is missing" + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/artifact-volume/environment/Dockerfile b/tests/integrations/harbor/tasks/artifact-volume/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/artifact-volume/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/artifact-volume/environment/docker-compose.yaml new file mode 100644 index 000000000..45a0a6a8a --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/environment/docker-compose.yaml @@ -0,0 +1,6 @@ +services: + main: + volumes: + - outputs:/app/outputs +volumes: + outputs: {} diff --git a/tests/integrations/harbor/tasks/artifact-volume/instruction.md b/tests/integrations/harbor/tasks/artifact-volume/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/artifact-volume/solution/solve.sh b/tests/integrations/harbor/tasks/artifact-volume/solution/solve.sh new file mode 100644 index 000000000..dcd2bf533 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +echo from-volume > /app/outputs/result.txt diff --git a/tests/integrations/harbor/tasks/artifact-volume/task.toml b/tests/integrations/harbor/tasks/artifact-volume/task.toml new file mode 100644 index 000000000..e0fe398d6 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/task.toml @@ -0,0 +1,8 @@ +artifacts = [{ source = "/app/outputs/result.txt", service = "main" }] + +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/artifact-volume/tests/Dockerfile b/tests/integrations/harbor/tasks/artifact-volume/tests/Dockerfile new file mode 100644 index 000000000..1db4095e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER verifier +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/artifact-volume/tests/test.sh b/tests/integrations/harbor/tasks/artifact-volume/tests/test.sh new file mode 100644 index 000000000..92d1a8b4d --- /dev/null +++ b/tests/integrations/harbor/tasks/artifact-volume/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p /logs/verifier +if [ "$(cat /app/outputs/result.txt 2>/dev/null)" = "from-volume" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/bad-agent-group/environment/Dockerfile b/tests/integrations/harbor/tasks/bad-agent-group/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-group/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/bad-agent-group/expected.toml b/tests/integrations/harbor/tasks/bad-agent-group/expected.toml new file mode 100644 index 000000000..f06e6c798 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-group/expected.toml @@ -0,0 +1,2 @@ +# The controller refuses to serve. +startup_error = "Harbor group 'ghosts' does not exist" diff --git a/tests/integrations/harbor/tasks/bad-agent-group/instruction.md b/tests/integrations/harbor/tasks/bad-agent-group/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-group/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/bad-agent-group/solution/solve.sh b/tests/integrations/harbor/tasks/bad-agent-group/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-group/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/bad-agent-group/task.toml b/tests/integrations/harbor/tasks/bad-agent-group/task.toml new file mode 100644 index 000000000..3aa079a2f --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-group/task.toml @@ -0,0 +1,5 @@ +[agent] +user = "root:ghosts" + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/bad-agent-group/tests/test.sh b/tests/integrations/harbor/tasks/bad-agent-group/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-group/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/bad-agent-user/environment/Dockerfile b/tests/integrations/harbor/tasks/bad-agent-user/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/bad-agent-user/expected.toml b/tests/integrations/harbor/tasks/bad-agent-user/expected.toml new file mode 100644 index 000000000..70635c316 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/expected.toml @@ -0,0 +1,2 @@ +# The controller refuses to serve. +startup_error = "Harbor user 'ghost' does not exist" diff --git a/tests/integrations/harbor/tasks/bad-agent-user/instruction.md b/tests/integrations/harbor/tasks/bad-agent-user/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/bad-agent-user/solution/solve.sh b/tests/integrations/harbor/tasks/bad-agent-user/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/bad-agent-user/task.toml b/tests/integrations/harbor/tasks/bad-agent-user/task.toml new file mode 100644 index 000000000..75c219938 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/task.toml @@ -0,0 +1,5 @@ +[agent] +user = "ghost" + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/bad-agent-user/tests/test.sh b/tests/integrations/harbor/tasks/bad-agent-user/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-agent-user/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/environment/Dockerfile b/tests/integrations/harbor/tasks/bad-verifier-user/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/expected.toml b/tests/integrations/harbor/tasks/bad-verifier-user/expected.toml new file mode 100644 index 000000000..e032647fc --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/expected.toml @@ -0,0 +1,2 @@ +reward = 0.0 +error = "Harbor user 'ghost' does not exist in this image" diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/instruction.md b/tests/integrations/harbor/tasks/bad-verifier-user/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/solution/solve.sh b/tests/integrations/harbor/tasks/bad-verifier-user/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/task.toml b/tests/integrations/harbor/tasks/bad-verifier-user/task.toml new file mode 100644 index 000000000..10327e812 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/task.toml @@ -0,0 +1,3 @@ +[verifier] +user = "ghost" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/bad-verifier-user/tests/test.sh b/tests/integrations/harbor/tasks/bad-verifier-user/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/bad-verifier-user/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/environment/Dockerfile b/tests/integrations/harbor/tasks/collect-hook-fails/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/expected.toml b/tests/integrations/harbor/tasks/collect-hook-fails/expected.toml new file mode 100644 index 000000000..717e9f1cd --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/expected.toml @@ -0,0 +1,2 @@ +reward = 0.0 +error = "collect hook on 'main' failed: broken" diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/instruction.md b/tests/integrations/harbor/tasks/collect-hook-fails/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/solution/solve.sh b/tests/integrations/harbor/tasks/collect-hook-fails/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/task.toml b/tests/integrations/harbor/tasks/collect-hook-fails/task.toml new file mode 100644 index 000000000..c5a6f0b42 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/task.toml @@ -0,0 +1,7 @@ +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +command = "echo broken >&2; exit 3" +timeout_sec = 1 diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/tests/Dockerfile b/tests/integrations/harbor/tasks/collect-hook-fails/tests/Dockerfile new file mode 100644 index 000000000..3d6ba54af --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +COPY . /tests diff --git a/tests/integrations/harbor/tasks/collect-hook-fails/tests/test.sh b/tests/integrations/harbor/tasks/collect-hook-fails/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-fails/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/collect-hook-times-out/environment/Dockerfile b/tests/integrations/harbor/tasks/collect-hook-times-out/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-times-out/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/collect-hook-times-out/expected.toml b/tests/integrations/harbor/tasks/collect-hook-times-out/expected.toml new file mode 100644 index 000000000..e31bffc6d --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-times-out/expected.toml @@ -0,0 +1,2 @@ +reward = 0.0 +error = "collect hook on 'main' timed out after 1s" diff --git a/tests/integrations/harbor/tasks/collect-hook-times-out/instruction.md b/tests/integrations/harbor/tasks/collect-hook-times-out/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-times-out/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/collect-hook-times-out/solution/solve.sh b/tests/integrations/harbor/tasks/collect-hook-times-out/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-times-out/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/collect-hook-times-out/task.toml b/tests/integrations/harbor/tasks/collect-hook-times-out/task.toml new file mode 100644 index 000000000..f450638a5 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-times-out/task.toml @@ -0,0 +1,7 @@ +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +command = "sleep 30" +timeout_sec = 1 diff --git a/tests/integrations/harbor/tasks/collect-hook-times-out/tests/Dockerfile b/tests/integrations/harbor/tasks/collect-hook-times-out/tests/Dockerfile new file mode 100644 index 000000000..3d6ba54af --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-times-out/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +COPY . /tests diff --git a/tests/integrations/harbor/tasks/collect-hook-times-out/tests/test.sh b/tests/integrations/harbor/tasks/collect-hook-times-out/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/collect-hook-times-out/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/environment/Dockerfile b/tests/integrations/harbor/tasks/entrypoint-exits/environment/Dockerfile new file mode 100644 index 000000000..e58726553 --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.11-slim +WORKDIR /workspace +ENTRYPOINT ["sh", "-c", "exit 7"] diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/expected.toml b/tests/integrations/harbor/tasks/entrypoint-exits/expected.toml new file mode 100644 index 000000000..a4fe759f4 --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/expected.toml @@ -0,0 +1,2 @@ +reward = 0.0 +error = "Harbor environment entrypoint exited with status 7" diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/instruction.md b/tests/integrations/harbor/tasks/entrypoint-exits/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/solution/solve.sh b/tests/integrations/harbor/tasks/entrypoint-exits/solution/solve.sh new file mode 100644 index 000000000..5cba02a37 --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sleep 1 diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/task.toml b/tests/integrations/harbor/tasks/entrypoint-exits/task.toml new file mode 100644 index 000000000..0db3dcc22 --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/task.toml @@ -0,0 +1,2 @@ +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/entrypoint-exits/tests/test.sh b/tests/integrations/harbor/tasks/entrypoint-exits/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/entrypoint-exits/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/env-template-missing/environment/Dockerfile b/tests/integrations/harbor/tasks/env-template-missing/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/env-template-missing/expected.toml b/tests/integrations/harbor/tasks/env-template-missing/expected.toml new file mode 100644 index 000000000..b0ed12eb8 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/expected.toml @@ -0,0 +1,2 @@ +# The controller refuses to serve. +startup_error = "HARBOR_MISSING_KEY" diff --git a/tests/integrations/harbor/tasks/env-template-missing/instruction.md b/tests/integrations/harbor/tasks/env-template-missing/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/env-template-missing/solution/solve.sh b/tests/integrations/harbor/tasks/env-template-missing/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/env-template-missing/task.toml b/tests/integrations/harbor/tasks/env-template-missing/task.toml new file mode 100644 index 000000000..97337976c --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/task.toml @@ -0,0 +1,5 @@ +[environment.env] +API_KEY = "${HARBOR_MISSING_KEY}" + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/env-template-missing/tests/test.sh b/tests/integrations/harbor/tasks/env-template-missing/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-template-missing/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/env-templates/environment/Dockerfile b/tests/integrations/harbor/tasks/env-templates/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/env-templates/expected.toml b/tests/integrations/harbor/tasks/env-templates/expected.toml new file mode 100644 index 000000000..3ed0f4e37 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/expected.toml @@ -0,0 +1,4 @@ +# An empty runtime value falls back to the template default. +[runtime.env_vars] +HARBOR_GREETING = "" +HARBOR_JUDGE_KEY = "judge-secret" diff --git a/tests/integrations/harbor/tasks/env-templates/instruction.md b/tests/integrations/harbor/tasks/env-templates/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/env-templates/solution/solve.sh b/tests/integrations/harbor/tasks/env-templates/solution/solve.sh new file mode 100644 index 000000000..f6695a2ad --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +[ "$JUDGE_KEY" = "judge-secret" ] && [ "$GREETING" = "hello" ] diff --git a/tests/integrations/harbor/tasks/env-templates/task.toml b/tests/integrations/harbor/tasks/env-templates/task.toml new file mode 100644 index 000000000..8c7fb1a83 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/task.toml @@ -0,0 +1,11 @@ +[environment.env] +GREETING = "${HARBOR_GREETING:-hello}" +JUDGE_KEY = "${HARBOR_JUDGE_KEY}" +EMBEDDED = "Bearer ${HARBOR_JUDGE_KEY}" + +[verifier] +timeout_sec = 120 + +[verifier.env] +VERIFIER_KEY = "${HARBOR_JUDGE_KEY}" +EMPTY_DEFAULT = "${HARBOR_UNSET:-}" diff --git a/tests/integrations/harbor/tasks/env-templates/tests/test.sh b/tests/integrations/harbor/tasks/env-templates/tests/test.sh new file mode 100644 index 000000000..6335fad86 --- /dev/null +++ b/tests/integrations/harbor/tasks/env-templates/tests/test.sh @@ -0,0 +1,8 @@ +#!/bin/bash +fail() { echo "unexpected $1"; echo "0.0" > /logs/verifier/reward.txt; exit 0; } +[ "$GREETING" = "hello" ] || fail "GREETING=$GREETING" +[ "$JUDGE_KEY" = "judge-secret" ] || fail "JUDGE_KEY=$JUDGE_KEY" +[ "$EMBEDDED" = 'Bearer ${HARBOR_JUDGE_KEY}' ] || fail "EMBEDDED=$EMBEDDED" +[ "$VERIFIER_KEY" = "judge-secret" ] || fail "VERIFIER_KEY=$VERIFIER_KEY" +[ "${EMPTY_DEFAULT-unset}" = "" ] || fail "EMPTY_DEFAULT=${EMPTY_DEFAULT-unset}" +echo "1.0" > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/environment/Dockerfile b/tests/integrations/harbor/tasks/fedora-bootstrap/environment/Dockerfile new file mode 100644 index 000000000..9f8bc3592 --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM fedora:42 +WORKDIR /app diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/instruction.md b/tests/integrations/harbor/tasks/fedora-bootstrap/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/solution/solve.sh b/tests/integrations/harbor/tasks/fedora-bootstrap/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/task.toml b/tests/integrations/harbor/tasks/fedora-bootstrap/task.toml new file mode 100644 index 000000000..a724393dc --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/task.toml @@ -0,0 +1,5 @@ +[task] +name = "fedora-bootstrap" + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/fedora-bootstrap/tests/test.sh b/tests/integrations/harbor/tasks/fedora-bootstrap/tests/test.sh new file mode 100644 index 000000000..3772af98f --- /dev/null +++ b/tests/integrations/harbor/tasks/fedora-bootstrap/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/environment/Dockerfile b/tests/integrations/harbor/tasks/healthcheck-fails/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/expected.toml b/tests/integrations/harbor/tasks/healthcheck-fails/expected.toml new file mode 100644 index 000000000..d1d63c417 --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/expected.toml @@ -0,0 +1,2 @@ +reward = 0.0 +error = "Harbor environment healthcheck failed after 3 attempts: not ready" diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/instruction.md b/tests/integrations/harbor/tasks/healthcheck-fails/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/solution/solve.sh b/tests/integrations/harbor/tasks/healthcheck-fails/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/task.toml b/tests/integrations/harbor/tasks/healthcheck-fails/task.toml new file mode 100644 index 000000000..1509363d2 --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/task.toml @@ -0,0 +1,8 @@ +[environment.healthcheck] +command = "echo not ready >&2; false" +interval_sec = 0.1 +timeout_sec = 5 +retries = 3 + +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/healthcheck-fails/tests/test.sh b/tests/integrations/harbor/tasks/healthcheck-fails/tests/test.sh new file mode 100644 index 000000000..86b198d44 --- /dev/null +++ b/tests/integrations/harbor/tasks/healthcheck-fails/tests/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/hello-mcp/expected.toml b/tests/integrations/harbor/tasks/hello-mcp/expected.toml new file mode 100644 index 000000000..e382031b7 --- /dev/null +++ b/tests/integrations/harbor/tasks/hello-mcp/expected.toml @@ -0,0 +1,2 @@ +# The host side opens the sidecar's MCP server through the control channel. +capability_probe = { name = "mcp-server", tools = ["get_secret"] } diff --git a/tests/integrations/harbor/tasks/hello-mcp/solution/solve.sh b/tests/integrations/harbor/tasks/hello-mcp/solution/solve.sh index 211690d9a..3e90f2cc8 100644 --- a/tests/integrations/harbor/tasks/hello-mcp/solution/solve.sh +++ b/tests/integrations/harbor/tasks/hello-mcp/solution/solve.sh @@ -1,3 +1,47 @@ #!/bin/sh set -eu -printf '%s' 'harbor-mcp-secret-12345' > /app/secret.txt +# Call the MCP server the way an agent in the workspace would. +python3 - <<'PY' +import json +from pathlib import Path +from urllib.request import Request, urlopen + +url = "http://mcp-server:8000/mcp" +session_id = None + + +def post(payload): + global session_id + headers = { + "Accept": "application/json, text/event-stream", + "Content-Type": "application/json", + } + if session_id is not None: + headers["Mcp-Session-Id"] = session_id + request = Request(url, data=json.dumps(payload).encode(), headers=headers) + with urlopen(request) as response: + body = response.read().decode() + session_id = response.headers.get("Mcp-Session-Id") or session_id + data = [line[6:] for line in body.splitlines() if line.startswith("data: ")] + return json.loads(data[-1] if data else body) if body else None + + +post({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "hud-test", "version": "1"}, + }, +}) +post({"jsonrpc": "2.0", "method": "notifications/initialized"}) +result = post({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "get_secret", "arguments": {}}, +}) +Path("/app/secret.txt").write_text(result["result"]["content"][0]["text"]) +PY diff --git a/tests/integrations/harbor/tasks/main-ports/environment/Dockerfile b/tests/integrations/harbor/tasks/main-ports/environment/Dockerfile new file mode 100644 index 000000000..802c34c04 --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/environment/Dockerfile @@ -0,0 +1,6 @@ +FROM debian:bookworm-slim +WORKDIR /app +COPY entrypoint.sh /usr/local/bin/start-environment +RUN chmod +x /usr/local/bin/start-environment && printf nested > /app/marker +EXPOSE 3128 3129 8765 8080 +ENTRYPOINT ["/usr/local/bin/start-environment"] diff --git a/tests/integrations/harbor/tasks/main-ports/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/main-ports/environment/docker-compose.yaml new file mode 100644 index 000000000..0dc19485f --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/environment/docker-compose.yaml @@ -0,0 +1,6 @@ +services: + main: {} + observer: + image: python:3.11-alpine + command: ["python", "-m", "http.server", "9000"] + expose: [9000] diff --git a/tests/integrations/harbor/tasks/main-ports/environment/entrypoint.sh b/tests/integrations/harbor/tasks/main-ports/environment/entrypoint.sh new file mode 100644 index 000000000..08bce4c9c --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/environment/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +python3 -m http.server 8765 --directory /app >/tmp/inner.log 2>&1 & +python3 -m http.server 8080 --directory /app >/tmp/app.log 2>&1 & +exec "$@" diff --git a/tests/integrations/harbor/tasks/main-ports/instruction.md b/tests/integrations/harbor/tasks/main-ports/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/main-ports/solution/solve.sh b/tests/integrations/harbor/tasks/main-ports/solution/solve.sh new file mode 100644 index 000000000..f388875fa --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +curl -fsS http://127.0.0.1:8765/marker > /app/result diff --git a/tests/integrations/harbor/tasks/main-ports/task.toml b/tests/integrations/harbor/tasks/main-ports/task.toml new file mode 100644 index 000000000..65d636cec --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/task.toml @@ -0,0 +1,18 @@ +[environment.healthcheck] +command = "curl -fsS http://127.0.0.1:8765/marker && curl -fsS http://127.0.0.1:8080/marker" +interval_sec = 0.1 +timeout_sec = 5 +retries = 50 + +[verifier] +timeout_sec = 30 + +[[verifier.collect]] +service = "observer" +command = ''' +python - <<'PY' +from urllib.request import urlopen +assert urlopen('http://main:8080/marker').read() == b'nested' +PY +''' +timeout_sec = 10 diff --git a/tests/integrations/harbor/tasks/main-ports/tests/test.sh b/tests/integrations/harbor/tasks/main-ports/tests/test.sh new file mode 100644 index 000000000..d3b77c54c --- /dev/null +++ b/tests/integrations/harbor/tasks/main-ports/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +test "$(cat /app/result)" = nested +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/managed-agents/environment/Dockerfile b/tests/integrations/harbor/tasks/managed-agents/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/managed-agents/expected.toml b/tests/integrations/harbor/tasks/managed-agents/expected.toml new file mode 100644 index 000000000..165db2ede --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/expected.toml @@ -0,0 +1,4 @@ +# A CLI bundle the runtime mounts at /usr/local/lib/agents runs in agent sessions. +[[runtime.bind_mounts]] +source = "managed/codex" +target = "/usr/local/lib/agents/codex" diff --git a/tests/integrations/harbor/tasks/managed-agents/instruction.md b/tests/integrations/harbor/tasks/managed-agents/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/managed-agents/managed/codex/bin/codex b/tests/integrations/harbor/tasks/managed-agents/managed/codex/bin/codex new file mode 100755 index 000000000..b793c8c55 --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/managed/codex/bin/codex @@ -0,0 +1,2 @@ +#!/bin/sh +echo managed-codex diff --git a/tests/integrations/harbor/tasks/managed-agents/solution/solve.sh b/tests/integrations/harbor/tasks/managed-agents/solution/solve.sh new file mode 100644 index 000000000..2afd7730b --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +/usr/local/lib/agents/codex/bin/codex > /workspace/result diff --git a/tests/integrations/harbor/tasks/managed-agents/task.toml b/tests/integrations/harbor/tasks/managed-agents/task.toml new file mode 100644 index 000000000..0db3dcc22 --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/task.toml @@ -0,0 +1,2 @@ +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/managed-agents/tests/test.sh b/tests/integrations/harbor/tasks/managed-agents/tests/test.sh new file mode 100644 index 000000000..ab9f0503a --- /dev/null +++ b/tests/integrations/harbor/tasks/managed-agents/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +test "$(cat /workspace/result)" = managed-codex +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/reward-files/environment/Dockerfile b/tests/integrations/harbor/tasks/reward-files/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/reward-files/expected.toml b/tests/integrations/harbor/tasks/reward-files/expected.toml new file mode 100644 index 000000000..b87709c2e --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/expected.toml @@ -0,0 +1,56 @@ +# reward.json wins over reward.txt; only a finite, non-boolean number counts. +[[case]] +name = "reward.json over reward.txt" +reward = 0.75 +runtime.env_vars = { REWARD_SHAPE = "json-preferred" } + +[[case]] +name = "a score key" +reward = 0.5 +runtime.env_vars = { REWARD_SHAPE = "json-score" } + +[[case]] +name = "a bare number" +reward = 0.25 +runtime.env_vars = { REWARD_SHAPE = "json-number" } + +[[case]] +name = "a boolean" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "json-bool" } + +[[case]] +name = "an infinite reward" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "json-infinite" } + +[[case]] +name = "invalid json" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "json-invalid" } + +[[case]] +name = "padded text" +reward = 0.5 +runtime.env_vars = { REWARD_SHAPE = "text" } + +[[case]] +name = "nan text" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "text-nan" } + +[[case]] +name = "words" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "text-words" } + +[[case]] +name = "no reward file" +reward = 0.0 +error = "Harbor verifier did not write a numeric reward" +runtime.env_vars = { REWARD_SHAPE = "none" } diff --git a/tests/integrations/harbor/tasks/reward-files/instruction.md b/tests/integrations/harbor/tasks/reward-files/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/reward-files/solution/solve.sh b/tests/integrations/harbor/tasks/reward-files/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/reward-files/task.toml b/tests/integrations/harbor/tasks/reward-files/task.toml new file mode 100644 index 000000000..bcd8bc462 --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/task.toml @@ -0,0 +1,5 @@ +[verifier] +timeout_sec = 30 + +[verifier.env] +SHAPE = "${REWARD_SHAPE}" diff --git a/tests/integrations/harbor/tasks/reward-files/tests/test.sh b/tests/integrations/harbor/tasks/reward-files/tests/test.sh new file mode 100644 index 000000000..5f5658c16 --- /dev/null +++ b/tests/integrations/harbor/tasks/reward-files/tests/test.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +mkdir -p /logs/verifier +cd /logs/verifier +case "$SHAPE" in + json-preferred) echo '{"reward": 0.75}' > reward.json; echo 0.1 > reward.txt ;; + json-score) echo '{"score": 0.5}' > reward.json ;; + json-number) echo '0.25' > reward.json ;; + json-bool) echo 'true' > reward.json; echo 1 > reward.txt ;; + json-infinite) echo '{"reward": Infinity}' > reward.json ;; + json-invalid) echo '{' > reward.json ;; + text) echo ' 0.5 ' > reward.txt ;; + text-nan) echo nan > reward.txt ;; + text-words) echo 'one point zero' > reward.txt ;; + none) ;; +esac diff --git a/tests/integrations/harbor/tasks/session-reliability/environment/Dockerfile b/tests/integrations/harbor/tasks/session-reliability/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/session-reliability/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/session-reliability/expected.toml b/tests/integrations/harbor/tasks/session-reliability/expected.toml new file mode 100644 index 000000000..db623d4f7 --- /dev/null +++ b/tests/integrations/harbor/tasks/session-reliability/expected.toml @@ -0,0 +1,2 @@ +# The solution runs in 40 separate SSH sessions. +repeat = 40 diff --git a/tests/integrations/harbor/tasks/session-reliability/instruction.md b/tests/integrations/harbor/tasks/session-reliability/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/session-reliability/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/session-reliability/solution/solve.sh b/tests/integrations/harbor/tasks/session-reliability/solution/solve.sh new file mode 100644 index 000000000..f34f230c0 --- /dev/null +++ b/tests/integrations/harbor/tasks/session-reliability/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +echo session >> /workspace/sessions diff --git a/tests/integrations/harbor/tasks/session-reliability/task.toml b/tests/integrations/harbor/tasks/session-reliability/task.toml new file mode 100644 index 000000000..0db3dcc22 --- /dev/null +++ b/tests/integrations/harbor/tasks/session-reliability/task.toml @@ -0,0 +1,2 @@ +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/session-reliability/tests/test.sh b/tests/integrations/harbor/tasks/session-reliability/tests/test.sh new file mode 100644 index 000000000..01336e8bd --- /dev/null +++ b/tests/integrations/harbor/tasks/session-reliability/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p /logs/verifier +if [ "$(wc -l < /workspace/sessions)" -eq 40 ]; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/environment/Dockerfile b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/docker-compose.yaml new file mode 100644 index 000000000..e6a19479e --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/docker-compose.yaml @@ -0,0 +1,10 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + build: ./workspace + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/environment/workspace/Dockerfile b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/workspace/Dockerfile new file mode 100644 index 000000000..2861eaa41 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/environment/workspace/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-alpine +EXPOSE 5678 5679 diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/instruction.md b/tests/integrations/harbor/tasks/sidecar-built-ports/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/solution/solve.sh b/tests/integrations/harbor/tasks/sidecar-built-ports/solution/solve.sh new file mode 100644 index 000000000..ae8788b18 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/solution/solve.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu +[ -c /dev/null ] +printf discarded > /dev/null +curl -fsS --max-time 10 http://workspace:5678/ > /app/sidecar.html +curl -fsS --max-time 10 http://workspace:5679/ >/dev/null +mkdir -p /app/results +printf keep > /app/results/keep.txt +printf drop > /app/results/drop.tmp +printf original > /app/binary +mkdir -p /opt/result +printf replacement > /opt/result/new.txt +printf private > /root/agent-output.txt +case ",${NO_PROXY:-}," in + *,main,*) ;; + *) echo "main is absent from NO_PROXY" >&2; exit 1 ;; +esac +curl -fsS --max-time 10 http://main:8080/ > /app/main.html +protected=/runtime/session-"keys" +[ ! -e "$protected" ] +(while :; do sleep 1; done) >/tmp/actor-background.log 2>&1 & +printf '%s\n' "$!" > /app/actor.pid +entrypoint_visible=false +for pid in $(pgrep -x python3); do + if tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null \ + | grep -F "python3 -m http.server 8080 --directory /app" >/dev/null; then + entrypoint_visible=true + break + fi +done +if [ "$entrypoint_visible" = false ]; then + echo "the environment entrypoint process is absent from the agent process namespace" >&2 + exit 1 +fi +processes=$(ps -ef) +if printf '%s\n' "$processes" | grep -F "$protected"; then + echo "protected bridge path is visible in the process list" >&2 + exit 1 +fi diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/task.toml b/tests/integrations/harbor/tasks/sidecar-built-ports/task.toml new file mode 100644 index 000000000..136c73399 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/task.toml @@ -0,0 +1,30 @@ +artifacts = [ + { source = "/app/sidecar.html", service = "main" }, + { source = "/app/main.html", service = "main" }, + { source = "/app/results", destination = "results", exclude = ["*.tmp"] }, + { source = "/app/binary", service = "main" }, + { source = "/opt/result", service = "main" }, + { source = "/root/agent-output.txt", service = "main" }, + { source = "/tmp/main.txt", service = "main" }, + { source = "/tmp/sidecar.txt", service = "workspace" }, +] + +[task] +name = "sidecar-reachability" + +[environment.env] +VERIFIER_PRECEDENCE = "actor" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +service = "main" +command = "test -s /app/actor.pid && ! kill -0 \"$(cat /app/actor.pid)\" 2>/dev/null && curl -fsS --max-time 10 http://127.0.0.1:8080/ >/dev/null && head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" +timeout_sec = 10 + +[[verifier.collect]] +service = "workspace" +command = "echo collected-from-sidecar > /tmp/sidecar.txt" +timeout_sec = 10 diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/tests/Dockerfile b/tests/integrations/harbor/tasks/sidecar-built-ports/tests/Dockerfile new file mode 100644 index 000000000..1db4095e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER verifier +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/sidecar-built-ports/tests/test.sh b/tests/integrations/harbor/tasks/sidecar-built-ports/tests/test.sh new file mode 100644 index 000000000..6dd69d428 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-built-ports/tests/test.sh @@ -0,0 +1,32 @@ +# generated task metadata +#!/bin/bash +set -uo pipefail +mkdir -p /logs/verifier +mv /app/binary /app/binary.original +printf regenerated > /app/binary +mv /opt/result /opt/result.moved +env -i PATH=/usr/bin:/bin getent hosts pypi.org >/dev/null \ + || { echo "the public verifier could not resolve an external host"; exit 1; } + +if grep -q "Directory listing" /app/sidecar.html 2>/dev/null \ + && grep -q "Directory listing" /app/main.html 2>/dev/null \ + && [ -c /dev/null ] \ + && [ "$(cat /tmp/main.txt 2>/dev/null)" = "collected-from-main" ] \ + && [ "$(cat /tmp/sidecar.txt 2>/dev/null)" = "collected-from-sidecar" ] \ + && [ "$(cat /app/results/keep.txt 2>/dev/null)" = "keep" ] \ + && [ ! -e /app/results/drop.tmp ] \ + && [ "$(cat /app/binary.original 2>/dev/null)" = "original" ] \ + && [ "$(cat /app/binary 2>/dev/null)" = "regenerated" ] \ + && [ "$(cat /opt/result.moved/new.txt 2>/dev/null)" = "replacement" ] \ + && [ ! -e /opt/result.moved/base.txt ] \ + && [ "$(cat /root/agent-output.txt 2>/dev/null)" = "private" ] \ + && [ "$(id -u)" = "1001" ] \ + && [ "$(stat -c %u /home/verifier/owned)" = "1001" ] \ + && [ "$PWD" = "/home/verifier" ] \ + && [ "$HOME" = "/home/verifier/dotfiles" ] \ + && [ "$VERIFIER_PRECEDENCE" = "verifier-image" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo "the agent output or collected sidecar artifact is missing" + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-placed/environment/Dockerfile b/tests/integrations/harbor/tasks/sidecar-reachability-placed/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-placed/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-placed/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/sidecar-reachability-placed/environment/docker-compose.yaml new file mode 100644 index 000000000..3829482e6 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-placed/environment/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + image: ${SIDECAR_IMAGE:-python:3.11-alpine} + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] + expose: ["5678", "5679"] diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-placed/instruction.md b/tests/integrations/harbor/tasks/sidecar-reachability-placed/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-placed/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-placed/solution/solve.sh b/tests/integrations/harbor/tasks/sidecar-reachability-placed/solution/solve.sh new file mode 100644 index 000000000..ae8788b18 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-placed/solution/solve.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu +[ -c /dev/null ] +printf discarded > /dev/null +curl -fsS --max-time 10 http://workspace:5678/ > /app/sidecar.html +curl -fsS --max-time 10 http://workspace:5679/ >/dev/null +mkdir -p /app/results +printf keep > /app/results/keep.txt +printf drop > /app/results/drop.tmp +printf original > /app/binary +mkdir -p /opt/result +printf replacement > /opt/result/new.txt +printf private > /root/agent-output.txt +case ",${NO_PROXY:-}," in + *,main,*) ;; + *) echo "main is absent from NO_PROXY" >&2; exit 1 ;; +esac +curl -fsS --max-time 10 http://main:8080/ > /app/main.html +protected=/runtime/session-"keys" +[ ! -e "$protected" ] +(while :; do sleep 1; done) >/tmp/actor-background.log 2>&1 & +printf '%s\n' "$!" > /app/actor.pid +entrypoint_visible=false +for pid in $(pgrep -x python3); do + if tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null \ + | grep -F "python3 -m http.server 8080 --directory /app" >/dev/null; then + entrypoint_visible=true + break + fi +done +if [ "$entrypoint_visible" = false ]; then + echo "the environment entrypoint process is absent from the agent process namespace" >&2 + exit 1 +fi +processes=$(ps -ef) +if printf '%s\n' "$processes" | grep -F "$protected"; then + echo "protected bridge path is visible in the process list" >&2 + exit 1 +fi diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-placed/task.toml b/tests/integrations/harbor/tasks/sidecar-reachability-placed/task.toml new file mode 100644 index 000000000..312671da8 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-placed/task.toml @@ -0,0 +1,33 @@ +artifacts = [ + { source = "/app/sidecar.html", service = "main" }, + { source = "/app/main.html", service = "main" }, + { source = "/app/results", destination = "results", exclude = ["*.tmp"] }, + { source = "/app/binary", service = "main" }, + { source = "/opt/result", service = "main" }, + { source = "/root/agent-output.txt", service = "main" }, + { source = "/tmp/main.txt", service = "main" }, + { source = "/tmp/sidecar.txt", service = "workspace" }, +] + +[task] +name = "sidecar-reachability" + +[environment.env] +VERIFIER_PRECEDENCE = "actor" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +service = "main" +command = "test -s /app/actor.pid && ! kill -0 \"$(cat /app/actor.pid)\" 2>/dev/null && curl -fsS --max-time 10 http://127.0.0.1:8080/ >/dev/null && head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" +timeout_sec = 10 + +[[verifier.collect]] +service = "workspace" +command = "echo collected-from-sidecar > /tmp/sidecar.txt" +timeout_sec = 10 + +[verifier.environment] +cpus = 1 diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-placed/tests/Dockerfile b/tests/integrations/harbor/tasks/sidecar-reachability-placed/tests/Dockerfile new file mode 100644 index 000000000..1db4095e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-placed/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER verifier +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-placed/tests/test.sh b/tests/integrations/harbor/tasks/sidecar-reachability-placed/tests/test.sh new file mode 100644 index 000000000..6dd69d428 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-placed/tests/test.sh @@ -0,0 +1,32 @@ +# generated task metadata +#!/bin/bash +set -uo pipefail +mkdir -p /logs/verifier +mv /app/binary /app/binary.original +printf regenerated > /app/binary +mv /opt/result /opt/result.moved +env -i PATH=/usr/bin:/bin getent hosts pypi.org >/dev/null \ + || { echo "the public verifier could not resolve an external host"; exit 1; } + +if grep -q "Directory listing" /app/sidecar.html 2>/dev/null \ + && grep -q "Directory listing" /app/main.html 2>/dev/null \ + && [ -c /dev/null ] \ + && [ "$(cat /tmp/main.txt 2>/dev/null)" = "collected-from-main" ] \ + && [ "$(cat /tmp/sidecar.txt 2>/dev/null)" = "collected-from-sidecar" ] \ + && [ "$(cat /app/results/keep.txt 2>/dev/null)" = "keep" ] \ + && [ ! -e /app/results/drop.tmp ] \ + && [ "$(cat /app/binary.original 2>/dev/null)" = "original" ] \ + && [ "$(cat /app/binary 2>/dev/null)" = "regenerated" ] \ + && [ "$(cat /opt/result.moved/new.txt 2>/dev/null)" = "replacement" ] \ + && [ ! -e /opt/result.moved/base.txt ] \ + && [ "$(cat /root/agent-output.txt 2>/dev/null)" = "private" ] \ + && [ "$(id -u)" = "1001" ] \ + && [ "$(stat -c %u /home/verifier/owned)" = "1001" ] \ + && [ "$PWD" = "/home/verifier" ] \ + && [ "$HOME" = "/home/verifier/dotfiles" ] \ + && [ "$VERIFIER_PRECEDENCE" = "verifier-image" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo "the agent output or collected sidecar artifact is missing" + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-repeated/environment/Dockerfile b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-repeated/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/environment/docker-compose.yaml new file mode 100644 index 000000000..3829482e6 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/environment/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + image: ${SIDECAR_IMAGE:-python:3.11-alpine} + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] + expose: ["5678", "5679"] diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-repeated/expected.toml b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/expected.toml new file mode 100644 index 000000000..edf0f5b76 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/expected.toml @@ -0,0 +1,3 @@ +# Two rollouts share one placement; the second sees a restored verifier root. +group = 2 +shared_width = 1 diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-repeated/instruction.md b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-repeated/solution/solve.sh b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/solution/solve.sh new file mode 100644 index 000000000..ae8788b18 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/solution/solve.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu +[ -c /dev/null ] +printf discarded > /dev/null +curl -fsS --max-time 10 http://workspace:5678/ > /app/sidecar.html +curl -fsS --max-time 10 http://workspace:5679/ >/dev/null +mkdir -p /app/results +printf keep > /app/results/keep.txt +printf drop > /app/results/drop.tmp +printf original > /app/binary +mkdir -p /opt/result +printf replacement > /opt/result/new.txt +printf private > /root/agent-output.txt +case ",${NO_PROXY:-}," in + *,main,*) ;; + *) echo "main is absent from NO_PROXY" >&2; exit 1 ;; +esac +curl -fsS --max-time 10 http://main:8080/ > /app/main.html +protected=/runtime/session-"keys" +[ ! -e "$protected" ] +(while :; do sleep 1; done) >/tmp/actor-background.log 2>&1 & +printf '%s\n' "$!" > /app/actor.pid +entrypoint_visible=false +for pid in $(pgrep -x python3); do + if tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null \ + | grep -F "python3 -m http.server 8080 --directory /app" >/dev/null; then + entrypoint_visible=true + break + fi +done +if [ "$entrypoint_visible" = false ]; then + echo "the environment entrypoint process is absent from the agent process namespace" >&2 + exit 1 +fi +processes=$(ps -ef) +if printf '%s\n' "$processes" | grep -F "$protected"; then + echo "protected bridge path is visible in the process list" >&2 + exit 1 +fi diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-repeated/task.toml b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/task.toml new file mode 100644 index 000000000..136c73399 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/task.toml @@ -0,0 +1,30 @@ +artifacts = [ + { source = "/app/sidecar.html", service = "main" }, + { source = "/app/main.html", service = "main" }, + { source = "/app/results", destination = "results", exclude = ["*.tmp"] }, + { source = "/app/binary", service = "main" }, + { source = "/opt/result", service = "main" }, + { source = "/root/agent-output.txt", service = "main" }, + { source = "/tmp/main.txt", service = "main" }, + { source = "/tmp/sidecar.txt", service = "workspace" }, +] + +[task] +name = "sidecar-reachability" + +[environment.env] +VERIFIER_PRECEDENCE = "actor" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +service = "main" +command = "test -s /app/actor.pid && ! kill -0 \"$(cat /app/actor.pid)\" 2>/dev/null && curl -fsS --max-time 10 http://127.0.0.1:8080/ >/dev/null && head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" +timeout_sec = 10 + +[[verifier.collect]] +service = "workspace" +command = "echo collected-from-sidecar > /tmp/sidecar.txt" +timeout_sec = 10 diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-repeated/tests/Dockerfile b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/tests/Dockerfile new file mode 100644 index 000000000..1db4095e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER verifier +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/sidecar-reachability-repeated/tests/test.sh b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/tests/test.sh new file mode 100644 index 000000000..6dd69d428 --- /dev/null +++ b/tests/integrations/harbor/tasks/sidecar-reachability-repeated/tests/test.sh @@ -0,0 +1,32 @@ +# generated task metadata +#!/bin/bash +set -uo pipefail +mkdir -p /logs/verifier +mv /app/binary /app/binary.original +printf regenerated > /app/binary +mv /opt/result /opt/result.moved +env -i PATH=/usr/bin:/bin getent hosts pypi.org >/dev/null \ + || { echo "the public verifier could not resolve an external host"; exit 1; } + +if grep -q "Directory listing" /app/sidecar.html 2>/dev/null \ + && grep -q "Directory listing" /app/main.html 2>/dev/null \ + && [ -c /dev/null ] \ + && [ "$(cat /tmp/main.txt 2>/dev/null)" = "collected-from-main" ] \ + && [ "$(cat /tmp/sidecar.txt 2>/dev/null)" = "collected-from-sidecar" ] \ + && [ "$(cat /app/results/keep.txt 2>/dev/null)" = "keep" ] \ + && [ ! -e /app/results/drop.tmp ] \ + && [ "$(cat /app/binary.original 2>/dev/null)" = "original" ] \ + && [ "$(cat /app/binary 2>/dev/null)" = "regenerated" ] \ + && [ "$(cat /opt/result.moved/new.txt 2>/dev/null)" = "replacement" ] \ + && [ ! -e /opt/result.moved/base.txt ] \ + && [ "$(cat /root/agent-output.txt 2>/dev/null)" = "private" ] \ + && [ "$(id -u)" = "1001" ] \ + && [ "$(stat -c %u /home/verifier/owned)" = "1001" ] \ + && [ "$PWD" = "/home/verifier" ] \ + && [ "$HOME" = "/home/verifier/dotfiles" ] \ + && [ "$VERIFIER_PRECEDENCE" = "verifier-image" ]; then + echo 1 > /logs/verifier/reward.txt +else + echo "the agent output or collected sidecar artifact is missing" + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/vendor-uv/environment/Dockerfile b/tests/integrations/harbor/tasks/vendor-uv/environment/Dockerfile new file mode 100644 index 000000000..d68e1c5bf --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.11-slim +WORKDIR /workspace +COPY pyproject.toml . diff --git a/tests/integrations/harbor/tasks/vendor-uv/environment/pyproject.toml b/tests/integrations/harbor/tasks/vendor-uv/environment/pyproject.toml new file mode 100644 index 000000000..cb4e94f57 --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/environment/pyproject.toml @@ -0,0 +1,4 @@ +[tool.uv] +required-version = "==0.0.0" +[tool.uv.pip] +no-index = true diff --git a/tests/integrations/harbor/tasks/vendor-uv/instruction.md b/tests/integrations/harbor/tasks/vendor-uv/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/vendor-uv/solution/solve.sh b/tests/integrations/harbor/tasks/vendor-uv/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/vendor-uv/task.toml b/tests/integrations/harbor/tasks/vendor-uv/task.toml new file mode 100644 index 000000000..0db3dcc22 --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/task.toml @@ -0,0 +1,2 @@ +[verifier] +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/vendor-uv/tests/test.sh b/tests/integrations/harbor/tasks/vendor-uv/tests/test.sh new file mode 100644 index 000000000..be1bb4062 --- /dev/null +++ b/tests/integrations/harbor/tasks/vendor-uv/tests/test.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p /logs/verifier +if grep -q 'required-version = "==0.0.0"' /workspace/pyproject.toml; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/verifier-child-user/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-child-user/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/verifier-child-user/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/verifier-child-user/environment/docker-compose.yaml new file mode 100644 index 000000000..3829482e6 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/environment/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + image: ${SIDECAR_IMAGE:-python:3.11-alpine} + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] + expose: ["5678", "5679"] diff --git a/tests/integrations/harbor/tasks/verifier-child-user/instruction.md b/tests/integrations/harbor/tasks/verifier-child-user/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/verifier-child-user/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-child-user/solution/solve.sh new file mode 100644 index 000000000..ae8788b18 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/solution/solve.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu +[ -c /dev/null ] +printf discarded > /dev/null +curl -fsS --max-time 10 http://workspace:5678/ > /app/sidecar.html +curl -fsS --max-time 10 http://workspace:5679/ >/dev/null +mkdir -p /app/results +printf keep > /app/results/keep.txt +printf drop > /app/results/drop.tmp +printf original > /app/binary +mkdir -p /opt/result +printf replacement > /opt/result/new.txt +printf private > /root/agent-output.txt +case ",${NO_PROXY:-}," in + *,main,*) ;; + *) echo "main is absent from NO_PROXY" >&2; exit 1 ;; +esac +curl -fsS --max-time 10 http://main:8080/ > /app/main.html +protected=/runtime/session-"keys" +[ ! -e "$protected" ] +(while :; do sleep 1; done) >/tmp/actor-background.log 2>&1 & +printf '%s\n' "$!" > /app/actor.pid +entrypoint_visible=false +for pid in $(pgrep -x python3); do + if tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null \ + | grep -F "python3 -m http.server 8080 --directory /app" >/dev/null; then + entrypoint_visible=true + break + fi +done +if [ "$entrypoint_visible" = false ]; then + echo "the environment entrypoint process is absent from the agent process namespace" >&2 + exit 1 +fi +processes=$(ps -ef) +if printf '%s\n' "$processes" | grep -F "$protected"; then + echo "protected bridge path is visible in the process list" >&2 + exit 1 +fi diff --git a/tests/integrations/harbor/tasks/verifier-child-user/task.toml b/tests/integrations/harbor/tasks/verifier-child-user/task.toml new file mode 100644 index 000000000..136c73399 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/task.toml @@ -0,0 +1,30 @@ +artifacts = [ + { source = "/app/sidecar.html", service = "main" }, + { source = "/app/main.html", service = "main" }, + { source = "/app/results", destination = "results", exclude = ["*.tmp"] }, + { source = "/app/binary", service = "main" }, + { source = "/opt/result", service = "main" }, + { source = "/root/agent-output.txt", service = "main" }, + { source = "/tmp/main.txt", service = "main" }, + { source = "/tmp/sidecar.txt", service = "workspace" }, +] + +[task] +name = "sidecar-reachability" + +[environment.env] +VERIFIER_PRECEDENCE = "actor" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[[verifier.collect]] +service = "main" +command = "test -s /app/actor.pid && ! kill -0 \"$(cat /app/actor.pid)\" 2>/dev/null && curl -fsS --max-time 10 http://127.0.0.1:8080/ >/dev/null && head -c 1 /dev/urandom >/dev/null && echo collected-from-main > /tmp/main.txt && sleep 0.2" +timeout_sec = 10 + +[[verifier.collect]] +service = "workspace" +command = "echo collected-from-sidecar > /tmp/sidecar.txt" +timeout_sec = 10 diff --git a/tests/integrations/harbor/tasks/verifier-child-user/tests/Dockerfile b/tests/integrations/harbor/tasks/verifier-child-user/tests/Dockerfile new file mode 100644 index 000000000..6108465e3 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER root +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/verifier-child-user/tests/test.sh b/tests/integrations/harbor/tasks/verifier-child-user/tests/test.sh new file mode 100644 index 000000000..dbec9e336 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-child-user/tests/test.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +mkdir -p /logs/verifier +if su verifier -s /bin/sh -c 'test "$(cat /root/agent-output.txt)" = private'; then + echo 1 > /logs/verifier/reward.txt +else + echo 0 > /logs/verifier/reward.txt +fi diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-separate/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-separate/expected.toml b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/expected.toml new file mode 100644 index 000000000..b647aab26 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/expected.toml @@ -0,0 +1 @@ +# The default stands in for a credential the runtime does not provide. diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-separate/instruction.md b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-separate/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/solution/solve.sh new file mode 100644 index 000000000..8a52a7e29 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +[ "${JUDGE_KEY-unset}" = unset ] diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-separate/task.toml b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/task.toml new file mode 100644 index 000000000..598db88d1 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/task.toml @@ -0,0 +1,6 @@ +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[verifier.env] +JUDGE_KEY = "${HARBOR_VERIFIER_KEY:-judge-value}" diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-separate/tests/Dockerfile b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/tests/Dockerfile new file mode 100644 index 000000000..3d6ba54af --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +COPY . /tests diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-separate/tests/test.sh b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/tests/test.sh new file mode 100644 index 000000000..706bd41ea --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-separate/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +[ "$JUDGE_KEY" = "judge-value" ] +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-shared/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-shared/expected.toml b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/expected.toml new file mode 100644 index 000000000..b647aab26 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/expected.toml @@ -0,0 +1 @@ +# The default stands in for a credential the runtime does not provide. diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-shared/instruction.md b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-shared/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/solution/solve.sh new file mode 100644 index 000000000..8a52a7e29 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +[ "${JUDGE_KEY-unset}" = unset ] diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-shared/task.toml b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/task.toml new file mode 100644 index 000000000..d5875a79b --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/task.toml @@ -0,0 +1,6 @@ +[verifier] +environment_mode = "shared" +timeout_sec = 30 + +[verifier.env] +JUDGE_KEY = "${HARBOR_VERIFIER_KEY:-judge-value}" diff --git a/tests/integrations/harbor/tasks/verifier-credentials-default-shared/tests/test.sh b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/tests/test.sh new file mode 100644 index 000000000..706bd41ea --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-default-shared/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +[ "$JUDGE_KEY" = "judge-value" ] +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-credentials-separate/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/expected.toml b/tests/integrations/harbor/tasks/verifier-credentials-separate/expected.toml new file mode 100644 index 000000000..77d633667 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/expected.toml @@ -0,0 +1,9 @@ +# A verifier credential is resolved only when grading needs it. +[[case]] +name = "missing" +reward = 0.0 +error = "HARBOR_VERIFIER_KEY" + +[[case]] +name = "provided" +runtime.env_vars = { HARBOR_VERIFIER_KEY = "judge-value" } diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/instruction.md b/tests/integrations/harbor/tasks/verifier-credentials-separate/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-credentials-separate/solution/solve.sh new file mode 100644 index 000000000..8a52a7e29 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +[ "${JUDGE_KEY-unset}" = unset ] diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/task.toml b/tests/integrations/harbor/tasks/verifier-credentials-separate/task.toml new file mode 100644 index 000000000..b5d08ed36 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/task.toml @@ -0,0 +1,6 @@ +[verifier] +environment_mode = "separate" +timeout_sec = 30 + +[verifier.env] +JUDGE_KEY = "${HARBOR_VERIFIER_KEY}" diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/tests/Dockerfile b/tests/integrations/harbor/tasks/verifier-credentials-separate/tests/Dockerfile new file mode 100644 index 000000000..3d6ba54af --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/tests/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +COPY . /tests diff --git a/tests/integrations/harbor/tasks/verifier-credentials-separate/tests/test.sh b/tests/integrations/harbor/tasks/verifier-credentials-separate/tests/test.sh new file mode 100644 index 000000000..706bd41ea --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-separate/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +[ "$JUDGE_KEY" = "judge-value" ] +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-credentials-shared/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/expected.toml b/tests/integrations/harbor/tasks/verifier-credentials-shared/expected.toml new file mode 100644 index 000000000..77d633667 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/expected.toml @@ -0,0 +1,9 @@ +# A verifier credential is resolved only when grading needs it. +[[case]] +name = "missing" +reward = 0.0 +error = "HARBOR_VERIFIER_KEY" + +[[case]] +name = "provided" +runtime.env_vars = { HARBOR_VERIFIER_KEY = "judge-value" } diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/instruction.md b/tests/integrations/harbor/tasks/verifier-credentials-shared/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-credentials-shared/solution/solve.sh new file mode 100644 index 000000000..8a52a7e29 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +[ "${JUDGE_KEY-unset}" = unset ] diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/task.toml b/tests/integrations/harbor/tasks/verifier-credentials-shared/task.toml new file mode 100644 index 000000000..4ce85c14e --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/task.toml @@ -0,0 +1,6 @@ +[verifier] +environment_mode = "shared" +timeout_sec = 30 + +[verifier.env] +JUDGE_KEY = "${HARBOR_VERIFIER_KEY}" diff --git a/tests/integrations/harbor/tasks/verifier-credentials-shared/tests/test.sh b/tests/integrations/harbor/tasks/verifier-credentials-shared/tests/test.sh new file mode 100644 index 000000000..706bd41ea --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-credentials-shared/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +[ "$JUDGE_KEY" = "judge-value" ] +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/verifier-restore/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-restore/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/verifier-restore/expected.toml b/tests/integrations/harbor/tasks/verifier-restore/expected.toml new file mode 100644 index 000000000..79ff50716 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/expected.toml @@ -0,0 +1,2 @@ +group = 2 +shared_width = 1 diff --git a/tests/integrations/harbor/tasks/verifier-restore/instruction.md b/tests/integrations/harbor/tasks/verifier-restore/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/verifier-restore/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-restore/solution/solve.sh new file mode 100644 index 000000000..75eebb66d --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/solution/solve.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +mkdir -p /opt/result +echo agent > /opt/result/agent.txt diff --git a/tests/integrations/harbor/tasks/verifier-restore/task.toml b/tests/integrations/harbor/tasks/verifier-restore/task.toml new file mode 100644 index 000000000..c5b491e33 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/task.toml @@ -0,0 +1,5 @@ +artifacts = [{ source = "/opt/result", service = "main" }] + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/verifier-restore/tests/Dockerfile b/tests/integrations/harbor/tasks/verifier-restore/tests/Dockerfile new file mode 100644 index 000000000..1a8b84d50 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/tests/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-alpine +RUN mkdir -p /opt/result && echo base > /opt/result/base.txt +COPY . /tests diff --git a/tests/integrations/harbor/tasks/verifier-restore/tests/test.sh b/tests/integrations/harbor/tasks/verifier-restore/tests/test.sh new file mode 100644 index 000000000..d4e1de262 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-restore/tests/test.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -u +mkdir -p /logs/verifier +fail() { echo "$1"; echo 0 > /logs/verifier/reward.txt; exit 0; } +[ ! -e /opt/stray.txt ] || fail "a previous verifier's file survived" +[ ! -e /opt/result.moved ] || fail "a previous verifier's move survived" +[ "$(ls /opt/result)" = "agent.txt" ] || fail "the artifact did not replace the image path" +echo stray > /opt/stray.txt +mv /opt/result /opt/result.moved +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/tasks/verifier-shebang/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-shebang/environment/Dockerfile new file mode 100644 index 000000000..fc3e06726 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-alpine +RUN apk add --no-cache curl +WORKDIR /app +EXPOSE 8080 +ENTRYPOINT ["sh", "-c", "python3 -m http.server 8080 --directory /app >/tmp/main-http.log 2>&1 & exec \"$@\"", "--"] diff --git a/tests/integrations/harbor/tasks/verifier-shebang/environment/docker-compose.yaml b/tests/integrations/harbor/tasks/verifier-shebang/environment/docker-compose.yaml new file mode 100644 index 000000000..3829482e6 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/environment/docker-compose.yaml @@ -0,0 +1,11 @@ +services: + main: + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8080"] + interval: 100ms + timeout: 2s + retries: 50 + workspace: + image: ${SIDECAR_IMAGE:-python:3.11-alpine} + command: ["sh", "-c", "python -m http.server 5678 & exec python -m http.server 5679"] + expose: ["5678", "5679"] diff --git a/tests/integrations/harbor/tasks/verifier-shebang/instruction.md b/tests/integrations/harbor/tasks/verifier-shebang/instruction.md new file mode 100644 index 000000000..a27a85c6e --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/instruction.md @@ -0,0 +1 @@ +Fetch the web service at http://web:5678 and save its response to /app/sidecar.html. diff --git a/tests/integrations/harbor/tasks/verifier-shebang/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-shebang/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/verifier-shebang/task.toml b/tests/integrations/harbor/tasks/verifier-shebang/task.toml new file mode 100644 index 000000000..1574ed7b9 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/task.toml @@ -0,0 +1,6 @@ +[task] +name = "sidecar-reachability" + +[verifier] +environment_mode = "separate" +timeout_sec = 30 diff --git a/tests/integrations/harbor/tasks/verifier-shebang/tests/Dockerfile b/tests/integrations/harbor/tasks/verifier-shebang/tests/Dockerfile new file mode 100644 index 000000000..1db4095e4 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-alpine + +ENV VERIFIER_PRECEDENCE=verifier-image HOME=/home/verifier/dotfiles +COPY . /tests +RUN apk add --no-cache bash \ + && addgroup -g 1001 verifier \ + && adduser -D -u 1001 -G verifier verifier \ + && mkdir -p /app /logs/verifier /opt/result \ + && echo base > /opt/result/base.txt \ + && touch /home/verifier/owned \ + && chown verifier:verifier /home/verifier/owned + +WORKDIR /home/verifier +USER verifier +ENTRYPOINT [] +CMD ["sleep", "infinity"] diff --git a/tests/integrations/harbor/tasks/verifier-shebang/tests/test.sh b/tests/integrations/harbor/tasks/verifier-shebang/tests/test.sh new file mode 100644 index 000000000..8d661956a --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-shebang/tests/test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +from pathlib import Path + +Path("/logs/verifier/reward.txt").write_text("1") diff --git a/tests/integrations/harbor/tasks/verifier-timeout/environment/Dockerfile b/tests/integrations/harbor/tasks/verifier-timeout/environment/Dockerfile new file mode 100644 index 000000000..e8f54bd51 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.11-slim +WORKDIR /workspace diff --git a/tests/integrations/harbor/tasks/verifier-timeout/expected.toml b/tests/integrations/harbor/tasks/verifier-timeout/expected.toml new file mode 100644 index 000000000..67e1681a1 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/expected.toml @@ -0,0 +1,2 @@ +reward = 0.0 +error = "Harbor verifier timed out after 2s" diff --git a/tests/integrations/harbor/tasks/verifier-timeout/instruction.md b/tests/integrations/harbor/tasks/verifier-timeout/instruction.md new file mode 100644 index 000000000..ad4fc2382 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/instruction.md @@ -0,0 +1 @@ +Solve the task. diff --git a/tests/integrations/harbor/tasks/verifier-timeout/solution/solve.sh b/tests/integrations/harbor/tasks/verifier-timeout/solution/solve.sh new file mode 100644 index 000000000..af3a4e4be --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/sh +true diff --git a/tests/integrations/harbor/tasks/verifier-timeout/task.toml b/tests/integrations/harbor/tasks/verifier-timeout/task.toml new file mode 100644 index 000000000..72939071e --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/task.toml @@ -0,0 +1,2 @@ +[verifier] +timeout_sec = 2 diff --git a/tests/integrations/harbor/tasks/verifier-timeout/tests/test.sh b/tests/integrations/harbor/tasks/verifier-timeout/tests/test.sh new file mode 100644 index 000000000..d85ae4225 --- /dev/null +++ b/tests/integrations/harbor/tasks/verifier-timeout/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +sleep 30 +echo 1 > /logs/verifier/reward.txt diff --git a/tests/integrations/harbor/test_controller.py b/tests/integrations/harbor/test_controller.py new file mode 100644 index 000000000..bcff0e176 --- /dev/null +++ b/tests/integrations/harbor/test_controller.py @@ -0,0 +1,219 @@ +"""The in-image Harbor controller, adapted from fixture tasks and run in Docker. + +Each directory under ``tasks/`` is a Harbor task with a reference +``solution/solve.sh`` and an optional ``expected.toml``. The scenario adapts +the task against this checkout's wheel and runs it under ``DockerRuntime`` with +an oracle that only runs the solution, so the verifier's reward proves the +controller kept each phase's contract. ``expected.toml`` keys, all optional, +at the top level or per ``[[case]]`` (a case adds to the top level): + +- ``reward``: the reward every run must get (default ``1.0``); +- ``error``: text the run's error or evaluation must contain; +- ``repeat``: run the solution in this many SSH sessions; +- ``group`` and ``shared_width``: rollouts per task, sharing placements; +- ``capability_probe``: ``{name, tools}`` an MCP capability opened host-side lists; +- ``runtime.env_vars`` and ``runtime.bind_mounts`` (``source`` relative to the + fixture): what ``DockerRuntime`` gives the container; +- ``startup_error``: the controller must refuse to serve, with this message. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.agents.base import Agent +from hud.capabilities import MCPClient, SSHClient +from hud.eval import DockerRuntime, Shared +from hud.eval.runtime import DockerBindMount +from hud.integrations import harbor + +if TYPE_CHECKING: + from hud.eval import Provider + from hud.eval.run import Run + +TASKS = Path(__file__).parent / "tasks" + +pytestmark = [ + pytest.mark.e2e, + pytest.mark.docker, + pytest.mark.skipif(sys.platform == "win32", reason="adapted images are Linux containers"), +] + + +@dataclass(frozen=True) +class Case: + fixture: str + name: str + reward: float + error: str | None + repeat: int + group: int | None + shared_width: int | None + probe: dict[str, Any] | None + env_vars: dict[str, str] + bind_mounts: tuple[DockerBindMount, ...] + startup_error: str | None = field(default=None) + + @property + def id(self) -> str: + return f"{self.fixture}[{self.name}]" if self.name else self.fixture + + +def load_cases() -> list[Case]: + cases = [] + for root in sorted(path for path in TASKS.iterdir() if (path / "task.toml").is_file()): + spec_path = root / "expected.toml" + spec = tomllib.loads(spec_path.read_text("utf-8")) if spec_path.is_file() else {} + for variant in spec.pop("case", None) or [{}]: + merged = {**spec, **variant} + runtime = {**spec.get("runtime", {}), **variant.get("runtime", {})} + cases.append( + Case( + fixture=root.name, + name=merged.get("name", ""), + reward=merged.get("reward", 1.0), + error=merged.get("error"), + repeat=merged.get("repeat", 1), + group=merged.get("group"), + shared_width=merged.get("shared_width"), + probe=merged.get("capability_probe"), + env_vars=runtime.get("env_vars", {}), + bind_mounts=tuple( + DockerBindMount(root / mount["source"], mount["target"]) + for mount in runtime.get("bind_mounts", []) + ), + startup_error=merged.get("startup_error"), + ) + ) + return cases + + +CASES = load_cases() +RUNNABLE = [case for case in CASES if case.startup_error is None] +REFUSED = [case for case in CASES if case.startup_error is not None] + + +class Oracle(Agent): + """Run the fixture's reference solution; its stdout is the answer.""" + + def __init__(self, solution: str, *, repeat: int, probe: str | None) -> None: + super().__init__() + self.solution = solution + self.repeat = repeat + self.probe = probe + self.probed: list[list[str]] = [] + + async def __call__(self, run: Run) -> None: + if self.probe is not None: + mcp = await run.client.open(self.probe) + assert isinstance(mcp, MCPClient) + self.probed.append(sorted(tool.name for tool in await mcp.list_tools())) + ssh = await run.client.open("ssh") + assert isinstance(ssh, SSHClient) + stdout = "" + for session in range(self.repeat): + result = await ssh.run(self.solution, check=False) + if result.exit_status != 0: + raise RuntimeError( + f"solution session {session} exited with {result.exit_status}:\n" + f"{result.stdout}\n{result.stderr}" + ) + stdout = str(result.stdout) + run.trace.content = stdout + + +def adapted(case: Case, wheel: Path, tmp_path: Path) -> harbor.AdaptResult: + dataset = tmp_path / "harbor-fixtures" + shutil.copytree( + TASKS / case.fixture, + dataset / case.fixture, + symlinks=True, + ignore=shutil.ignore_patterns("expected.toml", "managed"), + ) + result = harbor.adapt(dataset, hud_requirement=str(wheel)) + assert result.failures == () + return result + + +def report(run: Run) -> str: + """Everything a failed run says about why, for assertions and their messages.""" + evaluation = run.evaluation + info = evaluation.get("info") or {} + return "\n".join( + str(part) + for part in ( + run.trace.content, + run.trace.error, + evaluation.get("content"), + info.get("stdout"), + info.get("stderr"), + ) + if part + ) + + +@pytest.mark.parametrize("case", RUNNABLE, ids=[case.id for case in RUNNABLE]) +async def test_a_harbor_task_grades_as_its_fixture_expects( + case: Case, wheel: Path, tmp_path: Path +) -> None: + taskset = adapted(case, wheel, tmp_path).taskset + docker = DockerRuntime(env_vars=case.env_vars, bind_mounts=case.bind_mounts) + runtime: Provider = Shared(docker, width=case.shared_width) if case.shared_width else docker + oracle = Oracle( + (TASKS / case.fixture / "solution" / "solve.sh").read_text("utf-8"), + repeat=case.repeat, + probe=None if case.probe is None else case.probe["name"], + ) + + job = await taskset.run(oracle, runtime=runtime, group=case.group, max_concurrent=1) + + assert len(job.runs) == (case.group or 1) + for run in job.runs: + detail = report(run) + assert run.reward == case.reward, detail + if case.error is not None: + assert case.error in detail + if case.probe is not None: + assert oracle.probed == [sorted(case.probe["tools"])] * len(job.runs) + + +@pytest.mark.parametrize("case", REFUSED, ids=[case.id for case in REFUSED]) +def test_a_controller_with_an_unusable_configuration_refuses_to_serve( + case: Case, wheel: Path, tmp_path: Path +) -> None: + # Providers yield once the control port is published, before the serve + # process proves itself, so the refusal is observed by running the main + # service of the adapted project in the foreground. + (row,) = adapted(case, wheel, tmp_path).taskset + assert row.runtime_config is not None and row.runtime_config.compose is not None + compose = row.runtime_config.compose.document + assert isinstance(compose, Path) + + def compose_command(*args: str, timeout: float) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", "compose", "--file", str(compose), *args], + cwd=compose.parent, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + try: + built = compose_command("build", timeout=1200) + assert built.returncode == 0, built.stderr[-4000:] + served = compose_command("run", "--rm", "main", timeout=120) + finally: + compose_command("down", "--volumes", "--remove-orphans", timeout=120) + + assert served.returncode != 0 + assert case.startup_error is not None + assert case.startup_error in served.stdout + served.stderr diff --git a/tests/integrations/harbor/test_export.py b/tests/integrations/harbor/test_export.py index 926dbf036..9fb442ceb 100644 --- a/tests/integrations/harbor/test_export.py +++ b/tests/integrations/harbor/test_export.py @@ -17,7 +17,6 @@ import textwrap import tomllib import uuid -from pathlib import Path from typing import TYPE_CHECKING, Any import pytest @@ -27,11 +26,10 @@ if TYPE_CHECKING: from collections.abc import Callable + from pathlib import Path from tests.harness import HudEnv -REPO = Path(__file__).resolve().parents[3] - ENV_PY = """\ import os from pathlib import Path @@ -660,18 +658,6 @@ async def test_the_entrypoint_refuses_to_run_the_agent_when_task_setup_fails( ) -@pytest.fixture(scope="module") -def wheel(tmp_path_factory: pytest.TempPathFactory) -> Path: - wheels = tmp_path_factory.mktemp("wheels") - subprocess.run( - ["uv", "build", "--wheel", "--out-dir", str(wheels)], - cwd=REPO, - check=True, - capture_output=True, - ) - return next(wheels.glob("*.whl")) - - def docker(*args: str, timeout: float = 600) -> subprocess.CompletedProcess[str]: return subprocess.run( ["docker", *args], capture_output=True, text=True, timeout=timeout, check=False From 559bc1b2c4f0500f167ba28f048e5e04b30ea172 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 071/248] docs: state which verifier reward files and values count The verifier-environment docs named the two reward files but not that reward.json wins, which shapes it accepts, or that booleans, non-finite or missing rewards and timeouts are error results; the controller suite now pins all of these. --- docs/v6/experimental/verifier-environments.mdx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/v6/experimental/verifier-environments.mdx b/docs/v6/experimental/verifier-environments.mdx index 9fb02d568..393ba2480 100644 --- a/docs/v6/experimental/verifier-environments.mdx +++ b/docs/v6/experimental/verifier-environments.mdx @@ -137,7 +137,10 @@ runtime, the phases remain isolated: artifact paths and `/logs`; - `/tests/test.sh` runs from the verifier image with its own user, workdir, environment, network mode, allowlist, and credentials directory; and -- verifier output is read from `/logs/verifier/reward.json` or `/logs/verifier/reward.txt`. +- verifier output is read from `/logs/verifier/reward.json` (a number, or an object with a + `reward` or `score` number) or, when that file is absent, `/logs/verifier/reward.txt`. A + boolean, non-finite, or missing reward, or a verifier past its timeout, is an error result + rather than a score. `RuntimeConfig.compose.service_access=True` mounts a Docker socket in the generated `main` service. From 334d4c567a3604009fa66c43ac4bcba9810ec7c0 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 072/248] test: drive oracle environments end to end through hud eval and Taskset.run Four fixture environments whose reward is 1 only when their whole path works: a prompt echo, a token readable only through the workspace shell, a code revealed one digit per MCP tool call, and an actor graded 0 whose verifier reads its answer. A scripted model drives each through the real CLI and through Taskset.run on every PR, and the step spans must show the expected task phases and tool calls with no errors. The harness Hud takes an optional interpreter so later scenarios can run the CLI from a generated project's own virtual environment. --- tests/e2e/__init__.py | 0 tests/e2e/lifecycle.py | 29 ++++++++++ tests/e2e/oracles/__init__.py | 5 ++ tests/e2e/oracles/echo.py | 16 ++++++ tests/e2e/oracles/secret.py | 32 +++++++++++ tests/e2e/oracles/state.py | 62 ++++++++++++++++++++ tests/e2e/oracles/verifier.py | 24 ++++++++ tests/e2e/test_oracles.py | 104 ++++++++++++++++++++++++++++++++++ tests/harness/cli.py | 7 ++- 9 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/lifecycle.py create mode 100644 tests/e2e/oracles/__init__.py create mode 100644 tests/e2e/oracles/echo.py create mode 100644 tests/e2e/oracles/secret.py create mode 100644 tests/e2e/oracles/state.py create mode 100644 tests/e2e/oracles/verifier.py create mode 100644 tests/e2e/test_oracles.py diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/e2e/lifecycle.py b/tests/e2e/lifecycle.py new file mode 100644 index 000000000..bbcccbe85 --- /dev/null +++ b/tests/e2e/lifecycle.py @@ -0,0 +1,29 @@ +"""What a cleanly completed task lifecycle looks like in a run's step spans.""" + +from __future__ import annotations + +from typing import Any + +# Tool steps may fail without failing the run: a real model can call a tool badly. +LIFECYCLE_SOURCES = {"task", "agent", "system"} + + +def task_calls(steps: list[dict[str, Any]]) -> list[dict[str, Any]]: + """The ``task_call`` of every task step, in order.""" + return [step["task_call"] for step in steps if step["source"] == "task"] + + +def assert_clean_lifecycle(steps: list[dict[str, Any]]) -> tuple[dict[str, Any], dict[str, Any]]: + """Assert the run set up its task, graded it, and errored nowhere; return both results.""" + errors = [ + step["error"] for step in steps if step["source"] in LIFECYCLE_SOURCES and step.get("error") + ] + assert not errors, errors + calls = task_calls(steps) + assert [call["phase"] for call in calls] == ["setup", "evaluate"], calls + setup, grade = (call["result"] for call in calls) + assert setup.get("prompt"), f"task setup returned no prompt: {setup}" + assert not grade.get("isError"), grade + assert isinstance(grade["score"], int | float), grade + assert 0 <= grade["score"] <= 1, grade + return setup, grade diff --git a/tests/e2e/oracles/__init__.py b/tests/e2e/oracles/__init__.py new file mode 100644 index 000000000..dda42dfd2 --- /dev/null +++ b/tests/e2e/oracles/__init__.py @@ -0,0 +1,5 @@ +"""Oracle environments: each reward is trivial when its path works and impossible when it does not. + +Every module defines its environments and a ``tasks`` list, so ``hud eval .py`` +and ``Taskset.from_file`` both load it the way they load a user's tasks file. +""" diff --git a/tests/e2e/oracles/echo.py b/tests/e2e/oracles/echo.py new file mode 100644 index 000000000..c0a47070d --- /dev/null +++ b/tests/e2e/oracles/echo.py @@ -0,0 +1,16 @@ +"""The agent repeats a word from the prompt, so a reward of 1 proves the prompt reached it.""" + +from __future__ import annotations + +from hud import Environment + +env = Environment("oracle-echo") + + +@env.template(id="echo") +async def echo(word: str): + answer = yield f"Reply with this word and nothing else: {word}" + yield 1.0 if (answer or "").strip() == word else 0.0 + + +tasks = [echo(word="tangerine")] diff --git a/tests/e2e/oracles/secret.py b/tests/e2e/oracles/secret.py new file mode 100644 index 000000000..da3c9e6d4 --- /dev/null +++ b/tests/e2e/oracles/secret.py @@ -0,0 +1,32 @@ +"""A token only the workspace holds: the agent can answer only by reading it through a shell.""" + +from __future__ import annotations + +import os +import secrets +import shutil +import tempfile +from pathlib import Path + +from hud import Environment + +ROOT = Path(tempfile.gettempdir()) / f"hud-oracle-secret-{os.getpid()}" + +env = Environment("oracle-secret") +env.workspace(ROOT, guest_path=str(ROOT)) + + +@env.shutdown +async def remove_workspace() -> None: + shutil.rmtree(ROOT, ignore_errors=True) + + +@env.template(id="read-secret") +async def read_secret(): + token = secrets.token_hex(8) + (ROOT / "secret.txt").write_text(token, encoding="utf-8") + answer = yield f"Read the file {ROOT / 'secret.txt'} and reply with its contents only." + yield 1.0 if (answer or "").strip() == token else 0.0 + + +tasks = [read_secret()] diff --git a/tests/e2e/oracles/state.py b/tests/e2e/oracles/state.py new file mode 100644 index 000000000..4945e39c3 --- /dev/null +++ b/tests/e2e/oracles/state.py @@ -0,0 +1,62 @@ +"""A code revealed one digit per tool call: only state kept across calls yields all of it.""" + +from __future__ import annotations + +import asyncio +import secrets +import socket + +import uvicorn +from fastmcp import FastMCP + +from hud import Environment +from hud.capabilities import Capability + +server = FastMCP(name="oracle-state") +code: list[str] = [] +revealed: list[str] = [] + + +@server.tool +def next_digit() -> str: + """Return the next digit of the code, or "done" once every digit is revealed.""" + if len(revealed) == len(code): + return "done" + revealed.append(code[len(revealed)]) + return revealed[-1] + + +env = Environment("oracle-state") +http = uvicorn.Server(uvicorn.Config(server.http_app(path="/mcp"), log_level="error")) +serving: list[asyncio.Task[None]] = [] + + +@env.initialize +async def serve_tools() -> None: + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + serving.append(asyncio.create_task(http.serve(sockets=[listener]))) + while not http.started: # noqa: ASYNC110 - uvicorn signals readiness only by this flag + await asyncio.sleep(0.01) + port = listener.getsockname()[1] + env.add_capability(Capability.mcp(name="tools", url=f"http://127.0.0.1:{port}/mcp")) + + +@env.shutdown +async def stop_tools() -> None: + http.should_exit = True + await asyncio.gather(*serving) + + +@env.template(id="collect-code") +async def collect_code(length: int): + code[:] = [str(secrets.randbelow(10)) for _ in range(length)] + revealed.clear() + answer = yield ( + 'Call next_digit until it returns "done", then reply with every digit it returned, ' + "in order, as one string." + ) + yield 1.0 if (answer or "").strip() == "".join(code) and revealed == code else 0.0 + + +tasks = [collect_code(length=4)] diff --git a/tests/e2e/oracles/verifier.py b/tests/e2e/oracles/verifier.py new file mode 100644 index 000000000..23713ac7a --- /dev/null +++ b/tests/e2e/oracles/verifier.py @@ -0,0 +1,24 @@ +"""An actor that grades itself 0 and a verifier that reads its answer: 1 means the verifier won.""" + +from __future__ import annotations + +from hud import Environment + +actor = Environment("oracle-actor") +judge = Environment("oracle-judge") + + +@actor.template(id="relay") +async def relay(word: str): + answer = yield f"Reply with this word and nothing else: {word}" + yield {"score": 0.0, "answer": answer} + + +@judge.template(id="check-relay") +async def check_relay(expected: str): + actor_result = yield "" + yield 1.0 if (actor_result["answer"] or "").strip() == expected else 0.0 + + +tasks = [relay(word="lighthouse")] +tasks[0].verifier = check_relay(expected="lighthouse") diff --git a/tests/e2e/test_oracles.py b/tests/e2e/test_oracles.py new file mode 100644 index 000000000..61e1851e0 --- /dev/null +++ b/tests/e2e/test_oracles.py @@ -0,0 +1,104 @@ +"""Oracle environments driven end to end by a scripted model: ``hud eval`` and ``Taskset.run``. + +Each oracle's reward is 1 only when its whole path works: the prompt reaches the +model, the model's tool calls reach the environment, the environment's state +survives between calls, and the grade of record comes from the right task. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from hud.agents import ClaudeAgent +from hud.agents.types import ClaudeConfig +from hud.eval import Taskset +from tests.harness import call, say, steps + +from .lifecycle import task_calls + +if TYPE_CHECKING: + from collections.abc import Callable + + from tests.harness import Hud, HudEnv, ModelRequest, Models, Turn + +ORACLES = Path(__file__).parent / "oracles" +MODEL = "claude-sonnet-4-6" +MAX_STEPS = 8 +EVAL_ARGS = ("claude", "--model", MODEL, "--max-steps", str(MAX_STEPS), "-y", "--json") + + +def repeat_word(request: ModelRequest) -> Turn: + return say(request.prompt.rsplit(" ", 1)[-1]) + + +def read_token(request: ModelRequest) -> Turn: + """Cat the file the prompt names, then answer with the token the shell printed.""" + if not request.tool_results: + path = re.search(r"Read the file (\S+) and", request.prompt) + assert path is not None, request.prompt + return call("bash", command=f"cat {path.group(1)}") + token = re.search(r"^[0-9a-f]{16}$", request.tool_results[-1], re.MULTILINE) + return say(token.group(0) if token else "no token in the tool result") + + +def collect_digits(request: ModelRequest) -> Turn: + """Call ``next_digit`` until it says done, then answer with every digit it returned.""" + results = [result.strip() for result in request.tool_results] + if results and results[-1] == "done": + return say("".join(results[:-1])) + (tool,) = [name for name in request.tools if name.endswith("next_digit")] + return call(tool) + + +@dataclass(frozen=True) +class Oracle: + module: str + answer: Callable[[ModelRequest], Turn] + phases: tuple[str, ...] + tool_steps: int + + +ORACLE_TABLE = [ + Oracle("echo", repeat_word, ("setup", "evaluate"), tool_steps=0), + Oracle("secret", read_token, ("setup", "evaluate"), tool_steps=1), + Oracle("state", collect_digits, ("setup", "evaluate"), tool_steps=5), + Oracle("verifier", repeat_word, ("setup", "evaluate", "setup", "evaluate"), tool_steps=0), +] + + +@pytest.mark.parametrize("oracle", ORACLE_TABLE, ids=lambda oracle: oracle.module) +@pytest.mark.parametrize("entry", ["hud-eval", "taskset-run"]) +async def test_an_oracle_scores_one_only_through_its_whole_path( + oracle: Oracle, entry: str, models: Models, hud_env: HudEnv, hud: Hud +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(oracle.answer) + source = ORACLES / f"{oracle.module}.py" + + if entry == "hud-eval": + result = hud("eval", str(source), *EVAL_ARGS) + assert result.exit_code == 0, result + (run,) = result.json["runs"] + else: + agent = ClaudeAgent(ClaudeConfig(model=MODEL, max_steps=MAX_STEPS)) + (completed,) = (await Taskset.from_file(source).run(agent)).runs + run = { + "slug": completed.slug, + "reward": completed.reward, + "is_error": completed.trace.is_error, + "trace_id": completed.trace_id, + } + + assert run["reward"] == 1.0 + assert run["is_error"] is False + assert run["slug"] == next(iter(Taskset.from_file(source))).slug + recorded = steps(run["trace_id"]) + assert [step["error"] for step in recorded if step.get("error")] == [] + assert tuple(call["phase"] for call in task_calls(recorded)) == oracle.phases + assert sum(step["source"] == "tool" for step in recorded) == oracle.tool_steps + assert task_calls(recorded)[0]["result"]["prompt"] == models.requests()[0].prompt diff --git a/tests/harness/cli.py b/tests/harness/cli.py index bbbd47441..f2c85c6b5 100644 --- a/tests/harness/cli.py +++ b/tests/harness/cli.py @@ -37,10 +37,13 @@ class Hud: The subprocess inherits this process's environment, which the ``hud_env`` fixture has already isolated: a temporary home, no developer credentials, and every service URL pointed at the fake services or a closed port. + ``python`` runs the CLI from another interpreter, such as a project's own + virtual environment; it defaults to this one. """ - def __init__(self, cwd: Path) -> None: + def __init__(self, cwd: Path, *, python: Path | None = None) -> None: self.cwd = cwd + self.python = str(python) if python is not None else sys.executable cwd.mkdir(parents=True, exist_ok=True) def __call__( @@ -52,7 +55,7 @@ def __call__( timeout: float = 120, ) -> Result: completed = subprocess.run( - [sys.executable, "-m", "hud.cli", *args], + [self.python, "-m", "hud.cli", *args], cwd=cwd or self.cwd, env={**os.environ, "COLUMNS": "120", "NO_COLOR": "1", "TERM": "dumb", **(env or {})}, input=input, From c0c6a9968dffed1b38a5fffcc5170a0a14c3eb36 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 073/248] test: rewrite the template tests as black-box tests users can copy The hud init templates ship their tests to users, so they now show the supported way to test an environment: serve it in its own process with SubprocessRuntime, configure it through environment variables, start tasks and grade answers over the control channel, and fake only the outside world with small local servers (a VNC listener, a judge, the data-file API and its storage). They no longer drive .func generators, call private helpers, patch module globals or the settings singleton, import hud.environment.workspace.usable_bwrap, or leak REPO_URL. New rows cover what was untested at runtime: an uploaded file named ../../x fails the start before anything is written, as do an unknown file id and a file with no download URL; a non-top-level test_path is rejected. Literal-equals-literal task tests are gone. The coding tests that serve the isolated workspace are marked sandbox and skip unless Linux, root, bubblewrap and an environment under /usr are present, the layout of the coding image. The cua template reads VNC_PORT so its tests can supply their own listener. --- environments/argument-hints/tests/conftest.py | 120 ++++- environments/argument-hints/tests/test_env.py | 428 ++++++++---------- environments/coding/README.md | 8 +- environments/coding/pyproject.toml | 4 + environments/coding/tests/conftest.py | 93 +++- environments/coding/tests/test_env.py | 184 +++++++- environments/coding/tests/test_grader.py | 28 +- .../coding/tests/test_local_rollout.py | 235 ---------- environments/cua/env.py | 6 +- environments/cua/tests/conftest.py | 73 ++- environments/cua/tests/test_env.py | 239 +++++----- 11 files changed, 748 insertions(+), 670 deletions(-) delete mode 100644 environments/coding/tests/test_local_rollout.py diff --git a/environments/argument-hints/tests/conftest.py b/environments/argument-hints/tests/conftest.py index 069464691..2a8e101e3 100644 --- a/environments/argument-hints/tests/conftest.py +++ b/environments/argument-hints/tests/conftest.py @@ -1,8 +1,126 @@ -"""Make the environment project importable when tests run from the SDK root.""" +"""Serve the environment the way HUD does, against a local stand-in for the HUD platform. +The stand-in answers the three things this environment calls: the data-file API +(metadata and a presigned download URL), the storage behind that URL, and the +inference gateway the LLM judge uses. The environment runs in its own process, +configured only through environment variables, exactly as ``hud eval`` serves it. +""" + +from __future__ import annotations + +import json import sys +import threading +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +import pytest + PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) + + +@dataclass +class DataFile: + filename: str + content: bytes + downloadable: bool + + +@dataclass +class Received: + method: str + path: str + authorization: str | None + body: dict = field(default_factory=dict) + + +class Platform: + """The HUD API, storage and gateway this environment talks to. + + The judge grades like a perfect answer: positive criteria MET, negative ones + UNMET. An answer listed in ``rejected_answers`` gets an HTTP 400 instead. + """ + + def __init__(self) -> None: + self.files: dict[str, DataFile] = {} + self.rejected_answers: set[str] = set() + self.received: list[Received] = [] + self.judging = threading.Barrier(1) + platform = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + platform.received.append(Received("GET", self.path, self.headers["Authorization"])) + parts = self.path.strip("/").split("/") + if parts[0] == "storage" and parts[1] in platform.files: + self.reply(200, platform.files[parts[1]].content, "application/octet-stream") + elif parts[:2] == ["v2", "data"] and parts[2] in platform.files: + data_file = platform.files[parts[2]] + if parts[3:] == ["download"]: + url = f"{platform.url}/storage/{parts[2]}" if data_file.downloadable else None + self.reply_json(200, {"url": url}) + else: + self.reply_json(200, {"filename": data_file.filename}) + else: + self.reply_json(404, {"detail": "not found"}) + + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + platform.received.append(Received("POST", self.path, self.headers["Authorization"], body)) + prompt = body["messages"][-1]["content"] + answer = prompt.split("\n", 1)[1].split("\n", 1)[0] + platform.judging.wait(timeout=10) + if answer in platform.rejected_answers: + self.reply_json(400, {"error": {"message": "judge rejected request"}}) + return + met = "\npositive" in prompt + verdict = {"criterion_status": "MET" if met else "UNMET", "explanation": "stand-in"} + self.reply_json(200, {"choices": [{"message": {"content": json.dumps(verdict)}}]}) + + def reply_json(self, status: int, payload: dict) -> None: + self.reply(status, json.dumps(payload).encode(), "application/json") + + def reply(self, status: int, body: bytes, content_type: str) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + del format, args + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.server.server_port}" + + def upload(self, file_id: str, filename: str, content: bytes = b"", *, downloadable: bool = True) -> None: + """Make ``file_id`` known to the data-file API; ``downloadable=False`` withholds its URL.""" + self.files[file_id] = DataFile(filename, content, downloadable) + + def requests(self, method: str, prefix: str) -> list[Received]: + return [item for item in self.received if item.method == method and item.path.startswith(prefix)] + + +@pytest.fixture +def workspace(tmp_path: Path) -> Path: + return tmp_path / "workspace" + + +@pytest.fixture(autouse=True) +def platform(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, workspace: Path): + """Point the environment at the stand-in platform with a runtime key and no user config.""" + stand_in = Platform() + thread = threading.Thread(target=stand_in.server.serve_forever, daemon=True) + thread.start() + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("WORKSPACE_DIR", str(workspace)) + monkeypatch.setenv("HUD_API_KEY", "runtime-key") + monkeypatch.setenv("HUD_API_URL", stand_in.url) + monkeypatch.setenv("HUD_GATEWAY_URL", stand_in.url) + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "0") + yield stand_in + stand_in.server.shutdown() + thread.join() diff --git a/environments/argument-hints/tests/test_env.py b/environments/argument-hints/tests/test_env.py index 11ddb097a..1641670f3 100644 --- a/environments/argument-hints/tests/test_env.py +++ b/environments/argument-hints/tests/test_env.py @@ -1,258 +1,216 @@ -"""What the example promises: hinted arguments, safe staging, weighted grading.""" +"""What the example promises: hinted arguments, safe staging, weighted grading, the runtime key. + +Each test starts a fresh environment process against the stand-in platform in +``conftest.py``, starts a task with the arguments a task row carries, and grades +an answer: what a rollout does, without an agent. +""" from __future__ import annotations import asyncio -import json -import sys -from functools import partial -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import threading +from contextlib import asynccontextmanager from pathlib import Path -from threading import Barrier, Thread -import httpx import pytest -from hud import LocalRuntime, connect +from hud import SubprocessRuntime, Task, connect from hud.clients import HudProtocolError -from hud.graders import SubScore - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -import env as env_module # noqa: E402 - - -def test_every_editable_argument_declares_its_hint(): - """The hints are the point of this environment; a rename must fail here.""" - properties = env_module.review_files.manifest_entry()["args"]["properties"] - assert properties["prompt"]["x-hud-hint"] == "prompt" - assert properties["attachments"]["x-hud-hint"] == "data-files" - assert properties["criteria"]["x-hud-hint"] == "grading" - assert "hud_api_key" in properties - required = env_module.review_files.manifest_entry()["args"].get("required") or [] - assert "hud_api_key" not in required - # The console resolves the reference through the $defs discovery bundles. - assert properties["attachments"]["items"]["$ref"] == "#/$defs/DataFileRef" - assert "file_id" in env_module.review_files.manifest_entry()["args"]["$defs"]["DataFileRef"]["properties"] - - -def test_staging_refuses_a_path_that_leaves_the_files_directory(tmp_path: Path): - """An uploaded file names its own destination, so the path is untrusted.""" - assert env_module._destination(tmp_path, "notes/summary.md") == tmp_path / "notes/summary.md" - for hostile in ("../escape.md", "/etc/passwd", "..\\escape.md"): - with pytest.raises(env_module.DataFileError): - env_module._destination(tmp_path, hostile) - - -async def test_criteria_reach_the_judge_unchanged(monkeypatch: pytest.MonkeyPatch): - """The argument is the grader's input, so nothing may be rewritten en route.""" - seen: dict[str, object] = {} - - async def fake_compute_score(**kwargs: object): - seen.update(kwargs) - return SubScore(name="LLMJudgeGrader", value=1.0) - - monkeypatch.setattr(env_module.LLMJudgeGrader, "compute_score", fake_compute_score) - criteria = [ - env_module.Criterion(requirement="Recommends an interview.", weight=3), - env_module.Criterion(requirement="Invents an employer.", weight=-2), - ] - - result = await env_module._grade("Hire her.", "Should we interview her?", criteria) - - assert seen["criteria"] == [("Recommends an interview.", 3.0), ("Invents an employer.", -2.0)] - assert seen["answer"] == "Hire her." - assert seen["question"] == "Should we interview her?" - assert result.reward == pytest.approx(1.0) - - -async def test_no_criteria_scores_zero_without_raising(): - """An empty list is a misconfigured task, not a crashed rollout.""" - result = await env_module._grade("Hire her.", "", []) - assert result.reward == pytest.approx(0.0) - - -async def test_staging_directory_is_cleared_between_tasks( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -): - """A prior task's leftovers (including symlinks) must not survive into staging.""" - async def fake_compute_score(**kwargs: object): - return SubScore(name="LLMJudgeGrader", value=1.0) +from env import review_files - async def fake_stage(refs, root: Path): - return [] +ENV_SOURCE = Path(__file__).resolve().parent.parent / "env.py" +CRITERIA = [ + {"requirement": "Names exactly three primes.", "weight": 2}, + {"requirement": "Explains each one.", "weight": 1}, + {"requirement": "Calls a composite number prime.", "weight": -2}, +] - monkeypatch.setattr(env_module.LLMJudgeGrader, "compute_score", fake_compute_score) - monkeypatch.setattr(env_module, "_stage", fake_stage) - monkeypatch.setattr(env_module, "WORKSPACE_ROOT", tmp_path) - files_dir = tmp_path / env_module.FILES_DIRNAME - files_dir.mkdir(parents=True) - (files_dir / "escape").symlink_to(tmp_path / "outside") +def row(attachments: list[dict] | None = None, criteria: list[dict] | None = None) -> Task: + return review_files( + prompt="Name three primes.", + attachments=attachments or [], + criteria=CRITERIA if criteria is None else criteria, + hud_api_key="task-key", + ) - task = env_module.review_files.func(prompt="p", attachments=[], criteria=[]) - await task.asend(None) - assert files_dir.is_dir() - assert list(files_dir.iterdir()) == [] +@asynccontextmanager +async def served(task: Task): + """A fresh environment process serving ``task``, and a client connected to it.""" + async with SubprocessRuntime(ENV_SOURCE)(task) as runtime, connect(runtime) as client: + yield client -async def test_arguments_arrive_as_plain_json( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -): - """A task run sends JSON, not model instances, so the template must validate.""" - seen: dict[str, object] = {} +async def test_every_editable_argument_declares_its_hint(): + async with served(row()) as client: + (template,) = await client.list_tasks() - async def fake_compute_score(**kwargs: object): - seen.update(kwargs) - return SubScore(name="LLMJudgeGrader", value=1.0) + arguments = template["args"] + hints = {name: spec.get("x-hud-hint") for name, spec in arguments["properties"].items()} + assert hints == { + "prompt": "prompt", + "attachments": "data-files", + "criteria": "grading", + "hud_api_key": None, + } + assert "hud_api_key" not in arguments["required"] + # The console resolves the attachment reference through $defs. + assert arguments["properties"]["attachments"]["items"]["$ref"] == "#/$defs/DataFileRef" + assert "file_id" in arguments["$defs"]["DataFileRef"]["properties"] - staged: list[env_module.DataFileRef] = [] - async def fake_stage( - refs: list[env_module.DataFileRef], - root: Path, - ): - staged.extend(refs) - return [{"path": "files/resume.pdf", "file_id": refs[0].file_id}] - - monkeypatch.setattr(env_module.LLMJudgeGrader, "compute_score", fake_compute_score) - monkeypatch.setattr(env_module, "_stage", fake_stage) - monkeypatch.setattr(env_module, "WORKSPACE_ROOT", tmp_path) - monkeypatch.setattr(env_module.settings, "api_key", None) - - task = env_module.review_files.func( - prompt="Summarise the role.", - attachments=[{"file_id": "8b1f", "path": "resume.pdf"}], - criteria=[{"requirement": "Recommends an interview.", "weight": 2}], - hud_api_key="test-key", - ) - frame = await task.asend(None) +async def test_attachments_are_staged_with_the_runtime_key_and_declared(platform, workspace): + platform.upload("notes", "notes.txt", b"three primes") + platform.upload("report", "report.pdf", b"%PDF") + task = row(attachments=[{"file_id": "notes", "path": "reading/notes.md"}, {"file_id": "report"}]) - # Each argument reached its model rather than staying a bare dict. - assert staged == [env_module.DataFileRef(file_id="8b1f", path="resume.pdf")] - assert "Summarise the role." in frame["prompt"] - assert frame["data_files"] == [{"path": "files/resume.pdf", "file_id": "8b1f"}] + async with served(task) as client: + started = await client.start_task(task.id, task.args) - # A task run sends the agent's final text, the same as a real grade call. - result = await task.asend("Hire her.") - - assert seen["criteria"] == [("Recommends an interview.", 2.0)] - assert result.reward == pytest.approx(1.0) - - -@pytest.mark.parametrize("fail_second", [False, True]) -async def test_concurrent_sessions_use_the_runtime_credential( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, fail_second: bool + assert started["data_files"] == [ + {"path": "files/reading/notes.md", "file_id": "notes"}, + {"path": "files/report.pdf", "file_id": "report"}, + ] + assert "- files/reading/notes.md\n- files/report.pdf" in started["prompt"] + assert started["prompt"].endswith("Name three primes.") + assert (workspace / "files/reading/notes.md").read_bytes() == b"three primes" + assert (workspace / "files/report.pdf").read_bytes() == b"%PDF" + assert {item.authorization for item in platform.requests("GET", "/v2/data")} == {"Bearer runtime-key"} + # The presigned URL carries its own credentials; the key never goes to storage. + assert {item.authorization for item in platform.requests("GET", "/storage")} == {None} + + +@pytest.mark.parametrize( + ("upload", "attachment", "message"), + [ + pytest.param( + {"filename": "notes.txt"}, + {"file_id": "notes", "path": "../escape.md"}, + "unsafe data file path", + id="hostile-destination", + ), + pytest.param( + {"filename": "../../escape.md"}, + {"file_id": "notes"}, + "unsafe data file path", + id="hostile-uploaded-filename", + ), + pytest.param( + {"filename": "..\\escape.md"}, + {"file_id": "notes"}, + "unsafe data file path", + id="backslash-filename", + ), + pytest.param(None, {"file_id": "notes"}, "GET /v2/data/notes failed: 404", id="unknown-file"), + pytest.param( + {"filename": "notes.txt", "downloadable": False}, + {"file_id": "notes"}, + "has no download url", + id="no-download-url", + ), + ], +) +async def test_a_file_that_cannot_be_staged_fails_the_start_before_anything_is_written( + upload, attachment, message, platform, workspace ): - monkeypatch.setattr(env_module, "WORKSPACE_ROOT", tmp_path) - monkeypatch.setattr(env_module.settings, "api_key", "process-key") - arrived = Barrier(2, timeout=5) - seen = {} - - class Gateway(BaseHTTPRequestHandler): - def do_POST(self): - body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) - prompt = body["messages"][-1]["content"] - answer = prompt.split("\n", 1)[1].split("\n", 1)[0] - seen[answer] = self.headers["Authorization"] - arrived.wait() - assert env_module.settings.api_key == "process-key" - rejected = fail_second and answer == "second" - payload = json.dumps( - {"error": {"message": "judge rejected request"}} - if rejected - else { - "choices": [ - { - "message": { - "content": '{"criterion_status":"MET","explanation":"fixture"}', - } - } - ] - } - ).encode() - self.send_response(400 if rejected else 200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - first_task = env_module.review_files( - prompt="Give an answer.", - attachments=[], - criteria=[{"requirement": "An answer is present.", "weight": 1}], - hud_api_key="first-key", - ) - second_task = first_task.model_copy(update={"args": {**first_task.args, "hud_api_key": "second-key"}}) - with ThreadingHTTPServer(("127.0.0.1", 0), Gateway) as server: - monkeypatch.setattr(env_module.settings, "hud_gateway_url", f"http://127.0.0.1:{server.server_port}") - thread = Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - async with ( - LocalRuntime(env_module.env)(first_task) as runtime, - connect(runtime) as first, - connect(runtime) as second, - ): - await first.start_task(first_task.id, first_task.args) - await second.start_task(second_task.id, second_task.args) - results = await asyncio.gather( - first.grade({"answer": "first"}), - second.grade({"answer": "second"}), - return_exceptions=True, - ) - finally: - server.shutdown() - thread.join() - - assert seen == {"first": "Bearer process-key", "second": "Bearer process-key"} - assert env_module.settings.api_key == "process-key" - for index, result in enumerate(results): - if fail_second and index == 1: - assert isinstance(result, HudProtocolError) - assert "judge rejected request" in str(result) - continue - assert isinstance(result, dict) - assert result["score"] == 1.0 - assert "first-key" not in json.dumps(result) - assert "second-key" not in json.dumps(result) - - -async def test_file_staging_uses_the_runtime_credential(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): - monkeypatch.setattr(env_module, "WORKSPACE_ROOT", tmp_path) - monkeypatch.setattr(env_module.settings, "api_key", "runtime-key") - monkeypatch.setattr(env_module.settings, "hud_api_url", "https://api.test") - - async def handler(request: httpx.Request) -> httpx.Response: - if request.url.host == "storage.test": - assert "Authorization" not in request.headers - return httpx.Response(200, content=b"notes") - assert request.headers["Authorization"] == "Bearer runtime-key" - if request.url.path.endswith("/download"): - return httpx.Response(200, json={"url": "https://storage.test/notes.txt"}) - return httpx.Response(200, json={"filename": "notes.txt"}) - - monkeypatch.setattr( - env_module.httpx, - "AsyncClient", - partial( - httpx.AsyncClient, - transport=httpx.MockTransport(handler), + if upload is not None: + platform.upload("notes", **upload) + task = row(attachments=[attachment]) + + async with served(task) as client: + with pytest.raises(HudProtocolError) as failure: + await client.start_task(task.id, task.args) + + assert message in failure.value.message + assert platform.requests("GET", "/storage") == [] + assert list((workspace / "files").rglob("*")) == [] + assert not (workspace / "escape.md").exists() + + +async def test_staging_without_a_hud_key_fails_the_start(monkeypatch, platform): + monkeypatch.setenv("HUD_API_KEY", "") + platform.upload("notes", "notes.txt") + task = row(attachments=[{"file_id": "notes"}]) + + async with served(task) as client: + with pytest.raises(HudProtocolError) as failure: + await client.start_task(task.id, task.args) + + assert "HUD_API_KEY is unset" in failure.value.message + assert platform.received == [] + + +async def test_a_previous_task_leaves_nothing_in_the_files_directory(workspace): + files = workspace / "files" + files.mkdir(parents=True) + (files / "escape").symlink_to(workspace.parent / "outside") + task = row() + + async with served(task) as client: + started = await client.start_task(task.id, task.args) + + assert started["data_files"] == [] + assert "- (none)" in started["prompt"] + assert list(files.iterdir()) == [] + + +@pytest.mark.parametrize( + ("criteria", "score", "judged"), + [ + pytest.param( + CRITERIA, + 1.0, + [ + ("positive", "Names exactly three primes."), + ("positive", "Explains each one."), + ("negative", "Calls a composite number prime."), + ], + id="weighted-criteria", ), - ) - task = env_module.review_files.func( - prompt="Read the file.", - attachments=[{"file_id": "notes"}], - criteria=[], - hud_api_key="task-key", - ) - try: - frame = await anext(task) - assert frame["data_files"] == [{"path": "files/notes.txt", "file_id": "notes"}] - assert (tmp_path / "files/notes.txt").read_text() == "notes" - assert env_module.settings.api_key == "runtime-key" - finally: - await task.aclose() + pytest.param([], 0.0, [], id="no-criteria"), + ], +) +async def test_the_criteria_reach_the_judge_unchanged(criteria, score, judged, platform): + task = row(criteria=criteria) + + async with served(task) as client: + await client.start_task(task.id, task.args) + grade = await client.grade({"answer": "2, 3 and 5"}) + + assert grade["score"] == pytest.approx(score) + prompts = [item.body["messages"][-1]["content"] for item in platform.requests("POST", "/")] + assert sorted( + (prompt.split("\n")[1].split("\n")[0], prompt.split("\n")[1].split("\n")[0]) + for prompt in prompts + ) == sorted(judged) + assert all("\n2, 3 and 5\n" in prompt for prompt in prompts) + assert {item.authorization for item in platform.requests("POST", "/")} <= {"Bearer runtime-key"} + + +@pytest.mark.parametrize("rejected", [set(), {"second"}], ids=["both-graded", "one-judge-rejects"]) +async def test_concurrent_sessions_grade_with_the_runtime_key(rejected, platform): + platform.rejected_answers = rejected + platform.judging = threading.Barrier(2) + first = row(criteria=[{"requirement": "An answer is present.", "weight": 1}]) + second = first.model_copy(update={"args": {**first.args, "hud_api_key": "second-key"}}) + + async with ( + SubprocessRuntime(ENV_SOURCE)(first) as runtime, + connect(runtime) as one, + connect(runtime) as two, + ): + await one.start_task(first.id, first.args) + await two.start_task(second.id, second.args) + grades = await asyncio.gather( + one.grade({"answer": "first"}), two.grade({"answer": "second"}), return_exceptions=True + ) + + assert {item.authorization for item in platform.requests("POST", "/")} == {"Bearer runtime-key"} + assert grades[0]["score"] == 1.0 + if rejected: + assert isinstance(grades[1], HudProtocolError) + assert "judge rejected request" in grades[1].message + else: + assert grades[1]["score"] == 1.0 + assert "task-key" not in str(grades) and "second-key" not in str(grades) diff --git a/environments/coding/README.md b/environments/coding/README.md index 7e0a126ff..04d4d7346 100644 --- a/environments/coding/README.md +++ b/environments/coding/README.md @@ -37,8 +37,12 @@ namespaces. uv run pytest tests/ -q ``` -The suite covers repository and session isolation, task reset, hidden-test application, selected -JUnit scoring, and both bundled reference fixes. +The grader and task-row tests run anywhere. The `sandbox` tests serve the environment with its +isolated workspace and cover task reset, hidden-test application and tampering, selected JUnit +scoring, grading as the agent's user, and both bundled reference fixes. They need Linux, root, +bubblewrap, and the environment's Python installed under `/usr` (the sandbox shows the grader +nothing else), so they skip elsewhere; run them in the coding container, where the image installs +it at `/usr/local/venv`. See the [coding-agent cookbook](https://docs.hud.ai/v6/cookbooks/coding-agent) for the grading lifecycle and a task-authoring walkthrough. diff --git a/environments/coding/pyproject.toml b/environments/coding/pyproject.toml index 37c40abf6..43dba699c 100644 --- a/environments/coding/pyproject.toml +++ b/environments/coding/pyproject.toml @@ -37,5 +37,9 @@ quote-style = "double" indent-style = "space" line-ending = "auto" +[tool.pytest.ini_options] +asyncio_mode = "auto" +markers = ["sandbox: serves the isolated workspace; needs Linux, root, bubblewrap, and the env under /usr"] + [tool.hatch.build.targets.wheel] packages = [ ".",] diff --git a/environments/coding/tests/conftest.py b/environments/coding/tests/conftest.py index 5450d19f3..f8b04f0b0 100644 --- a/environments/coding/tests/conftest.py +++ b/environments/coding/tests/conftest.py @@ -1,36 +1,85 @@ -"""Make the environment project importable when tests run from the SDK root.""" +"""Run the environment the way HUD does, against a small fixture repository. + +Tests marked ``sandbox`` serve the real workspace, which refuses to run without +isolation. They need Linux, root, bubblewrap, and this project's Python +environment under /usr, the only prefix the sandbox shows the grader (the image +installs it at /usr/local/venv). Run them in the coding container. +""" import os +import shutil +import subprocess import sys from pathlib import Path import pytest -import pytest_asyncio from hud.environment import Workspace -from hud.environment.workspace import usable_bwrap PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) +SANDBOXED = ( + sys.platform == "linux" + and os.geteuid() == 0 + and shutil.which("bwrap") is not None + and Path(sys.prefix).is_relative_to("/usr") +) + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + if SANDBOXED: + return + skip = pytest.mark.skip(reason="needs Linux, root, bubblewrap, and the environment under /usr") + for item in items: + if "sandbox" in item.keywords: + item.add_marker(skip) + + +def git(cwd: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-c", f"safe.directory={cwd}", "-c", "user.name=t", "-c", "user.email=t@t", *args], + cwd=cwd, + check=False, + capture_output=True, + text=True, + ) + + +@pytest.fixture(scope="session") +def fixture_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: + """A repository whose ``bug_baseline`` branch is broken and ``bug_golden`` fixes it.""" + repo = tmp_path_factory.mktemp("fixture-repo") + (repo / "widget.py").write_text("BROKEN = True\n") + for args in ( + ("init", "-q", "-b", "main"), + ("add", "-A"), + ("commit", "-qm", "buggy widget"), + ("branch", "bug_baseline"), + ("checkout", "-qb", "bug_golden"), + ): + git(repo, *args).check_returncode() + (repo / "widget.py").write_text("BROKEN = False\n") + git(repo, "commit", "-qam", "reference fix").check_returncode() + return repo + + +@pytest.fixture +def repo(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, fixture_repo: Path) -> Path: + """Configure the environment to clone ``fixture_repo``; returns the agent's repository.""" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "0") + monkeypatch.setenv("REPO_URL", str(fixture_repo)) + monkeypatch.setenv("REPO_DIR", str(tmp_path / "repo")) + monkeypatch.setenv("BASELINE_DIR", str(tmp_path / "baseline")) + return tmp_path / "repo" + @pytest.fixture -def isolated_workspace(): - if sys.platform != "linux" or os.geteuid() != 0 or usable_bwrap() is None: - pytest.skip("requires Linux, root, and bubblewrap (run in the coding container)") - - -@pytest_asyncio.fixture(loop_scope="session") -async def grading_workspace(tmp_path, monkeypatch, isolated_workspace): - import env as coding_env - - root = tmp_path / "repo" - workspace = Workspace(root, guest_path=str(root), network=False, shell_uid=1000, require_isolation=True) - await workspace.start() - monkeypatch.setattr(coding_env, "workspace", workspace) - monkeypatch.setattr(coding_env, "REPO_DIR", root) - monkeypatch.setattr(coding_env, "_GIT", (*coding_env._GIT, "-c", f"safe.directory={root}")) - try: - yield workspace - finally: - await workspace.stop() +async def workspace(tmp_path: Path): + """An isolated workspace like the environment's, for grading commands directly.""" + root = tmp_path / "workspace" + served = Workspace(root, guest_path=str(root), network=False, shell_uid=1000, require_isolation=True) + await served.start() + yield served + await served.stop() diff --git a/environments/coding/tests/test_env.py b/environments/coding/tests/test_env.py index 38f64513f..e41de02e4 100644 --- a/environments/coding/tests/test_env.py +++ b/environments/coding/tests/test_env.py @@ -1,36 +1,172 @@ -"""In-process wiring smoke tests for the served environment.""" +"""The coding environment, served the way HUD serves it and driven over its control channel. -from unittest.mock import AsyncMock +Each test starts a fresh environment process, starts a task, edits the agent's +repository the way an agent would, and grades: what a rollout does, without an +agent. The workspace refuses to run unisolated, so every test here is ``sandbox``. +""" + +import shlex +import subprocess +import sys +from collections.abc import Callable +from contextlib import asynccontextmanager +from pathlib import Path import pytest +from hud import SubprocessRuntime, Task, connect +from hud.clients import HudProtocolError -import env as coding_env -from env import env +import tasks +from env import coding_task +pytestmark = pytest.mark.sandbox -def test_coding_template_registered(): - assert "coding-task" in env.tasks - assert env.tasks["coding-task"].manifest_entry()["id"] == "coding-task" +ENV_SOURCE = Path(__file__).resolve().parent.parent / "env.py" +PYTEST = f"{shlex.quote(sys.executable)} -m pytest -q -c /dev/null --rootdir=. test_widget.py --junitxml={{junit_path}}" +TEST_PATCH = """diff --git a/test_widget.py b/test_widget.py +new file mode 100644 +--- /dev/null ++++ b/test_widget.py +@@ -0,0 +1,10 @@ ++import unittest ++import widget ++ ++ ++class TestWidget(unittest.TestCase): ++ def test_not_broken(self): ++ self.assertFalse(widget.BROKEN) ++ ++ def test_existing_behavior(self): ++ self.assertTrue(hasattr(widget, 'BROKEN')) +""" -def test_description_declares_prompt_hint(): - args = env.tasks["coding-task"].manifest_entry()["args"]["properties"] - assert args["description"]["x-hud-hint"] == "prompt" +def widget_task(**overrides: object) -> Task: + return coding_task( + **{ + "description": "\nFix the widget.\n", + "test_command": PYTEST, + "test_patch": TEST_PATCH, + "test_path": "test_widget.py", + "base_ref": "origin/bug_baseline", + "fail_to_pass": ["test_widget.TestWidget.test_not_broken"], + "pass_to_pass": ["test_widget.TestWidget.test_existing_behavior"], + **overrides, + } + ) -@pytest.mark.asyncio -async def test_coding_task_uses_description_as_prompt(monkeypatch): - monkeypatch.setattr(coding_env, "_setup", AsyncMock()) - workspace = AsyncMock() - monkeypatch.setattr(coding_env, "workspace", workspace) - task = coding_env.coding_task.func( - description="\nFix the bug.\n", - test_command="pytest -q --junitxml={junit_path}", - test_patch="diff --git a/test_bug.py b/test_bug.py\n", - test_path="tests", - base_ref="main", +def git(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-c", f"safe.directory={repo}", *args], cwd=repo, check=False, capture_output=True, text=True ) - assert await anext(task) == "Fix the bug." - await task.aclose() - workspace.discard_sandbox.assert_awaited_once_with() + +@asynccontextmanager +async def served(row: Task): + """A fresh environment process serving ``row``, and a client connected to it.""" + async with SubprocessRuntime(ENV_SOURCE)(row) as runtime, connect(runtime) as client: + yield client + + +def leave_untouched(repo: Path) -> None: + del repo + + +def fix_the_widget(repo: Path) -> None: + (repo / "fix.py").write_text("BROKEN = False\n") + (repo / "widget.py").write_text("from fix import BROKEN\n") + + +def tamper_with_the_tests(repo: Path) -> None: + """Rewrite the hidden test's path and plant a git filter that would rewrite it on apply.""" + git(repo, "config", "filter.agent.clean", "cat").check_returncode() + (repo / ".gitattributes").write_text("* filter=agent\n") + (repo / "test_widget.py").write_text("def test_not_broken():\n assert True\n") + + +async def test_the_description_is_the_prompt(repo): + row = widget_task() + async with served(row) as client: + (template,) = await client.list_tasks() + started = await client.start_task(row.id, row.args) + + assert template["id"] == "coding-task" + assert template["args"]["properties"]["description"]["x-hud-hint"] == "prompt" + assert started["prompt"] == "Fix the widget." + assert (repo / "widget.py").read_text() == "BROKEN = True\n" + assert not (repo / "test_widget.py").exists() + + +@pytest.mark.parametrize( + ("overrides", "edit", "score"), + [ + pytest.param({}, leave_untouched, 0.5, id="untouched-baseline-keeps-regression-credit"), + pytest.param({}, fix_the_widget, 1.0, id="agent-fix"), + pytest.param({"base_ref": "origin/bug_golden"}, leave_untouched, 1.0, id="golden-ref"), + pytest.param({}, tamper_with_the_tests, 0.5, id="agent-test-changes-are-discarded"), + ], +) +async def test_hidden_tests_grade_the_agents_repository( + overrides: dict, edit: Callable[[Path], None], score: float, repo: Path +): + row = widget_task(**overrides) + async with served(row) as client: + await client.start_task(row.id, row.args) + edit(repo) + grade = await client.grade({"answer": "done"}) + + assert not grade["isError"], grade + assert grade["score"] == score + assert git(repo, "config", "--get", "filter.agent.clean").returncode == 1 + + +async def test_a_command_without_a_junit_report_is_a_grading_error(repo): + row = widget_task(test_command="true {junit_path}") + async with served(row) as client: + await client.start_task(row.id, row.args) + grade = await client.grade({"answer": "done"}) + + assert grade["isError"] is True + assert grade["content"] == "test command did not write JUnit XML" + assert grade["score"] == 0.0 + + +@pytest.mark.parametrize("test_path", ["tests/test_widget.py", "/test_widget.py", ".."]) +async def test_hidden_tests_must_sit_at_the_top_level(test_path: str, repo: Path): + row = widget_task(test_path=test_path) + async with served(row) as client: + with pytest.raises(HudProtocolError) as failure: + await client.start_task(row.id, row.args) + + assert "test_path must be a top-level file or directory" in failure.value.message + assert not repo.exists() or list(repo.iterdir()) == [] + + +async def test_a_new_task_starts_from_a_clean_baseline(repo): + row = widget_task() + async with served(row) as client: + await client.start_task(row.id, row.args) + fix_the_widget(repo) + await client.grade({"answer": "done"}) + await client.start_task(row.id, row.args) + + assert not (repo / "test_widget.py").exists() + assert not (repo / "fix.py").exists() + assert (repo / "widget.py").read_text() == "BROKEN = True\n" + + +@pytest.mark.parametrize("bundled", tasks.tasks, ids=lambda task: task.slug) +@pytest.mark.parametrize(("ref", "score"), [("baseline", 0.0), ("golden", 1.0)]) +async def test_bundled_tasks_score_zero_at_baseline_and_one_with_the_reference_fix( + bundled: Task, ref: str, score: float, repo: Path, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delenv("REPO_URL") + base_ref = bundled.args["base_ref"].replace("_baseline", f"_{ref}") + row = bundled.model_copy(update={"args": {**bundled.args, "base_ref": base_ref}}) + async with served(row) as client: + await client.start_task(row.id, row.args) + grade = await client.grade({"answer": "done"}) + + assert not grade["isError"], grade + assert grade["score"] == score diff --git a/environments/coding/tests/test_grader.py b/environments/coding/tests/test_grader.py index a9519f8db..70778b7ea 100644 --- a/environments/coding/tests/test_grader.py +++ b/environments/coding/tests/test_grader.py @@ -90,9 +90,9 @@ def test_parse_junit_rejects_malformed_report(tmp_path: Path): parse_junit(report) -@pytest.mark.asyncio(loop_scope="session") -async def test_junit_grader_reports_missing_output(grading_workspace): - result = await grade_tests(grading_workspace, "true {junit_path}") +@pytest.mark.sandbox +async def test_junit_grader_reports_missing_output(workspace): + result = await grade_tests(workspace, "true {junit_path}") assert result.value == 0.0 assert result.info is not None @@ -100,10 +100,10 @@ async def test_junit_grader_reports_missing_output(grading_workspace): assert result.info["exit_code"] == 0 -@pytest.mark.asyncio(loop_scope="session") -async def test_binary_junit_grader_rejects_passing_report_from_failed_command(grading_workspace): +@pytest.mark.sandbox +async def test_binary_junit_grader_rejects_passing_report_from_failed_command(workspace): result = await grade_tests( - grading_workspace, + workspace, command=( 'printf \'\' > {junit_path}; false' @@ -118,14 +118,14 @@ async def test_binary_junit_grader_rejects_passing_report_from_failed_command(gr assert "all_testcases" not in result.info -@pytest.mark.asyncio(loop_scope="session") -async def test_grading_runs_as_agent_after_session_termination(grading_workspace): - session = await grading_workspace.run(["/bin/bash", "-lc", "echo agent > agent.txt"]) +@pytest.mark.sandbox +async def test_grading_runs_as_agent_after_session_termination(workspace): + session = await workspace.run(["/bin/bash", "-lc", "echo agent > agent.txt"]) assert session.returncode == 0 - await grading_workspace.terminate_sessions() + await workspace.terminate_sessions() result = await grade_tests( - grading_workspace, + workspace, 'test "$(id -u)" = 1000 && test "$(cat agent.txt)" = agent && ' "grep -q localhost /etc/hosts && " 'printf \'\' > {junit_path}', @@ -136,10 +136,10 @@ async def test_grading_runs_as_agent_after_session_termination(grading_workspace assert result.info["exit_code"] == 0 -@pytest.mark.asyncio(loop_scope="session") -async def test_timed_out_grader_does_not_score_partial_report(grading_workspace): +@pytest.mark.sandbox +async def test_timed_out_grader_does_not_score_partial_report(workspace): result = await grade_tests( - grading_workspace, + workspace, 'printf \'\' > {junit_path}; sleep 30', timeout_seconds=1, ) diff --git a/environments/coding/tests/test_local_rollout.py b/environments/coding/tests/test_local_rollout.py deleted file mode 100644 index b444ce9c4..000000000 --- a/environments/coding/tests/test_local_rollout.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Hermetic end-to-end validation of repository setup and grading.""" - -import os -import shlex -import subprocess -import sys -from pathlib import Path - -import pytest -from hud import LocalRuntime, Run, connect -from hud.environment import workspace as workspace_lib - -import tasks - -PROJECT_ROOT = Path(__file__).resolve().parent.parent -PYTEST_COMMAND = ( - f"{shlex.quote(sys.executable)} -m pytest -q -c /dev/null --rootdir=. test_widget.py --junitxml={{junit_path}}" -) -TEST_PATCH = """diff --git a/test_widget.py b/test_widget.py -new file mode 100644 ---- /dev/null -+++ b/test_widget.py -@@ -0,0 +1,10 @@ -+import unittest -+import widget -+ -+ -+class TestWidget(unittest.TestCase): -+ def test_not_broken(self): -+ self.assertFalse(widget.BROKEN) -+ -+ def test_existing_behavior(self): -+ self.assertTrue(hasattr(widget, 'BROKEN')) -""" - -pytestmark = pytest.mark.asyncio(loop_scope="session") - - -def _git(cwd: Path, *args: str) -> None: - subprocess.run( - ["git", "-c", f"safe.directory={cwd}", "-c", "user.name=t", "-c", "user.email=t@t", *args], - cwd=cwd, - check=True, - capture_output=True, - ) - - -@pytest.fixture(scope="session") -def fixture_repo(tmp_path_factory) -> Path: - repo = tmp_path_factory.mktemp("fixture-repo") - (repo / "widget.py").write_text("BROKEN = True\n") - _git(repo, "init", "-q", "-b", "main") - _git(repo, "add", "-A") - _git(repo, "commit", "-qm", "buggy widget") - _git(repo, "branch", "bug_baseline") - - _git(repo, "checkout", "-qb", "bug_golden") - (repo / "widget.py").write_text("BROKEN = False\n") - _git(repo, "add", "-A") - _git(repo, "commit", "-qm", "reference fix") - return repo - - -def _coding_task(): - from env import coding_task - - return coding_task( - description="Fix the widget.", - test_command=f'test -n "${{HOME}}" && {PYTEST_COMMAND}', - test_patch=TEST_PATCH, - test_path="test_widget.py", - base_ref="origin/bug_baseline", - fail_to_pass=["test_widget.TestWidget.test_not_broken"], - pass_to_pass=["test_widget.TestWidget.test_existing_behavior"], - ) - - -async def _run_task(fixture_repo: Path, task, monkeypatch) -> float: - monkeypatch.setenv("REPO_URL", str(fixture_repo)) - monkeypatch.delenv("REPO_DIR", raising=False) - monkeypatch.delenv("BASELINE_DIR", raising=False) - runtime = LocalRuntime(str(PROJECT_ROOT / "env.py")) - async with runtime(task) as addr, connect(addr) as client: - async with Run(client, task.id, task.args) as run: - pass - assert not run.evaluation.get("isError"), run.evaluation - return run.reward - - -async def test_agent_fix_scores_one(fixture_repo, tmp_path, monkeypatch, grading_workspace): - import env as coding_env - - repo = grading_workspace.root - subprocess.run(["git", "clone", "-q", str(fixture_repo), str(repo)], check=True) - monkeypatch.setattr(coding_env, "REPO_DIR", repo) - monkeypatch.setattr(coding_env, "BASELINE_DIR", tmp_path / "baseline") - monkeypatch.setattr(coding_env, "REPO_SOURCE", str(fixture_repo)) - - task = coding_env.coding_task.func(**_coding_task().args) - await task.asend(None) - (repo / "fix.py").write_text("BROKEN = False\n") - (repo / "widget.py").write_text("from fix import BROKEN\n") - result = await task.asend("done") - - assert result.reward == 1.0, result - assert (repo / "fix.py").exists() - - -async def test_missing_junit_report_is_grading_error( - fixture_repo, - tmp_path, - monkeypatch, - grading_workspace, -): - import env as coding_env - - repo = grading_workspace.root - subprocess.run(["git", "clone", "-q", str(fixture_repo), str(repo)], check=True) - monkeypatch.setattr(coding_env, "REPO_DIR", repo) - monkeypatch.setattr(coding_env, "BASELINE_DIR", tmp_path / "baseline") - monkeypatch.setattr(coding_env, "REPO_SOURCE", str(fixture_repo)) - - task = coding_env.coding_task.func( - description="Fix the widget.", - test_command="true {junit_path}", - test_patch=TEST_PATCH, - test_path="test_widget.py", - base_ref="origin/bug_baseline", - ) - await task.asend(None) - result = await task.asend("done") - - assert result.reward == 0.0 - assert result.isError is True - assert result.content == "test command did not write JUnit XML" - assert result.info["exit_code"] == 0 - - -async def test_untouched_baseline_gets_regression_credit(fixture_repo, isolated_workspace, monkeypatch): - assert await _run_task(fixture_repo, _coding_task(), monkeypatch) == 0.5 - - -@pytest.mark.parametrize("task", tasks.tasks, ids=lambda task: task.slug) -@pytest.mark.parametrize("fixed", [False, True], ids=["baseline", "reference-fix"]) -async def test_bundled_tasks_grade_in_workspace(task, fixed, isolated_workspace, tmp_path, monkeypatch): - bundle = tmp_path / "flask.bundle" - bundle.write_bytes((PROJECT_ROOT / "flask.bundle").read_bytes()) - if fixed: - task = task.model_copy( - update={"args": {**task.args, "base_ref": f"origin/{task.slug.replace('-', '_')}_golden"}} - ) - assert await _run_task(bundle, task, monkeypatch) == float(fixed) - - -async def test_local_runtime_refuses_unisolated_non_root_workspace( - fixture_repo, - monkeypatch, -): - monkeypatch.setattr(workspace_lib, "usable_bwrap", lambda: None) - monkeypatch.setattr(os, "geteuid", lambda: 501) - os.environ["REPO_URL"] = str(fixture_repo) - - runtime = LocalRuntime(str(PROJECT_ROOT / "env.py")) - with pytest.raises(RuntimeError, match="isolation was required"): - async with runtime(_coding_task()): - pass - - -async def test_grading_discards_agent_git_config_and_test_changes( - fixture_repo, - tmp_path, - monkeypatch, - grading_workspace, -): - import env as coding_env - - repo = grading_workspace.root - subprocess.run(["git", "clone", "-q", str(fixture_repo), str(repo)], check=True) - monkeypatch.setattr(coding_env, "REPO_DIR", repo) - monkeypatch.setattr(coding_env, "BASELINE_DIR", tmp_path / "baseline") - monkeypatch.setattr(coding_env, "REPO_SOURCE", str(fixture_repo)) - - task = coding_env.coding_task.func( - description="Fix the widget.", - test_command=PYTEST_COMMAND, - test_patch=TEST_PATCH, - test_path="test_widget.py", - base_ref="origin/bug_baseline", - fail_to_pass=["test_widget.TestWidget.test_not_broken"], - pass_to_pass=["test_widget.TestWidget.test_existing_behavior"], - ) - await task.asend(None) - - _git(repo, "config", "filter.agent.clean", "cat") - (repo / ".gitattributes").write_text("* filter=agent\n") - (repo / "test_widget.py").write_text("def test_not_broken():\n assert True\n") - - result = await task.asend("done") - - assert result.reward == 0.5, result - config = subprocess.run( - ["git", "-c", f"safe.directory={repo}", "config", "--get", "filter.agent.clean"], - cwd=repo, - capture_output=True, - check=False, - ) - assert config.returncode == 1 - - -async def test_setup_removes_files_from_the_previous_hidden_patch( - fixture_repo, - tmp_path, - monkeypatch, -): - import env as coding_env - - repo = tmp_path / "repo" - subprocess.run(["git", "clone", "-q", str(fixture_repo), str(repo)], check=True) - monkeypatch.setattr(coding_env, "REPO_DIR", repo) - monkeypatch.setattr(coding_env, "BASELINE_DIR", tmp_path / "baseline") - monkeypatch.setattr(coding_env, "REPO_SOURCE", str(fixture_repo)) - - await coding_env._setup("origin/bug_baseline") - subprocess.run( - ["git", "apply", "-"], - cwd=repo, - input=TEST_PATCH, - text=True, - check=True, - ) - assert (repo / "test_widget.py").exists() - - await coding_env._setup("origin/bug_baseline") - - assert not (repo / "test_widget.py").exists() diff --git a/environments/cua/env.py b/environments/cua/env.py index 3405d2d5a..cccb0bb60 100644 --- a/environments/cua/env.py +++ b/environments/cua/env.py @@ -4,6 +4,7 @@ # param crashes the sync/deploy manifest path (TypeAdapter on a string forward-ref). Keep # annotations as real objects. import asyncio +import os from collections.abc import Awaitable from typing import Annotated, Any @@ -17,7 +18,8 @@ _task_started = False _HOST = "127.0.0.1" -_VNC_PORT = 5900 +# supervisord runs x11vnc on 5900; tests point this at their own VNC listener. +_VNC_PORT = int(os.environ.get("VNC_PORT", "5900")) async def _listening(host: str, port: int, timeout: float = 30.0) -> None: @@ -38,7 +40,7 @@ async def _listening(host: str, port: int, timeout: float = 30.0) -> None: @env.initialize async def _up() -> None: await _listening(_HOST, _VNC_PORT) - env.add_capability(Capability.rfb(name="screen", url=f"rfb://{_HOST}", display=0)) + env.add_capability(Capability.rfb(name="screen", url=f"rfb://{_HOST}:{_VNC_PORT}")) @env.template() diff --git a/environments/cua/tests/conftest.py b/environments/cua/tests/conftest.py index 84d71f1a7..4409a269e 100644 --- a/environments/cua/tests/conftest.py +++ b/environments/cua/tests/conftest.py @@ -1,8 +1,73 @@ -"""Test fixtures for the CUA environment grader logic (offline, no desktop).""" +"""Serve the environment the way HUD does, against local stand-ins for the desktop and the judge. +Nothing here needs a real desktop or a HUD account: a TCP listener stands in for +the VNC server, and a small HTTP server stands in for the inference gateway the +LLM judge calls. The environment runs in its own process, configured only +through environment variables, exactly as ``hud eval`` serves it. +""" + +from __future__ import annotations + +import asyncio +import json import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -ROOT = str(Path(__file__).parent.parent) -if ROOT not in sys.path: - sys.path.insert(0, ROOT) +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + +class Judge: + """The inference gateway, judging every criterion MET. ``requests`` records what it was sent.""" + + def __init__(self) -> None: + self.requests: list[tuple[str, dict]] = [] + judge = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + judge.requests.append((self.headers["Authorization"], body)) + verdict = json.dumps({"criterion_status": "MET", "explanation": "stand-in judge"}) + reply = json.dumps({"choices": [{"message": {"content": verdict}}]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(reply))) + self.end_headers() + self.wfile.write(reply) + + def log_message(self, format: str, *args: object) -> None: + del format, args + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.server.server_port}" + + +@pytest.fixture(autouse=True) +def judge(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + """Point the environment at the stand-in judge, with no HUD key and no user config.""" + stand_in = Judge() + thread = threading.Thread(target=stand_in.server.serve_forever, daemon=True) + thread.start() + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("HUD_API_KEY", "") + monkeypatch.setenv("HUD_GATEWAY_URL", stand_in.url) + monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "0") + yield stand_in + stand_in.server.shutdown() + thread.join() + + +@pytest.fixture(autouse=True) +async def desktop(monkeypatch: pytest.MonkeyPatch): + """A listener on the environment's VNC port, so it starts without a real desktop.""" + server = await asyncio.start_server(lambda reader, writer: writer.close(), "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + monkeypatch.setenv("VNC_PORT", str(port)) + async with server: + yield port diff --git a/environments/cua/tests/test_env.py b/environments/cua/tests/test_env.py index 618eedcc2..cf92db1a2 100644 --- a/environments/cua/tests/test_env.py +++ b/environments/cua/tests/test_env.py @@ -1,138 +1,115 @@ -"""Offline tests for the CUA environment grader composition. +"""The CUA environment, served the way HUD serves it and driven over its control channel. -These do NOT touch the virtual desktop (rfb is Linux-only); they drive the @env.template -generator directly and exercise its configuration and grader composition with deterministic shell -commands. The end-to-end desktop run is a `hud eval ... --runtime hud` rollout. +Each test starts a fresh environment process, starts a task with the arguments a +task row carries, and grades an answer: what a rollout does, without an agent. """ -from unittest.mock import AsyncMock +from __future__ import annotations + +from contextlib import asynccontextmanager +from pathlib import Path import pytest +from hud import SubprocessRuntime, Task, connect +from hud.clients import HudProtocolError + +from env import cua_task + +ENV_SOURCE = Path(__file__).resolve().parent.parent / "env.py" +PASS = {"name": "ok", "command": "true", "weight": 1.0} + + +@asynccontextmanager +async def served(row: Task): + """A fresh environment process serving ``row``, and a client connected to it.""" + async with SubprocessRuntime(ENV_SOURCE)(row) as runtime, connect(runtime) as client: + yield client + -import env as M -import tasks - -GEN = M.cua_task.func - - -@pytest.fixture(autouse=True) -def fresh_substrate(monkeypatch): - monkeypatch.setattr(M, "_task_started", False) - - -class TestGrading: - async def test_missing_graders_fails_before_prompt(self): - gen = GEN(prompt="p") - with pytest.raises(ValueError, match="at least one grader"): - await gen.asend(None) - - async def test_bash_check_passes(self): - gen = GEN(prompt="p", bash_checks=[{"name": "ok", "command": "true", "weight": 1.0}]) - await gen.asend(None) - assert (await gen.asend("a")).reward == 1.0 - - async def test_bash_check_fails(self): - gen = GEN(prompt="p", bash_checks=[{"name": "no", "command": "false", "weight": 1.0}]) - await gen.asend(None) - assert (await gen.asend("a")).reward == 0.0 - - async def test_negative_bash_weight_fails_before_prompt(self): - gen = GEN(prompt="p", bash_checks=[{"name": "no", "command": "false", "weight": -1.0}]) - with pytest.raises(ValueError, match="nonnegative"): - await gen.asend(None) - - @pytest.mark.parametrize("hud_api_key", [None, "task-key"]) - async def test_no_runtime_key_judge_fails_before_prompt(self, monkeypatch, hud_api_key): - monkeypatch.setattr(M.settings, "api_key", "", raising=False) - gen = GEN(prompt="p", grading_criteria=["anything"], hud_api_key=hud_api_key) - with pytest.raises(RuntimeError, match="HUD_API_KEY"): - await gen.asend(None) - - async def test_weights_normalize_across_bash_and_judge(self, monkeypatch): - monkeypatch.setattr(M.settings, "api_key", "key", raising=False) - monkeypatch.setattr( - M.LLMJudgeGrader, - "grade", - AsyncMock(return_value=M.SubScore(name="llm_judge", value=1.0, weight=0.5)), - ) - gen = GEN( - prompt="p", - bash_checks=[{"name": "ok", "command": "true", "weight": 0.3}], - grading_criteria=["x"], - ) - await gen.asend(None) - result = await gen.asend("answer") - assert result.reward == 1.0 - weights = {subscore.name: subscore.weight for subscore in result.subscores} - assert weights == {"ok": 0.5, "llm_judge": 0.5} - - async def test_second_task_requires_fresh_substrate(self): - first = GEN(prompt="first", bash_checks=[{"name": "ok", "command": "true"}]) - await first.asend(None) - - second = GEN(prompt="second", bash_checks=[{"name": "ok", "command": "true"}]) - with pytest.raises(RuntimeError, match="one task per substrate"): - await second.asend(None) - - await first.aclose() - - async def test_named_bash_subscores_preserved(self): - gen = GEN( - prompt="p", - bash_checks=[ - {"name": "alpha", "command": "true", "weight": 0.4}, - {"name": "beta", "command": "true", "weight": 0.6}, - ], - ) - await gen.asend(None) - result = await gen.asend("a") - names = {s.name for s in result.subscores} - assert {"alpha", "beta"} <= names - assert result.reward == 1.0 - - -def test_create_document_requires_exact_contents(): - command = tasks._create_document.args["bash_checks"][1]["command"] - assert command == "cmp -s /home/ubuntu/Desktop/hello.txt <(printf 'Hello from HUD!\\n')" - - -def test_open_website_requires_wikipedia_page(): - check = tasks._open_website.args["bash_checks"][0] - assert check == { - "name": "visited_wikipedia", - "command": ( - "curl -fsS http://127.0.0.1:9222/json/list " - "| jq -e 'any(.[]; .url == \"https://www.wikipedia.org/\")' >/dev/null" +async def test_the_manifest_publishes_the_screen_and_a_prompt_box(desktop): + row = cua_task(prompt="p", bash_checks=[PASS]) + async with served(row) as client: + (template,) = await client.list_tasks() + screens = [binding.url for binding in client.manifest.bindings if binding.name == "screen"] + + assert screens == [f"rfb://127.0.0.1:{desktop}"] + arguments = template["args"] + assert arguments["properties"]["prompt"]["x-hud-hint"] == "prompt" + assert "hud_api_key" in arguments["properties"] + assert arguments["required"] == ["prompt"] + + +@pytest.mark.parametrize( + ("arguments", "message"), + [ + pytest.param({}, "at least one grader", id="no-graders"), + pytest.param( + {"bash_checks": [{"name": "no", "command": "false", "weight": -1.0}]}, + "nonnegative", + id="negative-weight", + ), + pytest.param({"grading_criteria": ["Anything."]}, "HUD_API_KEY is required", id="judge-without-key"), + pytest.param( + {"grading_criteria": ["Anything."], "hud_api_key": "task-key"}, + "HUD_API_KEY is required", + id="a-task-key-is-not-the-runtime-key", + ), + ], +) +async def test_a_misconfigured_task_fails_before_the_agent_sees_a_prompt(arguments, message): + row = cua_task(prompt="p", **arguments) + async with served(row) as client: + with pytest.raises(HudProtocolError) as failure: + await client.start_task(row.id, row.args) + + assert message in failure.value.message + + +@pytest.mark.parametrize( + ("arguments", "score", "weights"), + [ + pytest.param({"bash_checks": [PASS]}, 1.0, {"ok": 1.0}, id="passing-check"), + pytest.param({"bash_checks": [{"name": "no", "command": "false"}]}, 0.0, {"no": 1.0}, id="failing-check"), + pytest.param( + { + "bash_checks": [ + {"name": "alpha", "command": "true", "weight": 0.4}, + {"name": "beta", "command": "false", "weight": 0.6}, + ] + }, + 0.4, + {"alpha": 0.4, "beta": 0.6}, + id="weighted-checks", + ), + pytest.param( + { + "bash_checks": [{"name": "ok", "command": "true", "weight": 0.3}], + "grading_criteria": ["Says hello."], + }, + 1.0, + {"ok": 0.5, "llm_judge": 0.5}, + id="checks-and-judge-split-the-score", ), - "weight": 1.0, - } - - -def test_shannon_research_requires_exact_file_and_open_pages(): - assert tasks._shannon_research.args["bash_checks"] == [ - { - "name": "file_contents", - "command": ( - "cmp -s /home/ubuntu/Desktop/shannon.txt " - "<(printf 'born: 1916\\nphd: Massachusetts Institute of Technology\\n" - "city: Cambridge, Massachusetts\\n')" - ), - "weight": 1.0, - }, - { - "name": "research_pages_open", - "command": ( - "curl -fsS http://127.0.0.1:9222/json/list | jq -e " - "--arg shannon https://en.wikipedia.org/wiki/Claude_Shannon " - "--arg mit https://en.wikipedia.org/wiki/Massachusetts_Institute_of_Technology " - "'map(.url) | contains([$shannon, $mit])' >/dev/null" - ), - "weight": 1.0, - }, - ] - - -def test_platform_args_declared(): - args = M.cua_task.manifest_entry()["args"]["properties"] - assert args["prompt"]["x-hud-hint"] == "prompt" - assert "hud_api_key" in args + ], +) +async def test_the_grade_weighs_named_checks_and_the_judge(arguments, score, weights, monkeypatch, judge): + monkeypatch.setenv("HUD_API_KEY", "runtime-key") + row = cua_task(prompt="Say hello.", hud_api_key="task-key", **arguments) + async with served(row) as client: + started = await client.start_task(row.id, row.args) + grade = await client.grade({"answer": "hello"}) + + assert started["prompt"].endswith("Say hello.") + assert grade["score"] == pytest.approx(score) + assert {sub["name"]: sub["weight"] for sub in grade["subscores"]} == pytest.approx(weights) + assert {authorization for authorization, _ in judge.requests} <= {"Bearer runtime-key"} + + +async def test_a_desktop_serves_one_task(): + row = cua_task(prompt="first", bash_checks=[PASS]) + async with served(row) as client: + await client.start_task(row.id, row.args) + with pytest.raises(HudProtocolError) as failure: + await client.start_task(row.id, {**row.args, "prompt": "second"}) + + assert "one task per substrate" in failure.value.message From 8381724b4b71762b8abefdaac20dd27cf637afab Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 074/248] test: run every hud init template and cookbook from a user's starting point test_templates.py discovers the templates under environments/ and takes each the way a user does: hud init, install this checkout's SDK into the generated project, hud task list, the template's own ruff and pytest, and one capped hud eval whose step spans must show a clean setup-then-evaluate lifecycle with a prompt and an in-range score. The default lane drives it with a scripted model through the fake gateway on every PR; the live lane swaps in a cheap real model; the hosted lane ports example_lifecycle.py: deploy, run on the HUD runtime, and read the trace back from the platform. coding serves an isolated workspace, so it joins the sandbox lane with its environment installed under /usr, where the image puts it and the only prefix the sandbox shows the grader. test_cookbooks.py, marked e2e for the nightly run, checks that every hud name a cookbook imports exists, that every cookbook module imports when its third-party packages are installed, and that connect4-selfplay and daytona-rl complete the local runs their READMEs describe. The a2a-chat rows fail: chat_env.py imports EvaluationResult from hud.agents.types. --- tests/e2e/conftest.py | 28 ++++++ tests/e2e/lifecycle.py | 25 ++++- tests/e2e/projects.py | 164 +++++++++++++++++++++++++++++++ tests/e2e/test_cookbooks.py | 124 +++++++++++++++++++++++ tests/e2e/test_oracles.py | 3 +- tests/e2e/test_templates.py | 190 ++++++++++++++++++++++++++++++++++++ 6 files changed, 531 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/conftest.py create mode 100644 tests/e2e/projects.py create mode 100644 tests/e2e/test_cookbooks.py create mode 100644 tests/e2e/test_templates.py diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 000000000..99f1f8046 --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,28 @@ +"""Fixtures for scenarios that generate projects and install this checkout's SDK into them.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from tests.conftest import REAL_ENVIRONMENT + +from .projects import ROOT, Uv + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture(scope="session") +def uv() -> Uv: + return Uv(REAL_ENVIRONMENT) + + +@pytest.fixture(scope="session") +def sdk_wheel(uv: Uv, tmp_path_factory: pytest.TempPathFactory) -> Path: + """This checkout's SDK as a wheel, built once per test worker.""" + out = tmp_path_factory.mktemp("sdk-wheel") + uv("build", "--wheel", "--no-create-gitignore", "--out-dir", str(out), str(ROOT), cwd=out) + (wheel,) = out.glob("hud-*.whl") + return wheel diff --git a/tests/e2e/lifecycle.py b/tests/e2e/lifecycle.py index bbcccbe85..6ac08a964 100644 --- a/tests/e2e/lifecycle.py +++ b/tests/e2e/lifecycle.py @@ -1,8 +1,15 @@ -"""What a cleanly completed task lifecycle looks like in a run's step spans.""" +"""Capped rollouts: the scripted answers they get and what a clean lifecycle leaves in the spans.""" from __future__ import annotations -from typing import Any +import json +from typing import TYPE_CHECKING, Any + +from tests.harness import say + +if TYPE_CHECKING: + from tests.harness import ModelRequest, Turn + from tests.harness.cli import Result # Tool steps may fail without failing the run: a real model can call a tool badly. LIFECYCLE_SOURCES = {"task", "agent", "system"} @@ -27,3 +34,17 @@ def assert_clean_lifecycle(steps: list[dict[str, Any]]) -> tuple[dict[str, Any], assert isinstance(grade["score"], int | float), grade assert 0 <= grade["score"] <= 1, grade return setup, grade + + +def scripted(request: ModelRequest) -> Turn: + """The agent answers at once; the LLM judge finds every criterion met.""" + if request.protocol == "chat": + return say(json.dumps({"criterion_status": "MET", "explanation": "scripted judge"})) + return say("done") + + +def evaluated(result: Result) -> dict[str, Any]: + assert result.exit_code == 0, result + assert result.json["error_count"] == 0, result + (run,) = result.json["runs"] + return run diff --git a/tests/e2e/projects.py b/tests/e2e/projects.py new file mode 100644 index 000000000..540c52ea8 --- /dev/null +++ b/tests/e2e/projects.py @@ -0,0 +1,164 @@ +"""Generated environment projects: ``hud init`` output with the SDK from this checkout installed.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from tests.harness import Hud + +from .lifecycle import assert_clean_lifecycle + +ROOT = Path(__file__).resolve().parents[2] +TEMPLATES = sorted( + path.name for path in (ROOT / "environments").iterdir() if (path / "env.py").is_file() +) +ROWS = "ci-tasks.json" +CHECKS = ( + ("-m", "ruff", "format", ".", "--check"), + ("-m", "ruff", "check", "."), + ("-m", "pytest", "-q", "-p", "no:cacheprovider"), +) +EXPORT_TASKS = ( + "from hud.eval import Taskset; Taskset.from_file('tasks.py').to_file('.hud/tasks.json')" +) + + +class Uv: + """``uv`` with the developer's cache and Python installs, whatever ``HOME`` a test set.""" + + def __init__(self, real_environment: dict[str, str]) -> None: + executable = shutil.which("uv", path=real_environment.get("PATH")) + if executable is None: + raise RuntimeError("the template scenarios need uv on PATH") + self.executable = executable + self.env = { + "UV_CACHE_DIR": self.ask(real_environment, "cache", "dir"), + "UV_PYTHON_INSTALL_DIR": self.ask(real_environment, "python", "dir"), + } + + def ask(self, environment: dict[str, str], *args: str) -> str: + return subprocess.run( + [self.executable, *args], env=environment, capture_output=True, text=True, check=True + ).stdout.strip() + + def __call__(self, *args: str, cwd: Path, env: dict[str, str] | None = None) -> None: + completed = subprocess.run( + [self.executable, *args], + cwd=cwd, + env={**os.environ, **self.env, **(env or {})}, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, f"uv {' '.join(args)}\n{completed.stderr}" + + +def system_python() -> str: + """A CPython under /usr: inside the workspace sandbox, only /usr is visible to the grader.""" + for version in ("3.12", "3.11"): + if found := shutil.which(f"python{version}", path="/usr/local/bin:/usr/bin"): + return found + raise RuntimeError("needs Python 3.11 or 3.12 under /usr/local/bin or /usr/bin") + + +@dataclass(frozen=True) +class Project: + """A generated project and its own ``hud``, run from the project's environment.""" + + root: Path + python: Path + + @property + def hud(self) -> Hud: + return Hud(self.root, python=self.python) + + def run(self, *args: str, timeout: float = 600) -> subprocess.CompletedProcess[str]: + """Run the project's Python with ``args``, the way ``uv run python`` would.""" + return subprocess.run( + [str(self.python), *args], + cwd=self.root, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + def check(self) -> None: + """The template's own lint and tests, as a user runs them before deploying.""" + for args in CHECKS: + completed = self.run(*args) + assert completed.returncode == 0, ( + f"{' '.join(args)}\n{completed.stdout}\n{completed.stderr}" + ) + + def bundle_sdk_into_image(self) -> None: + """Copy the bundled SDK into the image ahead of its ``uv sync``, for a deploy to build.""" + dockerfile = self.root / "Dockerfile.hud" + lines = dockerfile.read_text().splitlines(keepends=True) + sync = next( + index + for index, line in enumerate(lines) + if line.startswith("RUN") and "uv sync" in line + ) + lines.insert(sync, "COPY .hud-sdk .hud-sdk\n") + dockerfile.write_text("".join(lines)) + with (self.root / ".dockerignore").open("a") as ignored: + ignored.write("!.hud-sdk/\n!.hud-sdk/**\n") + + def write_rows(self, data_file: str | None) -> Path: + """Write the first task as a JSON row beside the env source, attaching ``data_file``.""" + exported = self.run("-c", EXPORT_TASKS) + assert exported.returncode == 0, exported.stderr + row = json.loads((self.root / ".hud" / "tasks.json").read_text())[0] + if data_file is not None: + row["args"]["attachments"] = [{"file_id": data_file, "path": "notes.txt"}] + row["args"]["prompt"] = "Read files/notes.txt first. " + row["args"]["prompt"] + path = self.root / ROWS + path.write_text(json.dumps([row], indent=2)) + return path + + +def generate( + hud: Hud, + template: str, + directory: Path, + *, + wheel: Path, + uv: Uv, + environment: Path | None = None, +) -> Project: + """``hud init`` a project from ``template``, then install this checkout's SDK into it. + + The wheel is copied into the project, as a deploy needs it inside the build + context. ``environment`` installs the project's Python somewhere other than + ``.venv``, the way an image does. + """ + created = hud("init", directory.name, "--dir", str(directory.parent), "--template", template) + assert created.exit_code == 0, created + bundled = directory / ".hud-sdk" / wheel.name + bundled.parent.mkdir() + shutil.copy(wheel, bundled) + uv("add", "--no-sync", f".hud-sdk/{wheel.name}", cwd=directory) + placement = {} + if environment is not None: + placement = {"UV_PROJECT_ENVIRONMENT": str(environment), "UV_PYTHON": system_python()} + uv("sync", "--all-extras", cwd=directory, env=placement) + python = (environment or directory / ".venv") / "bin" / "python" + return Project(directory, python) + + +def assert_lifecycle( + run: dict[str, Any], recorded: list[dict[str, Any]], data_file: str | None +) -> dict[str, Any]: + """A clean single-run lifecycle; a declared data file reaches the start frame.""" + assert run["is_error"] is False, run + setup, grade = assert_clean_lifecycle(recorded) + if data_file is not None: + assert setup["data_files"] == [{"path": "files/notes.txt", "file_id": data_file}] + return grade diff --git a/tests/e2e/test_cookbooks.py b/tests/e2e/test_cookbooks.py new file mode 100644 index 000000000..03bb981d5 --- /dev/null +++ b/tests/e2e/test_cookbooks.py @@ -0,0 +1,124 @@ +"""The cookbooks against this checkout's SDK: every module imports, and the local runs complete. + +Cookbooks live outside the package and its tests, so an SDK change can break +them silently: a moved import path, a renamed export. Every ``hud`` name a +cookbook imports must exist; every cookbook module must import, unless a +third-party package it needs is not installed; and the cookbooks whose README +runs them with no outside account complete one capped rollout with a scripted +model. Nightly, through the ``e2e`` marker. +""" + +from __future__ import annotations + +import ast +import importlib +import importlib.util +import subprocess +import sys +from typing import TYPE_CHECKING + +import pytest + +from tests.harness import steps + +from .lifecycle import assert_clean_lifecycle, evaluated, scripted +from .projects import ROOT + +if TYPE_CHECKING: + from pathlib import Path + + from tests.harness import Hud, HudEnv, Models + +COOKBOOKS = ROOT / "cookbooks" +MODULES = sorted(path for path in COOKBOOKS.rglob("*.py") if ".venv" not in path.parts) +# The README commands that need no outside account: (cookbook, task source). +LOCAL_RUNS = [("connect4-selfplay", "env.py"), ("daytona-rl", "tasks.py")] + +pytestmark = pytest.mark.e2e + + +def module_id(path: Path) -> str: + return str(path.relative_to(COOKBOOKS)) + + +def cookbook_of(path: Path) -> Path: + return COOKBOOKS / path.relative_to(COOKBOOKS).parts[0] + + +def search_path(path: Path) -> list[Path]: + """Where a cookbook's scripts find its sibling modules: their directory and the cookbook's.""" + return list(dict.fromkeys([path.parent, cookbook_of(path)])) + + +def imports(path: Path) -> list[tuple[str, str | None]]: + """``(module, name)`` for every import in ``path``; ``name`` is None for ``import module``.""" + found: list[tuple[str, str | None]] = [] + for node in ast.walk(ast.parse(path.read_text("utf-8"))): + if isinstance(node, ast.Import): + found.extend((alias.name, None) for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + found.extend((node.module, alias.name) for alias in node.names) + return found + + +def resolves(module: str, name: str | None) -> bool: + """``module`` imports and has ``name``, as an attribute or a submodule.""" + imported = importlib.import_module(module) + if name is None or hasattr(imported, name): + return True + return ( + hasattr(imported, "__path__") and importlib.util.find_spec(f"{module}.{name}") is not None + ) + + +@pytest.mark.parametrize("path", MODULES, ids=module_id) +def test_every_hud_name_a_cookbook_imports_exists(path: Path) -> None: + hud_imports = [ + (module, name) for module, name in imports(path) if module.split(".")[0] == "hud" + ] + + missing = [f"{module}:{name}" for module, name in hud_imports if not resolves(module, name)] + + assert missing == [] + + +@pytest.mark.parametrize("path", MODULES, ids=module_id) +def test_a_cookbook_module_imports(path: Path) -> None: + local = {sibling.stem for directory in search_path(path) for sibling in directory.glob("*.py")} + third_party = { + module.split(".")[0] + for module, _ in imports(path) + if module.split(".")[0] not in {*sys.stdlib_module_names, "hud", "__future__", *local} + } + if missing := sorted(name for name in third_party if importlib.util.find_spec(name) is None): + pytest.skip(f"needs {', '.join(missing)}") + program = ( + f"import sys; sys.path[:0] = {[str(p) for p in search_path(path)]!r}; import {path.stem}" + ) + + imported = subprocess.run( + [sys.executable, "-c", program], + cwd=cookbook_of(path), + capture_output=True, + text=True, + timeout=120, + check=False, + ) + + assert imported.returncode == 0, imported.stderr + + +@pytest.mark.parametrize(("cookbook", "source"), LOCAL_RUNS, ids=[run[0] for run in LOCAL_RUNS]) +def test_a_cookbook_completes_its_local_run( + cookbook: str, source: str, models: Models, hud_env: HudEnv, hud: Hud +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(scripted) + + result = hud( + "eval", source, "claude", "--max-steps", "3", "-y", "--json", cwd=COOKBOOKS / cookbook + ) + + run = evaluated(result) + assert run["is_error"] is False + assert_clean_lifecycle(steps(run["trace_id"])) diff --git a/tests/e2e/test_oracles.py b/tests/e2e/test_oracles.py index 61e1851e0..524b31754 100644 --- a/tests/e2e/test_oracles.py +++ b/tests/e2e/test_oracles.py @@ -10,7 +10,7 @@ import re from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest @@ -80,6 +80,7 @@ async def test_an_oracle_scores_one_only_through_its_whole_path( models.respond(oracle.answer) source = ORACLES / f"{oracle.module}.py" + run: dict[str, Any] if entry == "hud-eval": result = hud("eval", str(source), *EVAL_ARGS) assert result.exit_code == 0, result diff --git a/tests/e2e/test_templates.py b/tests/e2e/test_templates.py new file mode 100644 index 000000000..9eaf5ba05 --- /dev/null +++ b/tests/e2e/test_templates.py @@ -0,0 +1,190 @@ +"""Every ``hud init`` template, from ``hud init`` to a graded rollout, the way a user takes it. + +Each case generates a project from a template, installs the SDK from this +checkout into it, lists its tasks, passes the template's own lint and tests, +and runs one capped rollout of its first task with ``hud eval``. Three lanes run +that lifecycle: + +- default: a scripted model behind the fake gateway, on every PR; +- ``live``: a cheap real model through the HUD gateway; +- ``hosted``: deployed to HUD and run on its runtime, with the trace read back + from the platform. The pre-release workflow runs it one template per job. + +A template that serves an isolated workspace (coding) needs the sandbox lane +locally; its environment is installed under /usr, where its image puts it. +""" + +from __future__ import annotations + +import asyncio +import json +import shutil +import time +import uuid +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from tests.conftest import REAL_ENVIRONMENT +from tests.harness import Reply, fake_screen, steps + +from .lifecycle import evaluated, scripted +from .projects import ROWS, TEMPLATES, assert_lifecycle, generate + +if TYPE_CHECKING: + from collections.abc import Iterator + + from tests.harness import FakeServices, Hud, HudEnv, Models + from tests.harness.cli import Result + + from .projects import Uv + +SANDBOXED = {"coding"} +TAKES_DATA_FILES = {"argument-hints"} +EVAL = ("eval", ROWS, "claude", "--max-steps", "10", "-y", "--json") +LIVE_MODEL = "claude-haiku-4-5" +HOSTED_MODEL = "claude-sonnet-4-6" +REAL_MODEL = ("--gateway", "--config", "max_tokens=2048") + + +def lane(*marks: pytest.MarkDecorator) -> list[Any]: + """One case per template; templates serving an isolated workspace join the sandbox lane.""" + return [ + pytest.param( + name, + id=name, + marks=[*marks, *([pytest.mark.e2e, pytest.mark.sandbox] if name in SANDBOXED else [])], + ) + for name in TEMPLATES + ] + + +@pytest.fixture +def environment(request: pytest.FixtureRequest) -> Iterator[Path | None]: + """Where the project's Python goes: under /usr for a sandboxed template, else ``.venv``.""" + if request.node.callspec.params["template"] not in SANDBOXED: + yield None + return + location = Path("/usr/local/share/hud-e2e") / uuid.uuid4().hex + yield location + shutil.rmtree(location, ignore_errors=True) + + +@pytest.mark.timeout(900) +@pytest.mark.parametrize("template", lane()) +async def test_a_template_runs_from_init_to_a_graded_rollout( + template: str, + environment: Path | None, + services: FakeServices, + models: Models, + hud_env: HudEnv, + hud: Hud, + uv: Uv, + sdk_wheel: Path, + tmp_path: Path, +) -> None: + hud_env.set(HUD_API_KEY="k") + models.respond(scripted) + data_file = "ci-notes" if template in TAKES_DATA_FILES else None + services.route("api", "GET", "/v2/data/ci-notes", json={"filename": "notes.txt"}) + services.route( + "api", + "GET", + "/v2/data/ci-notes/download", + json={"url": f"{services.url('api')}/files/notes"}, + ) + services.route("api", "GET", "/files/notes", Reply(body=b"2, 3 and 5 are prime.")) + + project = generate( + hud, template, tmp_path / f"ci-{template}", wheel=sdk_wheel, uv=uv, environment=environment + ) + listed = project.hud("task", "list", "--json") + assert listed.exit_code == 0, listed + assert listed.json, "the template lists no tasks" + if (project.root / "tests").is_dir(): + project.check() + project.write_rows(data_file) + async with fake_screen() as screen: + hud_env.set(VNC_PORT=str(screen.port)) + run = evaluated(await asyncio.to_thread(project.hud, *EVAL, timeout=600)) + + assert_lifecycle(run, steps(run["trace_id"]), data_file) + + +@pytest.mark.e2e +@pytest.mark.live +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("template", lane()) +async def test_a_template_completes_a_rollout_with_a_real_model( + template: str, + environment: Path | None, + live: HudEnv, + hud: Hud, + uv: Uv, + sdk_wheel: Path, + tmp_path: Path, +) -> None: + data_file = ( + REAL_ENVIRONMENT.get("HUD_CI_DATA_FILE_ID") if template in TAKES_DATA_FILES else None + ) + project = generate( + hud, template, tmp_path / f"ci-{template}", wheel=sdk_wheel, uv=uv, environment=environment + ) + project.write_rows(data_file) + async with fake_screen() as screen: + live.set(VNC_PORT=str(screen.port)) + result = await asyncio.to_thread( + project.hud, *EVAL, *REAL_MODEL, "--model", LIVE_MODEL, timeout=900 + ) + + run = evaluated(result) + assert_lifecycle(run, steps(run["trace_id"]), data_file) + + +@pytest.mark.e2e +@pytest.mark.live +@pytest.mark.hosted +@pytest.mark.timeout(2700) +@pytest.mark.parametrize("template", TEMPLATES) +def test_a_deployed_template_runs_on_hud_and_its_trace_reads_back( + template: str, live: HudEnv, hud: Hud, uv: Uv, sdk_wheel: Path, tmp_path: Path +) -> None: + data_file = None + if template in TAKES_DATA_FILES: + data_file = REAL_ENVIRONMENT.get("HUD_CI_DATA_FILE_ID") + assert data_file, "HUD_CI_DATA_FILE_ID must name the shared CI attachment" + traces = Path(REAL_ENVIRONMENT.get("HUD_E2E_TRACE_DIR") or tmp_path / "traces") / template + # A stable directory name keeps every run deploying to the same environment. + project = generate(hud, template, tmp_path / f"ci-{template}", wheel=sdk_wheel, uv=uv) + project.bundle_sdk_into_image() + deployed = project.hud("deploy", "--no-env", timeout=1800) + assert deployed.exit_code == 0, deployed + project.write_rows(data_file) + live.set(HUD_TELEMETRY_ENABLED="1", HUD_TELEMETRY_LOCAL_DIR=str(traces)) + + run = evaluated( + project.hud(*EVAL, *REAL_MODEL, "--model", HOSTED_MODEL, "--runtime", "hud", timeout=900) + ) + + assert_lifecycle(run, steps(run["trace_id"], traces), data_file) + # Without a local span directory, `hud trace get` reads the trace from the platform. + live.set(HUD_TELEMETRY_LOCAL_DIR=None) + events = read_back(project.hud, run["trace_id"]) + (traces / "remote-events.json").write_text(json.dumps(events, indent=2)) + + +def read_back(hud: Hud, trace_id: str, attempts: int = 12, interval: float = 5.0) -> list[Any]: + """The platform's events for ``trace_id``, polled until they include the agent's messages.""" + for _ in range(attempts - 1): + result = hud("trace", "get", trace_id, "--json") + if agent_spoke(result): + return result.json + time.sleep(interval) + result = hud("trace", "get", trace_id, "--json") + assert agent_spoke(result), f"no agent events on the platform after {attempts} reads:\n{result}" + return result.json + + +def agent_spoke(result: Result) -> bool: + return result.exit_code == 0 and any(event["kind"] == "agent_message" for event in result.json) From 78472e5f1aaacdfb423f1514d932a0164ee4e318 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 075/248] fix: import EvaluationResult from hud.graders in the a2a-chat cookbook hud.agents.types does not export EvaluationResult, so chat_env.py failed to import and server.py, which serves it, could not start. The cookbook import rows in tests/e2e/test_cookbooks.py fail without this change. --- cookbooks/a2a-chat/chat_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/a2a-chat/chat_env.py b/cookbooks/a2a-chat/chat_env.py index de5e6b277..b473a9ad1 100644 --- a/cookbooks/a2a-chat/chat_env.py +++ b/cookbooks/a2a-chat/chat_env.py @@ -17,8 +17,8 @@ from mcp.types import PromptMessage, TextContent -from hud.agents.types import EvaluationResult from hud.environment import Environment +from hud.graders import EvaluationResult env = Environment(name="chat") From 12c91c670ce96d399b52ee4ec2f8a03fc7bfa364 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:34:26 -0700 Subject: [PATCH 076/248] ci: run the pre-release lifecycle as the hosted template scenario Each pre-release job now runs one template's hosted case from tests/e2e/test_templates.py, which generates the project, installs this checkout's SDK, deploys, runs one rollout on the HUD runtime with a live model, checks the lifecycle and data-file mapping, and reads the trace back from the platform: what the workflow's shell steps and scripts/example_lifecycle.py did, now in one place that the default and live lanes share. The script is gone. The workflow keeps its file name, trigger, main-only guard, pre-release environment, secrets, per-template concurrency, timeout and trace upload. A new step fails the job when HUD_API_KEY is missing, since the hosted test would otherwise skip and pass the release gate. The template's own lint and tests now run on every PR in the default lane instead. --- .github/workflows/pre-release.yml | 54 ++++++------------- scripts/example_lifecycle.py | 87 ------------------------------- tests/e2e/test_templates.py | 2 +- 3 files changed, 17 insertions(+), 126 deletions(-) delete mode 100644 scripts/example_lifecycle.py diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 26bb57909..8cdcaa509 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -1,5 +1,8 @@ name: Pre-release +# The release gate (release.yml) looks up a successful run of this workflow on the +# tagged commit, so keep its file name and trigger. + on: workflow_dispatch: @@ -15,8 +18,6 @@ jobs: concurrency: group: ci-example-${{ matrix.template }} cancel-in-progress: false - env: - EXAMPLE_NAME: ci-${{ matrix.template }} strategy: fail-fast: false matrix: @@ -27,53 +28,30 @@ jobs: uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Create example environment - run: uv run hud init "$RUNNER_TEMP/$EXAMPLE_NAME" --template "${{ matrix.template }}" - - - name: Install example with the SDK from this checkout - working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }} - run: | - uv build --wheel --no-create-gitignore --out-dir .hud-sdk "$GITHUB_WORKSPACE" - uv add --no-sync .hud-sdk/hud-*.whl - uv sync --all-extras - sed -i '/^RUN .*uv sync/i COPY .hud-sdk .hud-sdk' Dockerfile.hud - echo '!.hud-sdk/' >> .dockerignore - echo '!.hud-sdk/**' >> .dockerignore - - - name: Run example checks - working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }} - run: | - uv run --no-sync hud task list --source . - if [ -d tests ]; then - uv run --no-sync ruff format . --check - uv run --no-sync ruff check . - uv run --no-sync pytest -q - fi + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - - name: Deploy example environment - working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }} + # The hosted test skips without credentials, which must not pass the gate. + - name: Require credentials env: HUD_API_KEY: ${{ secrets.HUD_API_KEY }} - run: uv run --no-sync hud deploy --no-env + run: test -n "$HUD_API_KEY" - - name: Run hosted task lifecycle - working-directory: ${{ runner.temp }}/${{ env.EXAMPLE_NAME }} + # hud init, install this checkout's SDK, deploy, run one rollout on the HUD + # runtime with a live model, check its lifecycle, and read the trace back. + - name: Deploy and run the template on HUD env: HUD_API_KEY: ${{ secrets.HUD_API_KEY }} HUD_CI_DATA_FILE_ID: ${{ vars.HUD_CI_DATA_FILE_ID }} - HUD_TELEMETRY_LOCAL_DIR: ${{ runner.temp }}/traces-${{ matrix.template }} - run: | - uv run --no-sync "$GITHUB_WORKSPACE/scripts/example_lifecycle.py" prepare "${{ matrix.template }}" - timeout --signal=INT --kill-after=30s 15m uv run --no-sync hud eval .hud/ci-tasks.json claude \ - --model claude-sonnet-4-6 --gateway --runtime hud --max-steps 10 --config max_tokens=2048 -y - uv run --no-sync "$GITHUB_WORKSPACE/scripts/example_lifecycle.py" check + HUD_E2E_TRACE_DIR: ${{ runner.temp }}/traces + run: >- + uv run --python 3.12 --all-extras + pytest -m "e2e and hosted" -p no:cacheprovider -rs + "tests/e2e/test_templates.py::test_a_deployed_template_runs_on_hud_and_its_trace_reads_back[${{ matrix.template }}]" - name: Save traces if: always() uses: actions/upload-artifact@v4 with: name: example-traces-${{ matrix.template }} - path: ${{ runner.temp }}/traces-${{ matrix.template }} + path: ${{ runner.temp }}/traces/${{ matrix.template }} retention-days: 7 diff --git a/scripts/example_lifecycle.py b/scripts/example_lifecycle.py deleted file mode 100644 index 9c26afcfb..000000000 --- a/scripts/example_lifecycle.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Prepare live example tasks and check their traces.""" - -from __future__ import annotations - -import argparse -import json -import os -import subprocess -import sys -import time -from pathlib import Path - -from hud.eval import Taskset - - -def prepare(example: str) -> None: - task = next(iter(Taskset.from_file("tasks.py"))) - Path(".hud").mkdir(exist_ok=True) - if example == "argument-hints": - file_id = os.environ["HUD_CI_DATA_FILE_ID"] - if not file_id: - raise ValueError("HUD_CI_DATA_FILE_ID must reference the shared CI attachment") - task.args["attachments"] = [{"file_id": file_id, "path": "notes.txt"}] - task.args["prompt"] = "Read files/notes.txt first. " + task.args["prompt"] - Taskset(tasks=[task]).to_file(".hud/ci-tasks.json") - - -def check() -> None: - trace_dir = Path(os.environ["HUD_TELEMETRY_LOCAL_DIR"]) - traces = list(trace_dir.glob("*.jsonl")) - assert len(traces) == 1, f"Expected one rollout trace, found {len(traces)}" - steps = [ - span["attributes"]["hud.payload"] - for line in traces[0].read_text().splitlines() - if (span := json.loads(line))["attributes"].get("hud.schema") == "hud.step.v1" - ] - errors = [ - step["error"] - for step in steps - if step["source"] in {"task", "agent", "system"} and step.get("error") - ] - assert not errors, errors - calls = [step["task_call"] for step in steps if step["source"] == "task"] - assert [call["phase"] for call in calls] == ["setup", "evaluate"], calls - assert calls[0]["result"].get("prompt"), "Task setup did not return a prompt" - grade = calls[1]["result"] - assert not grade.get("isError"), grade - assert isinstance(grade["score"], int | float) and 0 <= grade["score"] <= 1, grade - task = next(iter(Taskset.from_file(".hud/ci-tasks.json"))) - if attachments := task.args.get("attachments"): - assert calls[0]["result"]["data_files"] == [ - {"path": "files/notes.txt", "file_id": attachments[0]["file_id"]} - ] - - # Unset local export so `hud trace` must read back from the platform. - env = {key: value for key, value in os.environ.items() if key != "HUD_TELEMETRY_LOCAL_DIR"} - for attempt in range(12): - result = subprocess.run( - [str(Path(sys.executable).with_name("hud")), "trace", "get", traces[0].stem, "--json"], - env=env, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - events = json.loads(result.stdout) if result.returncode == 0 else [] - if any(event["kind"] == "agent_message" for event in events): - (trace_dir / "remote-events.json").write_text(result.stdout) - print(f"Hosted lifecycle verified: {traces[0].stem}, score={grade['score']}") - return - if attempt < 11: - time.sleep(5) - raise AssertionError( - f"No persisted agent events after 12 attempts: {result.stdout}\n{result.stderr}" - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - commands.add_parser("prepare").add_argument("example") - commands.add_parser("check") - args = parser.parse_args() - if args.command == "prepare": - prepare(args.example) - elif args.command == "check": - check() diff --git a/tests/e2e/test_templates.py b/tests/e2e/test_templates.py index 9eaf5ba05..fd47bac4d 100644 --- a/tests/e2e/test_templates.py +++ b/tests/e2e/test_templates.py @@ -145,7 +145,7 @@ async def test_a_template_completes_a_rollout_with_a_real_model( @pytest.mark.e2e @pytest.mark.live @pytest.mark.hosted -@pytest.mark.timeout(2700) +@pytest.mark.timeout(2400) @pytest.mark.parametrize("template", TEMPLATES) def test_a_deployed_template_runs_on_hud_and_its_trace_reads_back( template: str, live: HudEnv, hud: Hud, uv: Uv, sdk_wheel: Path, tmp_path: Path From 6bc72d5de0f020c862a3bc57813c3b1aa0776980 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:39:44 -0700 Subject: [PATCH 077/248] test: track the compose-variables case's project .env The root .gitignore excludes every .env, so the case's dotenv never reached the repository and the case reported DOTENV_ONLY as unset. Harbor case fixtures carry a project's .env as data; re-include them. --- .gitignore | 2 ++ .../harbor/cases/compose-variables/task-a/environment/.env | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 tests/integrations/harbor/cases/compose-variables/task-a/environment/.env diff --git a/.gitignore b/.gitignore index 1fcc15104..4a516cd5e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ build/ !docs/v6/build/ *.egg-info/ uv.lock +# Harbor case fixtures carry a project's .env as test data. +!tests/integrations/harbor/cases/**/.env # Media files *.jpg diff --git a/tests/integrations/harbor/cases/compose-variables/task-a/environment/.env b/tests/integrations/harbor/cases/compose-variables/task-a/environment/.env new file mode 100644 index 000000000..465da46bd --- /dev/null +++ b/tests/integrations/harbor/cases/compose-variables/task-a/environment/.env @@ -0,0 +1,3 @@ +DOTENV_ONLY=dotenv +SIDECAR_TAG=7-alpine +LITERAL=from-dotenv From 4aa7b1f642b29e4e018322b9a37ec5a4a882b283 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:39:44 -0700 Subject: [PATCH 078/248] test: drop the gateway Bedrock row the Claude transport scenario covers Both rows sent a Bedrock inference profile to a fake Bedrock endpoint and checked the path and SigV4 signing; the transport scenario also checks the non-streaming body and skips without botocore, which the gateway row did not. --- tests/utils/test_gateway.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/tests/utils/test_gateway.py b/tests/utils/test_gateway.py index 3524f6d82..0d27da7ba 100644 --- a/tests/utils/test_gateway.py +++ b/tests/utils/test_gateway.py @@ -12,13 +12,13 @@ from hud.agents.types import ClaudeConfig, GeminiConfig, OpenAIChatConfig, OpenAIConfig from hud.eval import LocalRuntime, Task, Taskset, rollout from hud.utils.exceptions import HudAuthenticationError -from tests.harness import Reply, say +from tests.harness import say if TYPE_CHECKING: from collections.abc import Callable from hud.eval import Run - from tests.harness import FakeServices, HudEnv, Models + from tests.harness import HudEnv, Models BEDROCK_ARN = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/anthropic.claude" AWS = { @@ -145,26 +145,6 @@ def test_an_unroutable_model_is_refused_when_the_agent_is_built( build() -async def test_a_bedrock_profile_is_sent_to_bedrock_signed_with_aws_credentials( - models: Models, provider_server: FakeServices, hud_env: HudEnv -) -> None: - hud_env.set(HUD_API_KEY="hud-key", ANTHROPIC_BEDROCK_BASE_URL=provider_server.url("api"), **AWS) - provider_server.reset() - provider_server.route("api", "POST", "/model/{rest:path}", Reply(status=400, json={})) - - run = await rollout( - TASK, - ClaudeAgent(ClaudeConfig(model=BEDROCK_ARN, max_steps=1)), - runtime=LocalRuntime(_env()), - ) - - (request, *_) = provider_server.requests("api", "POST") - assert run.reward == 0.0 - assert models.requests() == [] - assert request.params["rest"] == f"{BEDROCK_ARN}/invoke" - assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 Credential=AKIATEST/") - - @pytest.mark.parametrize("family", list(AGENTS)) async def test_each_gateway_request_carries_its_own_runs_trace_id( models: Models, hud_env: HudEnv, family: str From bcf24eb42b503e141af7d7ae414d671edb150774 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:39:55 -0700 Subject: [PATCH 079/248] test: accept the host's platform in the missing-CLI error The workspace in this scenario runs on the test host, so the platform the error names is the host's, not always Linux. --- tests/agents/test_cli_agents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/agents/test_cli_agents.py b/tests/agents/test_cli_agents.py index 6f6294799..b0d328d9f 100644 --- a/tests/agents/test_cli_agents.py +++ b/tests/agents/test_cli_agents.py @@ -634,7 +634,7 @@ async def test_a_missing_cli_names_the_runtime_platform( run = await run_task(env, cli.agent()) assert run.trace.error == IsStr( - regex=r".* is unavailable for runtime platform linux-\w+; install it in the " + regex=r".* is unavailable for runtime platform [a-z]+-\w+; install it in the " r"environment or provide a managed runtime bundle" ) From bf175585fb0e1c172c4d038577bc1a72c69b86ee Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:39:55 -0700 Subject: [PATCH 080/248] ci: check cookbooks nightly and give the live lane its data file The cookbook scenarios run only from the e2e marker, which no job selected. The argument-hints live case attaches HUD_CI_DATA_FILE_ID when it is set. --- .github/workflows/e2e.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 44c403e8d..db7576f3f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -71,6 +71,7 @@ jobs: timeout-minutes: 45 env: HUD_API_KEY: ${{ secrets.HUD_API_KEY }} + HUD_CI_DATA_FILE_ID: ${{ vars.HUD_CI_DATA_FILE_ID }} steps: - uses: actions/checkout@v4 @@ -81,3 +82,20 @@ jobs: run: >- uv run --python 3.12 --all-extras pytest -m "e2e and live and not hosted" -n 4 --timeout 1200 --durations 20 + + cookbooks: + name: Cookbooks + # Cookbooks live outside the package, so SDK changes can break them silently. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + + - name: Check cookbooks + run: >- + uv run --python 3.12 --all-extras + pytest tests/e2e/test_cookbooks.py -m e2e -n 4 --timeout 1200 --durations 20 From 541a41c167be85564b0cf13293ed1e82afa766b3 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:42:53 -0700 Subject: [PATCH 081/248] test: keep Docker's config directory when isolating the home Docker Desktop installs CLI plugins and keeps the current context under ~/.docker, so with HOME moved to a temporary directory the docker lane lost compose and could not reach the daemon on macOS. --- tests/harness/config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/harness/config.py b/tests/harness/config.py index 07c6dafe4..5289aa9b9 100644 --- a/tests/harness/config.py +++ b/tests/harness/config.py @@ -10,6 +10,7 @@ from __future__ import annotations import os +from pathlib import Path from typing import TYPE_CHECKING from hud.settings import Settings, settings @@ -18,8 +19,6 @@ from .services import SERVICE_ENV if TYPE_CHECKING: - from pathlib import Path - import pytest # Every service the SDK calls points at a closed local port until a test opts into @@ -59,6 +58,11 @@ def __init__(self, monkeypatch: pytest.MonkeyPatch, home: Path) -> None: self._monkeypatch = monkeypatch self.home = home home.mkdir(parents=True, exist_ok=True) + # The Docker CLI finds its plugins (compose) and current context under its + # config directory; keep the real one so a real daemon stays reachable. + monkeypatch.setenv( + "DOCKER_CONFIG", os.environ.get("DOCKER_CONFIG") or str(Path.home() / ".docker") + ) monkeypatch.setenv("HOME", str(home)) monkeypatch.setenv("USERPROFILE", str(home)) for name in (*setting_variables(), *PROVIDER_ENV): From 0cb50827f343b44b0413f1d6be0c31a8b1d8bcce Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:42:53 -0700 Subject: [PATCH 082/248] docs: describe the template scenarios and cookbook lane in CONTRIBUTING The live selector now excludes hosted runs, which also carry the live marker and deploy to the platform. --- CONTRIBUTING.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7573bc633..548b74bd2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,19 +20,22 @@ CI runs the full suite; locally, run what you touched: ```bash uv run pytest tests/cli -n auto # one area, in parallel -uv run pytest -n auto # everything that needs nothing outside the process +uv run pytest -n auto # everything not marked e2e ``` -Tests run on Python 3.11 and 3.12 in CI. +Tests run on Python 3.11 and 3.12 in CI. The template scenarios in +`tests/e2e/test_templates.py` take each `hud init` template to a graded rollout +and install its packages, so they need network access to PyPI. Tests that need something outside the process are marked `e2e` and run only when selected. They skip themselves when what they need is missing: ```bash -uv run pytest -m "e2e and docker" # a running Docker daemon -uv run pytest -m "e2e and sandbox" # Linux, root and bubblewrap -uv run pytest -m "e2e and live" # a real HUD account: HUD_API_KEY; spends credits -uv run pytest -m "e2e and hosted" # deploys and runs on the HUD platform +uv run pytest -m "e2e and docker" # a running Docker daemon +uv run pytest -m "e2e and sandbox" # Linux, root and bubblewrap +uv run pytest -m "e2e and live and not hosted" # HUD_API_KEY; spends credits +uv run pytest -m "e2e and hosted" # deploys and runs on the HUD platform +uv run pytest -m e2e tests/e2e/test_cookbooks.py # the cookbooks; CI checks them nightly ``` ### Writing Tests From ca499e3aff9895ad9e5bb1b3916049e74fe3665f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:45:15 -0700 Subject: [PATCH 083/248] refactor(environment): handle SIGTERM in the serve entry points, not serve() serve() is importable library code; installing a SIGTERM handler there replaced an embedding program's handler on its loop and removed it on exit. serve_until_terminated keeps the clean stop for python -m hud.environment.server and hud serve, the two entry points that own their process. --- hud/cli/serve.py | 4 ++-- hud/environment/server.py | 42 +++++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/hud/cli/serve.py b/hud/cli/serve.py index 34aa5972d..ea30c2c8a 100644 --- a/hud/cli/serve.py +++ b/hud/cli/serve.py @@ -14,7 +14,7 @@ from hud.cli import parse_key_value from hud.environment import load_environment -from hud.environment.server import serve +from hud.environment.server import serve_until_terminated from hud.utils.hud_console import HUDConsole hud_console = HUDConsole() @@ -69,6 +69,6 @@ def serve_command( ) hud_console.hint("Press Ctrl+C to stop.") try: - asyncio.run(serve(env, host, port)) + asyncio.run(serve_until_terminated(env, host, port)) except KeyboardInterrupt: hud_console.info("Stopped.") diff --git a/hud/environment/server.py b/hud/environment/server.py index 8adf1db65..a81cf978e 100644 --- a/hud/environment/server.py +++ b/hud/environment/server.py @@ -574,11 +574,28 @@ async def _shutdown(server: asyncio.Server) -> None: async def serve(env: Environment, host: str = "127.0.0.1", port: int = 0) -> None: - """Start *env*'s daemons and serve its control channel until cancelled or SIGTERM. + """Start *env*'s daemons and serve its control channel until cancelled.""" + await env.start() + server: asyncio.Server | None = None + try: + server = await bind(env, host, port) + port_line = f"{PORT_ANNOUNCEMENT}{server.sockets[0].getsockname()[1]}" + print(port_line, flush=True) # noqa: T201 - the spawn provider reads this from stdout + async with server: + await server.serve_forever() + finally: + if server is not None: + await _shutdown(server) + await env.stop() + + +async def serve_until_terminated(env: Environment, host: str = "127.0.0.1", port: int = 0) -> None: + """``serve`` as a process's main task: SIGTERM stops it cleanly and it returns. SIGTERM (a container stop, the spawn provider's teardown) ends serving the way cancellation does, so shutdown hooks run and backing daemons don't - orphan, and then returns normally. + orphan. The handler is process-wide, so only entry points that own the + process call this. """ main_task = asyncio.current_task() assert main_task is not None @@ -594,7 +611,7 @@ def terminate() -> None: with contextlib.suppress(NotImplementedError): loop.add_signal_handler(signal.SIGTERM, terminate) try: - await _serve(env, host, port) + await serve(env, host, port) except asyncio.CancelledError: if not terminated: raise @@ -604,21 +621,6 @@ def terminate() -> None: loop.remove_signal_handler(signal.SIGTERM) -async def _serve(env: Environment, host: str, port: int) -> None: - await env.start() - server: asyncio.Server | None = None - try: - server = await bind(env, host, port) - port_line = f"{PORT_ANNOUNCEMENT}{server.sockets[0].getsockname()[1]}" - print(port_line, flush=True) # noqa: T201 - the spawn provider reads this from stdout - async with server: - await server.serve_forever() - finally: - if server is not None: - await _shutdown(server) - await env.stop() - - def main() -> None: from hud.environment import load_environment @@ -630,7 +632,9 @@ def main() -> None: ) parser.add_argument("--port", type=int, default=0, help="Port to bind (0 = ephemeral).") args = parser.parse_args() - asyncio.run(serve(load_environment(args.path, name=args.env), args.host, args.port)) + asyncio.run( + serve_until_terminated(load_environment(args.path, name=args.env), args.host, args.port) + ) if __name__ == "__main__": From 216f7848e68ceb041bf56ab9fd75176fc46275e6 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:45:15 -0700 Subject: [PATCH 084/248] ci: keep the sandbox job's environment off the mounted checkout Running the job's command from a developer checkout otherwise replaces the host's .venv with the container's. --- .github/workflows/e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index db7576f3f..ec4ce04e2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -56,6 +56,7 @@ jobs: --security-opt systempaths=unconfined --security-opt apparmor=unconfined --volume "$PWD:/src" --workdir /src + --env UV_PROJECT_ENVIRONMENT=/opt/venv ghcr.io/astral-sh/uv:python3.12-bookworm sh -c 'apt-get update -qq && apt-get install -y -qq bubblewrap openssh-client >/dev/null && uv run --all-extras pytest -m "e2e and sandbox" -n 4 --timeout 600' From 07561f170a35852a2d6f3c2e7425d191fef473b0 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:48:03 -0700 Subject: [PATCH 085/248] perf(harbor): install the controller before copying task-specific inputs config.json and the authored root were copied ahead of install.sh, so every task rebuilt the controller install (about 630 MB) as its own layer. Copying them after the install lets every task with the same SDK requirement reuse one install layer; the image contents are unchanged. --- hud/integrations/harbor/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hud/integrations/harbor/Dockerfile b/hud/integrations/harbor/Dockerfile index 5ffff5e3a..3872f1361 100644 --- a/hud/integrations/harbor/Dockerfile +++ b/hud/integrations/harbor/Dockerfile @@ -13,10 +13,12 @@ FROM python:3.12-slim AS runtime USER root ARG HUD_REQUIREMENT=hud COPY --from=ghcr.io/astral-sh/uv:0.8.15 /uv /controller/bin/uv -COPY install.sh config.json /controller/ +COPY install.sh /controller/ COPY packages /controller/packages -COPY --from=authored-root / /rootfs RUN sh /controller/install.sh "${HUD_REQUIREMENT}" && mkdir -p /runtime +# Task-specific inputs follow the install, so every task reuses its layer. +COPY config.json /controller/ +COPY --from=authored-root / /rootfs ENV HUD_SKIP_VERSION_CHECK=1 EXPOSE 8765 From 006881a12d5cd54b880d56a81f4e639cb48bbe9b Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 086/248] docs: align the runtime, task and job reference with the rollout engine Job.reward averages graded runs and excludes errored ones, which Job.errors lists; run.evaluation summarizes subscores rather than echoing grade.raw; Daytona rejects Compose projects; and a platform taskset without runtime= runs over the HUD tunnel rather than on the platform. --- docs/v6/reference/runtime.mdx | 10 ++++------ docs/v6/reference/tasks.mdx | 2 +- docs/v6/reference/types.mdx | 8 +++++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx index 2345f7d05..b2656fd84 100644 --- a/docs/v6/reference/runtime.mdx +++ b/docs/v6/reference/runtime.mdx @@ -34,7 +34,7 @@ HostedRuntime, Runtime`); `ModalRuntime` and `DaytonaRuntime` import from `hud.e **You can usually omit `runtime=`.** A run without one uses its known placement: -- a platform taskset (`Taskset.from_api`) runs on the platform +- a platform taskset (`Taskset.from_api`) runs against its deployed environment over the HUD tunnel (`HUDRuntime`), with the agent loop in this process - a row whose `runtime_config` names an image or Compose project starts it under `DockerRuntime` - a task created by `@env.template` runs against that environment, in this process @@ -83,7 +83,7 @@ is best effort: `DockerRuntime` admits against available disk and `DaytonaRuntim enough whole GiB, while providers without disk sizing proceed with their default capacity. `HUDRuntime` proceeds on the deployed environment's defaults and warns when it ignores CPU, memory, or storage requirements; it rejects GPU, OS, TPU, and `limits`. Daytona accepts a list of GPU -alternatives; Modal accepts one GPU type. Daytona rejects `run_timeout_s` and resource overrides +alternatives; Modal accepts one GPU type. Daytona rejects Compose, `run_timeout_s`, and resource overrides when booting from an already-built snapshot. `LocalRuntime` rejects a per-task `runtime_config`. @@ -219,10 +219,8 @@ DaytonaRuntime(snapshot_name=None, *, image=None, command=None, workdir="/app", Resources (cpu/memory/gpu) are fixed on the snapshot at build time. With `image`, a task's `runtime_config.resources` builds a sized variant under a suffixed name (`my-env-4cpu`); without `image`, an already-built snapshot cannot be resized. -For Compose input, Daytona runs `main` as the rollout sandbox and each sidecar -as a linked sandbox. Sidecar images resolve to reusable Daytona snapshots. -Compose authoring conventions are documented in -[Compose environments](/v6/experimental/compose). +`DaytonaRuntime` does not run Compose projects: a `runtime_config.compose` is rejected. Use +`DockerRuntime` or `ModalRuntime` for [Compose environments](/v6/experimental/compose). Measured behavior at scale, up to 256 concurrent sandboxes with warm pools, is written up in the [Daytona cookbook](/v6/cookbooks/daytona-rl) and the [full guide on Daytona's docs](https://www.daytona.io/docs/guides/reinforcement-learning/hud-rl-cookbook). diff --git a/docs/v6/reference/tasks.mdx b/docs/v6/reference/tasks.mdx index 10ac99d25..20c96daf5 100644 --- a/docs/v6/reference/tasks.mdx +++ b/docs/v6/reference/tasks.mdx @@ -158,7 +158,7 @@ job = await ts.run(agent, runtime=LocalRuntime("env.py"), group=8, max_concurren | Parameter | Type | Description | |-----------|------|-------------| | `agent` | `Agent` | The shared, stateless agent driven across every rollout. | -| `runtime` | `Provider \| HostedRuntime \| None` | Where each rollout runs; omit it and placement is inferred (local source serves itself, else the HUD tunnel by env name). See [runtime](/v6/reference/runtime). | +| `runtime` | `Provider \| HostedRuntime \| None` | Where each rollout runs; omit it and placement is inferred: a platform taskset (`Taskset.from_api`) runs over the HUD tunnel, a row whose `runtime_config` names an image or Compose project starts it under `DockerRuntime`, a row minted by a live environment runs against it, and any other row raises `ValueError`. See [runtime](/v6/reference/runtime). | | `group` | `int \| None` | Repeats each task N times for GRPO. | | `max_concurrent` | `int \| None` | Caps how many rollouts run in parallel. | | `job` | `Job \| None` | An open [`Job`](/v6/reference/types#job) to accumulate into; defaults to a fresh job per call. | diff --git a/docs/v6/reference/types.mdx b/docs/v6/reference/types.mdx index 21c77c91c..31c51237c 100644 --- a/docs/v6/reference/types.mdx +++ b/docs/v6/reference/types.mdx @@ -26,7 +26,7 @@ The live handle for one task - the lifecycle plus the agent's `Trace`. You get t | `run.trace` | `Trace` | The trajectory the agent fills. **The answer is `run.trace.content`.** | | `run.grade` | `Grade` | Structured grade result. | | `run.reward` | `float` | The graded reward (`grade.reward`, set on exit). | -| `run.evaluation` | `dict` | The raw grade payload (`grade.raw`). | +| `run.evaluation` | `dict` | A persistence-safe copy of the grade frame: `grade.raw` with each `subscores` entry reduced to its summary. | | `run.runtime` | `str \| None` | Control-channel url the run executed against (the placement record). | | `run.trace_id` | `str \| None` | Keys the trajectory for training. | | `run.slug` | `str \| None` | The task slug this run came from, set by the runner; how [`Job.results`](#job) keys runs back to their task without positional zip. | @@ -63,12 +63,14 @@ run reports under a job, so even a single `task.run` returns a job of one. You g | `runs` | `list[Run]` | The graded runs, in expansion order (task-major, then group). | | `group` | `int` | Rollouts per task. | | `taskset_id` | `str \| None` | Platform taskset id when the job runs a synced taskset (`Taskset.from_api`), else `None`. | -| `reward` | `float` | Property: mean reward across runs (`0.0` for an empty job). | +| `reward` | `float` | Property: mean reward across graded runs (`0.0` when none were graded). Runs in `errors` are excluded rather than averaged in as zero. | +| `errors` | `list[Run]` | Property: runs that ended without a valid grade - a grading error, or a failed trace with no grade frame. A run that timed out or raised after the task started still counts toward `reward` when its environment returned a grade. | | `results` | `dict[str, list[Run]]` | Property: runs grouped by task slug - the alignment-safe alternative to `zip(tasks, runs)`, list-valued since `group > 1` gives several runs per task. | ```python job = await ts.run(agent, runtime=LocalRuntime("env.py"), group=4) -print(job.reward) # mean across every run +print(job.reward) # mean across graded runs +print(len(job.errors)) # runs excluded for lack of a grade ``` Each `run` call mints its own job by default. To gather many calls under one id - a training session, a From 03f3478cb90ea388eec8257afbdf94863e217039 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 087/248] test: let fake docker emulate container files and scripted agents linger A rule given a rootfs runs docker cp and docker exec against per-container directories, so session transfer between two containers can run end to end. ScriptedAgent(linger=) keeps an agent running after it answers, so a deadline can land on an answered run. --- tests/harness/docker.py | 60 +++++++++++++++++++++++++++++++++++++-- tests/harness/scenario.py | 7 ++++- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/tests/harness/docker.py b/tests/harness/docker.py index 2c6a2f3df..ca26587e0 100644 --- a/tests/harness/docker.py +++ b/tests/harness/docker.py @@ -6,6 +6,11 @@ joined by spaces. Tests add rules ahead of the defaults with :meth:`FakeDocker.on`. :meth:`FakeDocker.images` adds an image store that answers ``version``, ``build``, ``compose build|pull`` and ``image inspect`` between the two. + +A rule given a ``rootfs`` directory emulates ``docker cp`` and ``docker exec`` +instead of answering: ``/`` stands in for each container's +filesystem, ``cp`` copies between it and the host, and ``exec`` runs the +command on the host with its absolute-path arguments moved under it. """ from __future__ import annotations @@ -22,7 +27,7 @@ import pytest SCRIPT = """#!{python} -import json, os, pathlib, re, runpy, sys, time +import json, os, pathlib, re, runpy, shutil, subprocess, sys, time state = pathlib.Path(os.environ["FAKE_DOCKER_STATE"]) argv = sys.argv[1:] @@ -33,11 +38,45 @@ with (state / "calls.jsonl").open("a") as log: log.write(json.dumps({{"argv": argv, "files": files}}) + "\\n") joined = " ".join(argv) + + +def inside(root, reference): + container, _, path = reference.partition(":") + return root / container / path.lstrip("/") + + +def emulate(root): + if argv[0] == "cp": + source, target = ( + inside(root, item) if re.match(r"^[^:/]+:/", item) else pathlib.Path(item) + for item in argv[-2:] + ) + if argv[-2].endswith("/."): + shutil.copytree(source, target, dirs_exist_ok=True) + else: + if target.is_dir(): + target = target / source.name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(source, target) + return 0 + index = 1 + while argv[index].startswith("-"): + index += 2 if argv[index] in ("--user", "-u", "--env", "-e", "--workdir", "-w") else 1 + container_root = root / argv[index] + command = [ + str(container_root / item.lstrip("/")) if item.startswith("/") else item + for item in argv[index + 1 :] + ] + return subprocess.run(command, check=False).returncode + + rules = json.loads((state / "rules.json").read_text()) for default in (False, True): for rule in rules: if rule.get("default", False) == default and re.search(rule["match"], joined): time.sleep(rule["delay"]) + if rule.get("rootfs"): + sys.exit(emulate(pathlib.Path(rule["rootfs"]))) sys.stdout.write(rule["stdout"]) sys.stderr.write(rule["stderr"]) sys.exit(rule["exit"]) @@ -170,9 +209,22 @@ def on( stderr: str = "", exit: int = 0, delay: float = 0.0, + rootfs: Path | None = None, ) -> None: - """Answer invocations whose argv matches the regex ``match``; newest rules win.""" - self._rule(match, stdout=stdout, stderr=stderr, exit=exit, delay=delay, default=False) + """Answer invocations whose argv matches the regex ``match``; newest rules win. + + With ``rootfs``, a matching ``cp`` or ``exec`` runs against container + filesystems under that directory instead of answering. + """ + self._rule( + match, + stdout=stdout, + stderr=stderr, + exit=exit, + delay=delay, + rootfs=rootfs, + default=False, + ) def images( self, @@ -212,6 +264,7 @@ def _rule( stderr: str = "", exit: int = 0, delay: float = 0.0, + rootfs: Path | None = None, default: bool, ) -> None: self._rules.insert( @@ -222,6 +275,7 @@ def _rule( "stderr": stderr, "exit": exit, "delay": delay, + "rootfs": str(rootfs) if rootfs is not None else None, "default": default, }, ) diff --git a/tests/harness/scenario.py b/tests/harness/scenario.py index 84bdcec19..fefda114d 100644 --- a/tests/harness/scenario.py +++ b/tests/harness/scenario.py @@ -22,7 +22,8 @@ class ScriptedAgent(Agent): """Answers every run with ``answer``, or ``answer(prompt)`` when it is callable. ``delay`` waits before answering; ``fail_before`` raises instead of answering - and ``fail_after`` raises after the answer is recorded. + and ``fail_after`` raises after the answer is recorded. ``linger`` keeps the + agent running that long after answering, so a deadline can land on it. """ def __init__( @@ -32,6 +33,7 @@ def __init__( delay: float = 0.0, fail_before: BaseException | None = None, fail_after: BaseException | None = None, + linger: float = 0.0, ) -> None: super().__init__() self._answer: Callable[[str], str] = ( @@ -40,6 +42,7 @@ def __init__( self._delay = delay self._fail_before = fail_before self._fail_after = fail_after + self._linger = linger self.prompts: list[str] = [] async def __call__(self, run: Run) -> None: @@ -49,6 +52,8 @@ async def __call__(self, run: Run) -> None: if self._fail_before is not None: raise self._fail_before run.trace.content = self._answer(run.prompt_text) + if self._linger: + await asyncio.sleep(self._linger) if self._fail_after is not None: raise self._fail_after From 3918335009f616a8d74a4a1b2590bbd831c9372c Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 088/248] test: rebuild the rollout lifecycle as one scenario table Every failure point (agent, deadline, grader, start frame, frame limit, provider) is a row asserting the run fields, the exported step spans and the environment's hook log, against fixture environments that really fail the way the row names. HudClient, trace_exit, trace_enter and asyncio.wait are no longer patched; the prompt views are driven by templates instead of hand-built runs. The rows for runs that never launched fail: their failure step is never exported as a span. --- tests/eval/envs.py | 170 ++++ tests/eval/test_rollout.py | 1830 ++++++++---------------------------- 2 files changed, 562 insertions(+), 1438 deletions(-) create mode 100644 tests/eval/envs.py diff --git a/tests/eval/envs.py b/tests/eval/envs.py new file mode 100644 index 000000000..300e8093b --- /dev/null +++ b/tests/eval/envs.py @@ -0,0 +1,170 @@ +"""Fixture environments for the rollout engine scenarios. + +Each template's reward proves the path it names ran: ``add`` pays 1.0 only for +the right sum, so a graded run shows the prompt reached the agent and the answer +reached the grader. Hooks append to an ``events`` list the test owns. +""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any + +from hud import Environment +from hud.environment.env import current_session_id + +if TYPE_CHECKING: + from collections.abc import Callable + +SUMS_SOURCE = """ +from hud import Environment + +env = Environment("lab") + + +@env.template() +async def add(a: int, b: int): + answer = yield f"add {a} {b}" + yield 1.0 if answer == str(a + b) else 0.0 +""" + + +async def eventually(condition: Callable[[], bool], *, within: float = 10.0) -> None: + """Wait for work a rollout left running in the background to finish.""" + deadline = asyncio.get_running_loop().time() + within + while not condition(): + if asyncio.get_running_loop().time() > deadline: + raise AssertionError(f"condition still false after {within}s") + await asyncio.sleep(0.01) + + +def solve(prompt: str) -> str: + """The right answer to an ``add`` prompt.""" + _, a, b = prompt.split() + return str(int(a) + int(b)) + + +def lab(events: list[str] | None = None) -> Environment: + """The ``lab`` environment: one template per way a task can start or grade.""" + log = events if events is not None else [] + env = Environment("lab") + + @env.initialize + async def initialize() -> None: + log.append("initialize") + + @env.shutdown + async def shutdown() -> None: + log.append("shutdown") + + @env.template() + async def add(a: int, b: int): + log.append(f"start add {a} {b}") + try: + answer = yield f"add {a} {b}" + log.append(f"grade {answer}") + yield 1.0 if answer == str(a + b) else 0.0 + finally: + log.append("end add") + + @env.template() + async def chat(messages: list[dict[str, Any]]): + yield messages + yield 1.0 + + @env.template() + async def silent(): + yield None + yield 1.0 + + @env.template() + async def claim(published: Any): + yield {"prompt": "go", "bindings": published} + yield 1.0 + + @env.template() + async def grade_raises(): + yield "go" + raise RuntimeError("grader exploded") + yield 0.0 + + @env.template() + async def frame(result: dict[str, Any]): + yield "go" + yield result + + @env.template() + async def hang_grading(): + try: + yield "go" + await asyncio.Event().wait() + yield 1.0 + finally: + log.append("grading cancelled") + + @env.template() + async def hang_start(): + try: + await asyncio.Event().wait() + yield "never" + yield 0.0 + finally: + log.append("start cancelled") + + @env.template() + async def large(criteria: str): + log.append("large started") + yield f"{len(criteria)}" + yield 1.0 + + return env + + +def actor(answers: list[str], *, grade: str = "score", sessions: Any = None) -> Environment: + """The ``actor`` side of a verifier pair: ``solve`` grades 0.25 and carries the answer. + + ``grade="raise"`` makes its grading raise and ``grade="scoreless"`` yields a + frame without a score. With ``sessions``, the template writes the answer into + its control session's directory under that root, as a container would. + """ + env = Environment("actor") + + @env.template() + async def solve(): + answer = yield "answer secret" + answers.append(str(answer)) + if sessions is not None: + session = sessions / "runtime" / "sessions" / str(current_session_id.get()) + session.mkdir(parents=True) + (session / "work.txt").write_text(str(answer)) + if grade == "raise": + raise RuntimeError("actor grade exploded") + yield {"score": 0.25, "answer": answer} if grade == "score" else {"answer": answer} + + return env + + +def judge(*, verdict: str = "check", sessions: Any = None) -> Environment: + """The ``judge`` side: ``verify`` pays 1.0 when the actor's answer was ``secret``. + + ``verdict`` picks a failure instead: ``"raise"``, ``"scoreless"`` or ``"zero"``. + With ``sessions``, it reads the answer from its restored session directory. + """ + env = Environment("judge") + + @env.template() + async def verify(): + result = yield "" + if verdict == "raise": + raise RuntimeError("verifier exploded") + if verdict == "scoreless": + yield {"verdict": "pass"} + elif verdict == "zero": + yield 0.0 + elif sessions is not None: + session = sessions / "runtime" / "sessions" / str(current_session_id.get()) + yield 1.0 if (session / "work.txt").read_text() == "secret" else 0.0 + else: + yield 1.0 if result["answer"] == "secret" else 0.0 + + return env diff --git a/tests/eval/test_rollout.py b/tests/eval/test_rollout.py index e786a3dde..9238499a3 100644 --- a/tests/eval/test_rollout.py +++ b/tests/eval/test_rollout.py @@ -1,1538 +1,492 @@ -"""The rollout engine: ``rollout(task, agent)`` and its schedulers. - -These drive the engine end-to-end through the real placement path: a pure-data -``Task`` row plus ``runtime=SubprocessRuntime(env_file)`` — a child process serves the env, the -engine connects over the wire, the agent answers, grading comes back. The -engine contract is a graded :class:`Run` with a trace id (always under a job — -there are no standalone traces), and failure isolation that never raises: a -pre-launch failure yields a synthesized ``Run.failed``; a mid-run failure -keeps the real run and its evidence. ``Task.run`` / ``Taskset.run`` schedule -the atom and return a :class:`Job`. -""" +"""The rollout atom: one task driven to a graded ``Run``, whatever fails along the way.""" from __future__ import annotations import asyncio -import contextlib -import json -import os -import signal -import subprocess -import sys -import textwrap from contextlib import asynccontextmanager -from types import SimpleNamespace +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any -import mcp.types as mcp_types import pytest -from pydantic import BaseModel +from dirty_equals import IsStr -import hud.eval.run as run_module +from hud import Environment from hud.agents.base import Agent from hud.agents.openai_compatible import OpenAIChatAgent from hud.agents.types import OpenAIChatConfig from hud.capabilities import Connection -from hud.clients.client import HudClient -from hud.environment import Answer, Environment -from hud.eval import ( - Job, - LocalRuntime, - Runtime, - RuntimeConfig, - RuntimeLimits, - SubprocessRuntime, - Task, - Taskset, -) -from hud.eval.run import Run, rollout -from hud.telemetry.context import get_current_trace_id, get_trace_headers, set_trace_context - - -async def test_taskset_rejects_zero_group_before_execution(): - with pytest.raises(ValueError, match="group must be >= 1"): - await Taskset("empty", []).run(_FnAgent(lambda _: "answer"), group=0) - +from hud.eval import LocalRuntime, Run, Runtime, SubprocessRuntime, Task, rollout +from hud.telemetry.context import get_current_trace_id +from tests.eval.envs import SUMS_SOURCE, eventually, lab, solve +from tests.harness import ScriptedAgent, call, steps if TYPE_CHECKING: from collections.abc import AsyncIterator from pathlib import Path - from hud.eval.task import Task as TaskRow - -_SUMS_ENV = """\ -from hud import Environment - -env = Environment("sums") - - -@env.template() -async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 -""" - - -@pytest.fixture(scope="module") -def env_file(tmp_path_factory: pytest.TempPathFactory) -> Path: - path = tmp_path_factory.mktemp("sums") / "env.py" - path.write_text(textwrap.dedent(_SUMS_ENV), encoding="utf-8") - return path - - -class _FnAgent(Agent): - """Stateless agent: answers each run by applying ``fn`` to ``run.prompt``.""" - - def __init__(self, fn: Any) -> None: - super().__init__() - self._fn = fn + from hud.eval import Provider + from tests.harness import HudEnv, Models - async def __call__(self, run: Any) -> None: - run.trace.content = self._fn(run.prompt) - - -class _SequencedCompletions: - def __init__(self, responses: list[Any]) -> None: - self._responses = responses - self.requests: list[dict[str, Any]] = [] - - async def create(self, **kwargs: Any) -> Any: - self.requests.append(kwargs) - return self._responses.pop(0) - - -class _FakeOpenAI: - def __init__(self, responses: list[Any]) -> None: - self.chat = SimpleNamespace(completions=_SequencedCompletions(responses)) - - -def _chat_response(content: str, tool_calls: list[Any] | None = None) -> Any: - message = SimpleNamespace( - content=content, - tool_calls=tool_calls or [], - refusal=None, - model_dump=lambda exclude_none=True: {"role": "assistant", "content": content}, - ) - choice = SimpleNamespace(message=message, finish_reason="stop", logprobs=None) - return SimpleNamespace( - choices=[choice], - model="fake-openai-compatible", - usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, prompt_tokens_details=None), - ) - - -def _tool_call(name: str, arguments: str) -> Any: - return SimpleNamespace( - type="function", - id=f"call_{name}", - function=SimpleNamespace(name=name, arguments=arguments), - ) +HEX_ID = IsStr(regex=r"[0-9a-f]{32}") +LOOPBACK = IsStr(regex=r"tcp://127\.0\.0\.1:\d+") +GRADED_SPANS = ["task", "user", "task"] -def _add_task(a: int, b: int) -> Task: - """A pure data row; the env it names is defined by the spawned file.""" - return Task(env="sums", id="add", args={"a": a, "b": b}) - - -def _solve_add(prompt: str) -> str: - _, a, b = prompt.split(":") - return str(int(a) + int(b)) - - -def _pid_status(pid: int) -> str | None: - result = subprocess.run( - ["ps", "-o", "stat=", "-p", str(pid)], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ) - return result.stdout.strip() or None +def broken_provider(error: BaseException) -> Provider: + @asynccontextmanager + async def provider(task: Task) -> AsyncIterator[Runtime]: + raise error + yield Runtime("tcp://127.0.0.1:1") + return provider -def _pid_is_running(pid: int) -> bool: - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - status = _pid_status(pid) - if status is None: - return False - return not status.startswith("Z") +def hanging_provider(task: Task) -> Any: + @asynccontextmanager + async def provider() -> AsyncIterator[Runtime]: + await asyncio.Event().wait() + yield Runtime("tcp://127.0.0.1:1") -async def _wait_for_pid_inactive(pid: int, max_wait: float = 2.0) -> bool: - loop = asyncio.get_running_loop() - deadline = loop.time() + max_wait - while loop.time() < deadline: - if not _pid_is_running(pid): - return True - await asyncio.sleep(0.05) - return not _pid_is_running(pid) + return provider() -async def test_rollout_returns_graded_run_with_trace_id(env_file: Path) -> None: - run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file)) +def noted(message: str, note: str) -> BaseException: + error = EOFError(message) + error.add_note(note) + return error - assert run.reward == 1.0 - assert run.trace.content == "5" - assert run.trace_id is not None - # No standalone traces: a bare rollout registers a single-run job itself. - assert run.job_id is not None - # The factual placement record: the runtime this run executed against. - assert run.runtime is not None - assert run.runtime.startswith("tcp://127.0.0.1:") - - -def test_run_owns_named_controller_connections() -> None: - connection = Connection( - name="inference", - capability="ssh", - url="https://inference.hud.so", - headers={"Authorization": "Bearer scoped-runtime-token"}, - ) - run = Run(None, "task", {}, connections=(connection,)) - assert run.connections == {"inference": connection} - assert "scoped-runtime-token" not in repr(connection) +@dataclass +class Case: + task: Task + agent: ScriptedAgent + expected: dict[str, Any] + events: list[str] = field(default_factory=list) + provider: str = "live" + rollout_timeout: float | None = None -async def test_verifier_task_replaces_the_actor_grade_in_the_same_runtime() -> None: - env = Environment("reviewed") - completed: list[str] = [] +def add(a: int = 2, b: int = 3, **row: Any) -> Task: + return Task(env="lab", id="add", args={"a": a, "b": b}, **row) - @env.template() - async def solve(): - answer = yield "answer secret" - completed.append(f"actor:{answer}") - yield {"score": 0.25, "answer": answer} - @env.template() - async def verify(expected: str): - result = yield "" - completed.append(f"verifier:{result['answer']}") - yield 1.0 if result["answer"] == expected else 0.0 - - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify", args={"expected": "secret"}), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) - - assert run.reward == 1.0 - assert completed == ["actor:secret", "verifier:secret"] - evaluations = [ - step.task_call.name - for step in run.trace.steps - if step.task_call is not None and step.task_call.phase == "evaluate" - ] - assert evaluations == ["solve", "verify"] +def failed(error: Any, *, stop_reason: str | None = None) -> dict[str, Any]: + """A run that never went live: no prompt, no placement, no grade.""" + return { + "status": "error", + "stop_reason": stop_reason, + "reward": 0.0, + "raw": {}, + "error": error, + "prompt": None, + "runtime": None, + "spans": ["system"], + } -async def test_actor_result_is_forwarded_to_the_verifier() -> None: - env = Environment("reviewed") - received: list[str] = [] +def errored(error: Any, *, stop_reason: str | None = None) -> dict[str, Any]: + """A run whose grading failed: the prompt and placement survive, the grade does not.""" + return { + "status": "error", + "stop_reason": stop_reason, + "reward": 0.0, + "raw": {}, + "error": error, + "prompt": "go", + "runtime": LOOPBACK, + "spans": ["task", "user", "system"], + } - class ActorResult(BaseModel): - score: float - answer: str - @env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.0, "answer": answer} - - @env.template(returns=ActorResult) - async def verify(): - answer = yield "" - assert isinstance(answer, Answer) - assert isinstance(answer.content, ActorResult) - received.append(answer.content.answer) - yield 1.0 - - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify"), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) +COMPLETED = { + "status": "completed", + "stop_reason": None, + "reward": 1.0, + "raw": {"score": 1.0}, + "error": None, + "prompt": "add 2 3", + "runtime": LOOPBACK, + "spans": GRADED_SPANS, +} +HOOKS = ["initialize", "start add 2 3", "grade 5", "end add", "shutdown"] + +CASES = { + "an answer is graded": Case(add(), ScriptedAgent(solve), COMPLETED, HOOKS), + "a constructor source serves the row": Case( + add(), ScriptedAgent(solve), COMPLETED, HOOKS, provider="constructor" + ), + # The child process logs its hooks in its own memory. + "a subprocess source serves the row": Case( + add(), ScriptedAgent(solve), COMPLETED, provider="subprocess" + ), + "an agent raising before it answers is graded on no answer": Case( + add(), + ScriptedAgent(fail_before=RuntimeError("agent exploded")), + COMPLETED + | { + "status": "error", + "reward": 0.0, + "raw": {"score": 0.0}, + "error": "[agent loop] RuntimeError: agent exploded", + "spans": [*GRADED_SPANS, "system"], + }, + ["initialize", "start add 2 3", "grade None", "end add", "shutdown"], + ), + "an agent raising after it answers keeps its grade": Case( + add(), + ScriptedAgent(solve, fail_after=RuntimeError("agent exploded")), + COMPLETED + | { + "status": "error", + "error": "[agent loop] RuntimeError: agent exploded", + "spans": [*GRADED_SPANS, "system"], + }, + HOOKS, + ), + "an agent's own TimeoutError is an ordinary failure": Case( + add(), + ScriptedAgent(solve, fail_after=TimeoutError("provider timed out")), + COMPLETED + | { + "status": "error", + "error": "[agent loop] TimeoutError: provider timed out", + "spans": [*GRADED_SPANS, "system"], + }, + HOOKS, + ), + "an agent's own TimeoutError under an unexpired deadline is an ordinary failure": Case( + add(agent_config={"timeout_seconds": 10.0}), + ScriptedAgent(solve, fail_after=TimeoutError("provider timed out")), + COMPLETED + | { + "status": "error", + "error": "[agent loop] TimeoutError: provider timed out", + "spans": [*GRADED_SPANS, "system"], + }, + HOOKS, + ), + "the agent deadline stops the agent and still grades": Case( + add(agent_config={"timeout_seconds": 0.1}), + ScriptedAgent(solve, linger=30), + COMPLETED + | { + "status": "error", + "stop_reason": "timeout", + "error": "agent timed out after 0.1s", + "spans": ["task", "user", "system", "task"], + }, + HOOKS, + ), + "the rollout deadline during the agent loop leaves the run ungraded": Case( + add(), + ScriptedAgent(solve, linger=30), + COMPLETED + | { + "status": "error", + "stop_reason": "timeout", + "raw": {}, + "reward": 0.0, + "error": "rollout timed out after 0.2s during agent loop", + "spans": ["task", "user", "system"], + }, + ["initialize", "start add 2 3", "end add", "shutdown"], + rollout_timeout=0.2, + ), + "the rollout deadline covers grading": Case( + Task(env="lab", id="hang_grading"), + ScriptedAgent("x"), + errored("rollout timed out after 0.2s during grading", stop_reason="timeout"), + ["initialize", "grading cancelled", "shutdown"], + rollout_timeout=0.2, + ), + "the rollout deadline covers provisioning": Case( + add(), + ScriptedAgent(solve), + failed("rollout timed out after 0.2s during provisioning", stop_reason="timeout"), + provider="hanging", + rollout_timeout=0.2, + ), + "a grader that raises fails grading": Case( + Task(env="lab", id="grade_raises"), + ScriptedAgent("x"), + errored(IsStr(regex=r"\[grading\] .*HudProtocolError: .*grader exploded")), + ["initialize", "shutdown"], + ), + "a grade frame without a score fails grading": Case( + Task(env="lab", id="frame", args={"result": {"done": True}}), + ScriptedAgent("x"), + errored(IsStr(regex=r"\[grading\] .*missing a numeric 'score' \(keys: \['done'\]\)")), + ["initialize", "shutdown"], + ), + "a boolean score fails grading": Case( + Task(env="lab", id="frame", args={"result": {"score": True}}), + ScriptedAgent("x"), + errored(IsStr(regex=r"\[grading\] .*result must include a numeric 'score'")), + ["initialize", "shutdown"], + ), + "a subscore without a value fails grading": Case( + Task(env="lab", id="frame", args={"result": {"score": 1.0, "subscores": [{"name": "m"}]}}), + ScriptedAgent("x"), + errored(IsStr(regex=r"(?s)\[grading\] .*ValidationError: .*SubScore\nvalue\n.*")), + ["initialize", "shutdown"], + ), + "malformed bindings fail the start": Case( + Task(env="lab", id="claim", args={"published": {"robot": "slot-2"}}), + ScriptedAgent("x"), + failed( + "[starting task] TypeError: task start frame 'bindings' must map capability " + "name -> object, got {'robot': 'slot-2'}" + ), + ["initialize", "shutdown"], + ), + "args over the frame limit fail the start before the template runs": Case( + Task(env="lab", id="large", args={"criteria": "x" * (16 * 1024 * 1024)}), + ScriptedAgent("x"), + failed( + IsStr( + regex=r"(?s)\[starting task\] .*'tasks\.start' request.*" + r"limit is 16777216 bytes.*file ID.*" + ) + ), + ["initialize", "shutdown"], + ), + "args over the frame limit once JSON-escaped fail the start": Case( + Task(env="lab", id="large", args={"criteria": "é" * (16 * 1024 * 1024 // 6 + 1)}), + ScriptedAgent("x"), + failed(IsStr(regex=r"(?s)\[starting task\] .*limit is 16777216 bytes.*")), + ["initialize", "shutdown"], + ), + "a provider that fails reports its notes": Case( + add(), + ScriptedAgent(solve), + failed("[provisioning] EOFError: sandbox closed\nenv output: ImportError: bugs"), + provider="broken", + ), +} + + +@pytest.mark.parametrize("case", CASES.values(), ids=CASES.keys()) +async def test_a_rollout_always_returns_a_run_that_says_where_it_failed( + case: Case, tmp_path: Path +) -> None: + events: list[str] = [] + env = lab(events) + source = tmp_path / "env.py" + source.write_text(SUMS_SOURCE) + providers: dict[str, Provider] = { + "live": LocalRuntime(env), + "constructor": LocalRuntime(lambda task: env), + "subprocess": SubprocessRuntime(source), + "hanging": hanging_provider, + "broken": broken_provider(noted("sandbox closed", "env output: ImportError: bugs")), + } - assert run.reward == 1.0 - assert received == ["secret"] + run = await rollout( + case.task, + case.agent, + runtime=providers[case.provider], + rollout_timeout=case.rollout_timeout, + ) + await eventually(lambda: events[-1:] in ([], ["shutdown"])) + + assert { + "status": run.trace.status, + "stop_reason": run.trace.stop_reason, + "reward": run.reward, + "raw": run.grade.raw, + "error": run.trace.error, + "prompt": run.prompt_text if run.prompt is not None else None, + "runtime": run.runtime, + "spans": [step["source"] for step in steps(run.trace_id)], + } == case.expected + assert events == case.events + assert (run.trace_id, run.job_id, run.slug) == (HEX_ID, HEX_ID, case.task.slug) + + +def chat_row(messages: list[Any]) -> Task: + return Task(env="lab", id="chat", args={"messages": messages}) + + +PROMPTS = { + "text is one user turn": (add(), [("user", "add 2 3")], "add 2 3"), + "no prompt is one empty user turn": (Task(env="lab", id="silent"), [("user", "")], ""), + "chat turns keep their roles and system becomes user": ( + chat_row( + [ + {"role": "system", "content": "be brief"}, + {"role": "assistant", "content": {"type": "text", "text": "hi"}}, + {"role": "user", "content": "add 2 3"}, + ] + ), + [("user", "be brief"), ("assistant", "hi"), ("user", "add 2 3")], + "be brief\n\nhi\n\nadd 2 3", + ), + "non-text turns are dropped from the text": ( + chat_row( + [ + {"role": "user", "content": "first"}, + {"role": "user", "content": {"type": "image", "data": "aW1n", "mimeType": "x"}}, + "second", + ] + ), + [("user", "first"), ("user", None), ("user", "second")], + "first\n\nsecond", + ), +} -async def test_malformed_subscores_fail_inside_the_rollout_boundary( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize(("row", "messages", "text"), PROMPTS.values(), ids=PROMPTS.keys()) +async def test_the_agent_sees_the_prompt_as_turns_and_as_text( + row: Task, messages: list[tuple[str, str | None]], text: str ) -> None: - from hud.clients import HudClient + seen: list[tuple[list[tuple[str, str | None]], str]] = [] - env = Environment("malformed-grade") + class Reader(Agent): + async def __call__(self, run: Run) -> None: + turns = [ + (message.role, getattr(message.content, "text", None)) + for message in run.prompt_messages + ] + seen.append((turns, run.prompt_text)) - @env.template() - async def solve(): - yield "answer" - yield 1.0 + run = await rollout(row, Reader(), runtime=LocalRuntime(lab())) - async def malformed_grade( - _client: HudClient, - _payload: dict[str, Any], - ) -> dict[str, Any]: - return {"score": 1.0, "subscores": [{"name": "missing-value"}]} + assert seen == [(messages, text)] + assert run.trace.status == "completed" - reported: list[dict[str, Any]] = [] - async def report(run: Run) -> None: - reported.append(run.evaluation) +async def test_episode_bindings_from_the_start_frame_reach_the_agent() -> None: + seen: list[tuple[dict[str, Any], str | None]] = [] - monkeypatch.setattr(HudClient, "grade", malformed_grade) - monkeypatch.setattr("hud.eval.run.trace_exit", report) + class Inspector(Agent): + async def __call__(self, run: Run) -> None: + seen.append((run.bindings, get_current_trace_id())) run = await rollout( - Task(env="malformed-grade", id="solve"), - _FnAgent(lambda _prompt: "done"), - runtime=LocalRuntime(env), + Task(env="lab", id="claim", args={"published": {"robot": {"token": "slot-2"}}}), + Inspector(), + runtime=LocalRuntime(lab()), ) - assert run.trace.is_error - assert "value" in (run.trace.error or "") - assert reported == [{}] - - -async def test_independent_verifier_receives_runtime_session_files(tmp_path: Path) -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - transfers: list[tuple[str, str]] = [] - - @actor_env.template() - async def solve(): - yield "answer secret" - yield {"score": 0.0} - - @verifier_env.template() - async def verify(): - yield "" - yield 1.0 - - class ActorRuntime(Runtime): - @asynccontextmanager - async def snapshot_session(self, session_id: str) -> AsyncIterator[Path | None]: - destination = tmp_path / "session.tar.gz" - await asyncio.to_thread(destination.write_text, session_id, encoding="utf-8") - transfers.append(("actor", session_id)) - yield destination - - class VerifierRuntime(Runtime): - async def restore_session(self, session_id: str, source: Path) -> None: - content = await asyncio.to_thread(source.read_text, encoding="utf-8") - assert content.startswith("sess-") - transfers.append(("verifier", session_id)) + assert seen == [({"robot": {"token": "slot-2"}}, run.trace_id)] + assert run.bindings == {"robot": {"token": "slot-2"}} - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - env = actor_env if row.env == "actor" else verifier_env - async with LocalRuntime(env)(row) as runtime: - yield ( - ActorRuntime(runtime.url) if row.env == "actor" else VerifierRuntime(runtime.url) - ) - task = Task( - env="actor", - id="solve", - verifier=Task(env="judge", id="verify"), +@pytest.mark.e2e +@pytest.mark.sandbox +async def test_connections_reach_the_agent_only_while_it_runs(tmp_path: Path) -> None: + env = lab() + env.workspace(tmp_path / "ws") + search = Connection( + name="search", + capability="shell", + url="https://search.example/v1", + headers={"Authorization": "Bearer secret"}, ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.reward == 1.0 - assert [runtime for runtime, _ in transfers] == ["actor", "verifier"] - assert transfers[0][1].startswith("sess-") - assert transfers[1][1].startswith("sess-") - assert transfers[0][1] != transfers[1][1] + seen: list[list[str]] = [] + class Inspector(Agent): + async def __call__(self, run: Run) -> None: + seen.append(sorted(run.connections)) + run.trace.content = "5" -async def test_same_env_verifier_runs_beside_an_actor_that_cannot_transport_its_session() -> None: - env = Environment("solo") - - @env.template() - async def solve(): - yield "answer secret" - yield {"score": 0.0} - - @env.template() - async def verify(): - yield "" - yield 1.0 + run = await rollout(add(), Inspector(), runtime=LocalRuntime(env), connections=[search]) - placements: list[str] = [] + assert seen == [["search"]] + assert (run.reward, run.connections) == (1.0, {}) - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.id) - async with LocalRuntime(env)(row) as runtime: - yield runtime - - verifier_config = RuntimeConfig(limits=RuntimeLimits(run_timeout_s=60)) - task = Task( - env="solo", - id="solve", - verifier=Task(env="solo", id="verify", runtime_config=verifier_config), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - assert run.reward == 1.0 - assert placements == ["solve"] +IDENTITIES = { + "a bare rollout mints its job and trace": ({}, (HEX_ID, None, HEX_ID)), + "threaded ids are kept": ( + {"job_id": "j1", "group_id": "g1", "trace_id": "0" * 31 + "1"}, + ("j1", "g1", "0" * 31 + "1"), + ), +} -async def test_same_env_verifier_gets_its_own_runtime_when_the_session_transports( - tmp_path: Path, +@pytest.mark.parametrize(("ids", "expected"), IDENTITIES.values(), ids=IDENTITIES.keys()) +async def test_a_run_carries_its_job_group_and_trace( + ids: dict[str, str], expected: tuple[Any, ...] ) -> None: - env = Environment("solo") - - @env.template() - async def solve(): - yield "answer secret" - yield {"score": 0.0} - - @env.template() - async def verify(): - yield "" - yield 1.0 - - placements: list[str] = [] - - class TransportingRuntime(Runtime): - transports_sessions = True - - @asynccontextmanager - async def snapshot_session(self, session_id: str) -> AsyncIterator[Path | None]: - destination = tmp_path / "session.tar.gz" - await asyncio.to_thread(destination.write_text, session_id, encoding="utf-8") - yield destination - - async def restore_session(self, session_id: str, source: Path) -> None: - pass - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.id) - async with LocalRuntime(env)(row.model_copy(update={"runtime_config": None})) as runtime: - yield TransportingRuntime(runtime.url) - - verifier_config = RuntimeConfig(limits=RuntimeLimits(run_timeout_s=60)) - task = Task( - env="solo", - id="solve", - verifier=Task(env="solo", id="verify", runtime_config=verifier_config), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) + run = await rollout(add(), ScriptedAgent(solve), runtime=LocalRuntime(lab()), **ids) + assert (run.job_id, run.group_id, run.trace_id) == expected assert run.reward == 1.0 - assert placements == ["solve", "verify"] - - -async def test_runtime_session_transfer_rejects_invalid_ids(tmp_path: Path) -> None: - runtime = Runtime("tcp://127.0.0.1:8765") - - with pytest.raises(ValueError, match="single path component"): - async with runtime.snapshot_session("../actor"): - pass - with pytest.raises(ValueError, match="single path component"): - await runtime.restore_session("", tmp_path / "session.tar.gz") - -async def test_verifier_with_its_own_environment_is_placed_after_the_actor() -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - placements: list[str] = [] - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 +def blocking_teardown(env: Environment, release: asyncio.Event, stopped: list[str]) -> Provider: + """Places rows on ``env`` and holds teardown until ``release`` is set.""" @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(f"start:{row.env}") + async def provider(task: Task) -> AsyncIterator[Runtime]: try: - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)( - row - ) as runtime: + async with LocalRuntime(env)(task) as runtime: yield runtime finally: - placements.append(f"stop:{row.env}") - - task = Task( - env="actor", - id="solve", - verifier=Task(env="judge", id="verify"), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.reward == 1.0 - assert placements == ["start:actor", "stop:actor", "start:judge", "stop:judge"] - - -async def test_verifier_remains_authoritative_after_an_agent_error() -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - placements: list[str] = [] - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.env) - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)(row) as runtime: - yield runtime - - task = Task( - env="actor", - id="solve", - verifier=Task(env="judge", id="verify"), - ) - run = await rollout(task, _AnswerThenBoomAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.trace.is_error - assert "agent exploded after answering" in (run.trace.error or "") - assert run.reward == 1.0 - assert placements == ["actor", "judge"] - - -@pytest.mark.parametrize("agent_fails", [False, True]) -async def test_verifier_remains_authoritative_when_actor_grading_fails( - agent_fails: bool, -) -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - placements: list[str] = [] - - @actor_env.template() - async def solve(): - yield "answer secret" - raise RuntimeError("actor grade exploded") - - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.env) - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)(row) as runtime: - yield runtime - - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) - agent = ( - _AnswerThenBoomAgent(lambda _prompt: "secret") - if agent_fails - else _FnAgent(lambda _prompt: "secret") - ) - run = await rollout(task, agent, runtime=provider) - - assert run.trace.is_error - assert "actor grade exploded" in (run.trace.error or "") - assert run.reward == 1.0 - assert placements == ["actor", "judge"] + await release.wait() + stopped.append(task.env) + return provider -async def test_verifier_remains_authoritative_when_actor_grade_is_scoreless( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.clients import HudClient - - actor_env = Environment("actor") - verifier_env = Environment("judge") - placements: list[str] = [] - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.env) - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)(row) as runtime: - yield runtime - - original_grade = HudClient.grade - grade_calls = 0 - - async def return_scoreless_actor_grade( - self: HudClient, payload: dict[str, Any] - ) -> dict[str, Any]: - nonlocal grade_calls - grade_calls += 1 - if grade_calls == 1: - return {"reward": 1.0} - return await original_grade(self, payload) - - monkeypatch.setattr(HudClient, "grade", return_scoreless_actor_grade) - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) - - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - job = Job(id="actor-scoreless", name="actor-scoreless", runs=[run]) - - assert run.trace.is_error - assert "numeric 'score'" in (run.trace.error or "") - assert run.reward == 1.0 - assert run.grade.raw["score"] == 1.0 - assert placements == ["actor", "judge"] - assert job.errors == [] - - -async def test_verifier_provisioning_failure_leaves_the_run_ungraded() -> None: - actor_env = Environment("actor") - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - if row.env == "judge": - raise RuntimeError("verifier unavailable") - async with LocalRuntime(actor_env)(row) as runtime: - yield runtime - - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=provider) - - assert run.trace.is_error - assert "verifier unavailable" in (run.trace.error or "") - assert run.grade.raw == {} - assert run.reward == 0.0 - - -async def test_verifier_channel_failure_leaves_the_run_ungraded( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.clients import HudClient - - env = Environment("reviewed") - - @env.template() - async def solve(): - yield "answer secret" - yield 0.25 - - @env.template() - async def verify(): - yield "" - yield 1.0 - - original_grade = HudClient.grade - grade_calls = 0 - - async def fail_verifier_grade(self: HudClient, payload: dict[str, Any]) -> dict[str, Any]: - nonlocal grade_calls - grade_calls += 1 - if grade_calls == 2: - raise ConnectionError("channel died after a partial grade frame") - return await original_grade(self, payload) - - monkeypatch.setattr(HudClient, "grade", fail_verifier_grade) - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify"), - ) - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) - job = Job(id="verify-failed", name="verify-failed", runs=[run]) +async def test_a_rollout_deadline_returns_without_waiting_for_teardown() -> None: + release = asyncio.Event() + stopped: list[str] = [] - assert run.trace.is_error - assert "channel died after a partial grade frame" in (run.trace.error or "") - assert run.grade.raw == {} - assert job.errors == [run] - - -async def test_scoreless_verifier_frame_is_an_ungraded_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from hud.clients import HudClient - - env = Environment("reviewed") - - @env.template() - async def solve(): - yield "answer secret" - yield 0.25 - - @env.template() - async def verify(): - yield "" - yield 1.0 - - original_grade = HudClient.grade - grade_calls = 0 - - async def return_scoreless_frame(self: HudClient, payload: dict[str, Any]) -> dict[str, Any]: - nonlocal grade_calls - grade_calls += 1 - if grade_calls == 2: - return {"reward": 1.0} - return await original_grade(self, payload) - - monkeypatch.setattr(HudClient, "grade", return_scoreless_frame) - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify"), - ) - - run = await rollout(task, _FnAgent(lambda _prompt: "secret"), runtime=LocalRuntime(env)) - job = Job(id="verify-scoreless", name="verify-scoreless", runs=[run]) - - assert run.trace.is_error - assert "numeric 'score'" in (run.trace.error or "") - assert run.grade.raw == {} - assert job.errors == [run] - - -async def test_zero_score_verifier_grade_counts_an_errored_rollout() -> None: - env = Environment("reviewed") - - @env.template() - async def solve(): - yield "answer secret" - yield 0.25 - - @env.template() - async def verify(): - yield "" - yield 0.0 - - task = Task( - env="reviewed", - id="solve", - verifier=Task(env="reviewed", id="verify"), - ) - run = await rollout( - task, - _AnswerThenBoomAgent(lambda _prompt: "secret"), - runtime=LocalRuntime(env), - ) - job = Job(id="verify-zero", name="verify-zero", runs=[run]) - - assert run.trace.is_error - assert run.grade.raw["score"] == 0.0 - assert job.reward == 0.0 - assert job.errors == [] - - -def _bindings_env(published: Any) -> Environment: - """An env whose task publishes per-episode binding data alongside the prompt.""" - env = Environment("slots") - - @env.template() - async def claim(): - yield {"prompt": "go", "bindings": published} - yield 1.0 - - return env - - -async def test_episode_bindings_from_the_start_frame_reach_the_agent() -> None: - seen: dict[str, dict[str, Any]] = {} - - class _Claiming(Agent): - async def __call__(self, run: Any) -> None: - seen.update(run.bindings) - - env = _bindings_env({"robot": {"token": "slot-2"}}) - run = await rollout(Task(env="slots", id="claim"), _Claiming(), runtime=LocalRuntime(env)) - - # Episode-scoped connection data reaches the agent by capability name, - # without reading it back off the recorded setup step. - assert seen == {"robot": {"token": "slot-2"}} - assert run.bindings == {"robot": {"token": "slot-2"}} - - -async def test_a_malformed_bindings_frame_fails_the_rollout_loudly() -> None: - env = _bindings_env({"robot": "slot-2"}) # capability data must be an object run = await rollout( - Task(env="slots", id="claim"), _FnAgent(lambda _p: ""), runtime=LocalRuntime(env) + add(), + ScriptedAgent(solve), + runtime=blocking_teardown(lab(), release, stopped), + rollout_timeout=0.2, ) + assert stopped == [] + release.set() + await eventually(lambda: stopped == ["lab"]) - assert run.trace.status == "error" - assert "bindings" in str(run.trace.steps[-1].error) + assert (run.reward, run.trace.status, run.trace.stop_reason) == (1.0, "error", "timeout") + assert run.trace.error == "rollout timed out after 0.2s during actor cleanup" -async def test_openai_compatible_write_reaches_workspace_grader(tmp_path: Path) -> None: - workspace = tmp_path / "workspace" - report = workspace / "REPORT.md" - env = Environment("opencode_report") - env.workspace(workspace, guest_path=str(workspace)) +async def test_cancelling_a_rollout_cancels_the_task_and_propagates() -> None: + events: list[str] = [] + agent = ScriptedAgent(solve, delay=30) + pending = asyncio.create_task(rollout(add(), agent, runtime=LocalRuntime(lab(events)))) + await eventually(lambda: agent.prompts != []) - @env.initialize - async def seed() -> None: - workspace.mkdir(parents=True, exist_ok=True) - report.unlink(missing_ok=True) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + await eventually(lambda: events[-1:] == ["shutdown"]) - @env.template() - async def write_report(): - yield "Write PASS to REPORT.md." - yield 1.0 if report.exists() and report.read_text().strip() == "PASS" else 0.0 - - model_client = _FakeOpenAI( - [ - _chat_response( - "", - [_tool_call("write", json.dumps({"filePath": str(report), "content": "PASS"}))], - ), - _chat_response("done"), - ] - ) - agent = OpenAIChatAgent( - OpenAIChatConfig(model="qwen3.6-plus", model_client=model_client, max_steps=4) - ) + assert events == ["initialize", "start add 2 3", "end add", "shutdown"] - run = await rollout( - Task(env="opencode_report", id="write_report"), - agent, - runtime=LocalRuntime(env), - ) - assert run.reward == 1.0 - assert report.read_text() == "PASS" - tools = model_client.chat.completions.requests[0]["extra_body"]["tools"] - assert [tool["function"]["name"] for tool in tools] == [ - "bash", - "read", - "glob", - "grep", - "edit", - "write", - ] - - -async def test_tool_agent_timeout_stops_running_workspace_command_before_grading( - tmp_path: Path, +async def test_the_agent_deadline_kills_a_running_workspace_command_before_grading( + models: Models, hud_env: HudEnv, tmp_path: Path ) -> None: + hud_env.set(HUD_API_KEY="k") workspace = tmp_path / "workspace" - started = workspace / "started" - late = workspace / "late" - env = Environment("timeout_cleanup") + env = Environment("cleanup") env.workspace(workspace, guest_path=str(workspace)) @env.initialize async def seed() -> None: workspace.mkdir(parents=True, exist_ok=True) - started.unlink(missing_ok=True) - late.unlink(missing_ok=True) - # Timing contract, sized to survive loaded CI: the command must start - # within the 2s agent budget, still be running when that budget expires - # (it sleeps 3s), and the grading delay must outlast the sleep so a - # command that survived the timeout provably leaves `late` behind. + # The command outlives the deadline, and grading outlives the command, so a + # command that survived the deadline would leave ``late`` behind. @env.template() async def wait_for_cleanup(): yield "Start the requested command." - await asyncio.sleep(4.0) - yield 1.0 if started.exists() and not late.exists() else 0.0 - - model_client = _FakeOpenAI( - [ - _chat_response( - "", - [_tool_call("bash", '{"command":"touch started; sleep 3; touch late"}')], - ), - ] - ) - agent = OpenAIChatAgent( - OpenAIChatConfig( - model="qwen3.6-plus", - model_client=model_client, - max_steps=2, - timeout_seconds=2.0, - ) - ) - - run = await rollout( - Task(env="timeout_cleanup", id="wait_for_cleanup"), - agent, - runtime=LocalRuntime(env), - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert started.exists() - assert not late.exists() - assert run.reward == 1.0 - - -@pytest.mark.skipif(sys.platform == "win32", reason="POSIX process-group regression") -async def test_local_runtime_startup_failure_kills_spawned_children(tmp_path: Path) -> None: - env_file = tmp_path / "env.py" - env_file.write_text( - textwrap.dedent( - """ - import asyncio - from pathlib import Path - - from hud import Environment - - env = Environment("leaky") - - - @env.initialize - async def start_child(): - proc = await asyncio.create_subprocess_exec( - "sleep", - "120", - stdin=asyncio.subprocess.DEVNULL, - stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.DEVNULL, - ) - Path("child.pid").write_text(str(proc.pid), encoding="utf-8") - raise RuntimeError("startup boom") - """ - ), - encoding="utf-8", - ) - pid_file = tmp_path / "child.pid" - pid: int | None = None - - try: - with pytest.raises(RuntimeError, match="startup boom"): - async with SubprocessRuntime(env_file, ready_timeout=30.0)( - Task(env="leaky", id="noop") - ): - pass - pid = int(pid_file.read_text()) - assert await _wait_for_pid_inactive(pid) - finally: - if pid is not None and _pid_is_running(pid): - with contextlib.suppress(ProcessLookupError): - os.kill(pid, signal.SIGKILL) - - -async def test_mid_run_failure_keeps_the_real_run_and_its_evidence(env_file: Path) -> None: - def boom(prompt: str) -> str: - raise RuntimeError("agent exploded") - - run = await rollout(_add_task(2, 3), _FnAgent(boom), runtime=SubprocessRuntime(env_file)) - - assert run.trace.is_error - assert "agent exploded" in (run.trace.error or "") - assert run.trace_id is not None # failed runs still key a trajectory - # The session was live, so the receipt keeps the evidence: the prompt the - # agent saw and the runtime the rollout executed against. - assert run.prompt == "add:2:3" - assert run.runtime is not None - assert run.reward == 0.0 # graded best-effort, but the agent never answered → 0.0 - - -class _AnswerThenBoomAgent(Agent): - """Records a correct answer, then raises — a mid-run failure after the env - already has a gradable answer in hand.""" - - def __init__(self, fn: Any) -> None: - super().__init__() - self._fn = fn - - async def __call__(self, run: Any) -> None: - run.trace.content = self._fn(run.prompt) - raise RuntimeError("agent exploded after answering") - - -async def test_mid_run_failure_still_grades_best_effort(env_file: Path) -> None: - # The agent answers correctly, then fails. The env is still alive, so the - # run is graded best-effort: the reward is captured even though it errored. - run = await rollout( - _add_task(2, 3), _AnswerThenBoomAgent(_solve_add), runtime=SubprocessRuntime(env_file) - ) - - assert run.trace.is_error - assert "agent exploded after answering" in (run.trace.error or "") - assert run.reward == 1.0 # graded despite the failure - assert run.trace.status == "error" # the failure is preserved, not masked - - -class _SlowAgent(Agent): - """Answers, then hangs — to exercise the agent-loop timeout.""" - - def __init__(self, fn: Any) -> None: - super().__init__() - self._fn = fn - self.cancelled = asyncio.Event() - - async def __call__(self, run: Any) -> None: - run.trace.content = self._fn(run.prompt) - try: - await asyncio.sleep(30) - finally: - self.cancelled.set() - - -async def test_agent_loop_timeout_is_an_explicit_failure() -> None: - env = Environment("sums") - task_cancelled = asyncio.Event() - - @env.template() - async def add(a: int, b: int): - try: - yield f"add:{a}:{b}" - yield 1.0 - finally: - task_cancelled.set() - - agent = _SlowAgent(_solve_add) - run = await rollout( - _add_task(2, 3), - agent, - runtime=LocalRuntime(env), - rollout_timeout=0.2, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert run.grade.raw == {} - await asyncio.wait_for(agent.cancelled.wait(), 1.0) - await asyncio.wait_for(task_cancelled.wait(), 1.0) - assert run.trace_id is not None - - -async def test_task_agent_timeout_still_grades_completed_work() -> None: - env = Environment("sums") - - @env.template() - async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - agent = _SlowAgent(_solve_add) - task = _add_task(2, 3).model_copy(update={"agent_config": {"timeout_seconds": 0.05}}) - - run = await rollout(task, agent, runtime=LocalRuntime(env)) - - assert run.reward == 1.0 - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert any("agent timed out" in (step.error or "") for step in run.trace.steps) - assert agent.cancelled.is_set() - job = Job(id="timeout-job", name="timeout", runs=[run]) - assert job.reward == 1.0 - assert job.errors == [] - - -@pytest.mark.parametrize("agent_timeout", [None, 10.0]) -async def test_agent_timeout_error_is_not_the_phase_deadline(agent_timeout: float | None) -> None: - env = Environment("sums") - - @env.template() - async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - class TimeoutAgent(Agent): - async def __call__(self, run: Any) -> None: - run.trace.content = _solve_add(run.prompt) - raise TimeoutError("provider timed out") - - task = _add_task(2, 3).model_copy( - update={"agent_config": {"timeout_seconds": agent_timeout} if agent_timeout else None} - ) - run = await rollout(task, TimeoutAgent(), runtime=LocalRuntime(env)) - - assert run.reward == 1.0 - assert run.trace.status == "error" - assert run.trace.stop_reason != "timeout" - assert "provider timed out" in (run.trace.error or "") - assert not any("agent timed out after" in (step.error or "") for step in run.trace.steps) - - -async def test_timeout_includes_grading() -> None: - env = Environment("sums") - grading_cancelled = asyncio.Event() - never = asyncio.Event() - - @env.template() - async def add(a: int, b: int): - yield f"add:{a}:{b}" - try: - await never.wait() - finally: - grading_cancelled.set() - yield 1.0 - - run = await rollout( - _add_task(2, 3), - _FnAgent(_solve_add), - runtime=LocalRuntime(env), - rollout_timeout=0.2, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert run.grade.raw == {} - await asyncio.wait_for(grading_cancelled.wait(), 1.0) - - -async def test_timeout_aborts_when_cancel_rpc_hangs(monkeypatch: pytest.MonkeyPatch) -> None: - env = Environment("sums") - agent_started = asyncio.Event() - cancel_started = asyncio.Event() - - @env.template() - async def add(a: int, b: int): - yield f"add:{a}:{b}" - await asyncio.Event().wait() - - class WaitingAgent(Agent): - async def __call__(self, run: Run) -> None: - agent_started.set() - await asyncio.Event().wait() - - aborted: list[bool] = [] - - async def hang_cancel(self: HudClient) -> None: - cancel_started.set() - await asyncio.Event().wait() - - real_abort = HudClient.abort - real_wait = asyncio.wait - - def track_abort(self: HudClient) -> None: - aborted.append(True) - real_abort(self) - - async def wait_after_agent_starts( - tasks: set[asyncio.Task[None]], **kwargs: Any - ) -> tuple[set[asyncio.Task[None]], set[asyncio.Task[None]]]: - # Exercise cancel/abort after connection setup, independent of startup speed. - await asyncio.wait_for(agent_started.wait(), timeout=5.0) - return await real_wait(tasks, **kwargs) - - monkeypatch.setattr(HudClient, "cancel", hang_cancel) - monkeypatch.setattr(HudClient, "abort", track_abort) - monkeypatch.setattr(run_module.asyncio, "wait", wait_after_agent_starts) - - run = await asyncio.wait_for( - rollout( - _add_task(2, 3), - WaitingAgent(), - runtime=LocalRuntime(env), - rollout_timeout=0.2, - ), - timeout=10.0, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert cancel_started.is_set() - assert aborted - - -async def test_timeout_does_not_wait_for_provider_cleanup() -> None: - env = Environment("sums") - cleanup_started = asyncio.Event() - cleanup_finished = asyncio.Event() - release_cleanup = asyncio.Event() - - @env.template() - async def add(a: int, b: int): - yield f"add:{a}:{b}" - yield 1.0 - - @asynccontextmanager - async def provider(_task: TaskRow) -> AsyncIterator[Runtime]: - try: - async with LocalRuntime(env)(_task) as runtime: - yield runtime - finally: - cleanup_started.set() - await release_cleanup.wait() - cleanup_finished.set() - - run = await rollout( - _add_task(2, 3), - _FnAgent(_solve_add), - runtime=provider, - rollout_timeout=0.2, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert run.reward == 1.0 - assert cleanup_started.is_set() - assert not cleanup_finished.is_set() - - release_cleanup.set() - await asyncio.wait_for(cleanup_finished.wait(), 1.0) - - -async def test_timeout_during_actor_cleanup_does_not_start_the_verifier() -> None: - actor_env = Environment("actor") - cleanup_started = asyncio.Event() - cleanup_finished = asyncio.Event() - release_cleanup = asyncio.Event() - placements: list[str] = [] - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} + await asyncio.sleep(2.0) + yield 1.0 if (workspace / "started").exists() and not (workspace / "late").exists() else 0.0 - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - placements.append(row.env) - if row.env == "judge": - raise AssertionError("verifier started after the rollout returned") - try: - async with LocalRuntime(actor_env)(row) as runtime: - yield runtime - finally: - cleanup_started.set() - await release_cleanup.wait() - cleanup_finished.set() - - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) - run = await rollout( - task, - _FnAgent(lambda _prompt: "secret"), - runtime=provider, - rollout_timeout=0.2, - ) - - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert placements == ["actor"] - release_cleanup.set() - await asyncio.wait_for(cleanup_finished.wait(), 1.0) - await asyncio.sleep(0) - assert placements == ["actor"] - - -async def test_timeout_does_not_cancel_verifier_provider_cleanup() -> None: - actor_env = Environment("actor") - verifier_env = Environment("judge") - cleanup_started = asyncio.Event() - cleanup_finished = asyncio.Event() - release_cleanup = asyncio.Event() - - @actor_env.template() - async def solve(): - answer = yield "answer secret" - yield {"score": 0.25, "answer": answer} + models.script([call("bash", command="touch started; sleep 1.5; touch late")]) + agent = OpenAIChatAgent(OpenAIChatConfig(model="scripted", max_steps=2, timeout_seconds=1.5)) - @verifier_env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "secret" else 0.0 - - @asynccontextmanager - async def provider(row: TaskRow) -> AsyncIterator[Runtime]: - try: - async with LocalRuntime(actor_env if row.env == "actor" else verifier_env)( - row - ) as runtime: - yield runtime - finally: - if row.env == "judge": - cleanup_started.set() - await release_cleanup.wait() - cleanup_finished.set() - - task = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) run = await rollout( - task, - _FnAgent(lambda _prompt: "secret"), - runtime=provider, - rollout_timeout=0.2, + Task(env="cleanup", id="wait_for_cleanup"), agent, runtime=LocalRuntime(env) ) - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - assert run.reward == 1.0 - assert cleanup_started.is_set() - assert not cleanup_finished.is_set() - - release_cleanup.set() - await asyncio.wait_for(cleanup_finished.wait(), 1.0) - - -async def test_pre_launch_failure_yields_a_synthesized_failed_run() -> None: - @asynccontextmanager - async def broken_provider(task: TaskRow) -> AsyncIterator[Runtime]: - raise RuntimeError("no substrate for you") - yield # pragma: no cover - - run = await rollout(_add_task(1, 1), _FnAgent(_solve_add), runtime=broken_provider) - - assert run.trace.is_error - assert "no substrate for you" in (run.trace.error or "") - assert run.trace_id is not None - assert run.prompt is None # nothing ever started - assert run.runtime is None - - -async def test_pre_launch_failure_keeps_the_notes_the_provider_attached() -> None: - # A provider attaches what only it can see — the env's output inside a remote - # sandbox — as a note, and str(exc) drops those on the way to the receipt. - @asynccontextmanager - async def broken_provider(task: TaskRow) -> AsyncIterator[Runtime]: - exc = EOFError("env closed connection during 'hello'") - exc.add_note("env output in sandbox ae964e25:\nModuleNotFoundError: No module named 'bugs'") - raise exc - yield # pragma: no cover - - run = await rollout(_add_task(1, 1), _FnAgent(_solve_add), runtime=broken_provider) - - assert "No module named 'bugs'" in (run.trace.error or "") - - -async def test_provider_is_called_with_the_task_row_being_placed(env_file: Path) -> None: - placed: list[str] = [] - - def placer(task: TaskRow) -> Any: - # The scheduler half of placement: the row is the request, so a - # provider can size/route each substrate per task. - placed.append(f"{task.env}/{task.id}:{task.args['a']}") - return SubprocessRuntime(env_file)(task) - - run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=placer) - - assert run.reward == 1.0 - assert placed == ["sums/add:2"] - - -async def test_task_run_schedules_a_single_task_job(env_file: Path) -> None: - job = await _add_task(2, 3).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(env_file)) - - (run,) = job.runs - assert job.reward == 1.0 - assert run.trace.content == "5" - assert run.job_id == job.id # the run's trace reports under the job - - -async def test_task_run_has_taskset_scheduling_semantics(env_file: Path) -> None: - job = await _add_task(1, 2).run( - _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file), group=2, max_concurrent=1 - ) - - assert job.group == 2 - assert [run.reward for run in job.runs] == [1.0, 1.0] - # The group repeats one task, so they share a GRPO group id. - assert len({run.group_id for run in job.runs}) == 1 - - -async def test_open_job_spans_multiple_scheduler_calls(env_file: Path) -> None: - session = await Job.start("session", group=2) - provider = SubprocessRuntime(env_file) - - job1 = await _add_task(1, 1).run(_FnAgent(_solve_add), runtime=provider, job=session) - job2 = await _add_task(2, 2).run(_FnAgent(_solve_add), runtime=provider, job=session) - - # Both calls accumulate into the one open job (group defaults to the job's). - assert job1 is session - assert job2 is session - assert len(session.runs) == 4 - assert {run.job_id for run in session.runs} == {session.id} - assert session.reward == 1.0 - - -_TWO_ENVS = """\ -from hud import Environment - -alpha = Environment("alpha") -beta = Environment("beta") - - -@alpha.template() -async def add_a(a: int, b: int): - answer = yield f"alpha:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - -@beta.template() -async def add_b(a: int, b: int): - answer = yield f"beta:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 -""" - - -async def test_one_spawn_serves_each_rows_env_in_a_mixed_taskset( - tmp_path_factory: pytest.TempPathFactory, -) -> None: - path = tmp_path_factory.mktemp("zoo") / "envs.py" - path.write_text(_TWO_ENVS, encoding="utf-8") - rows = [ - Task(env="alpha", id="add_a", args={"a": 1, "b": 2}), - Task(env="beta", id="add_b", args={"a": 3, "b": 4}), - ] - - # One provider, two envs: each acquisition serves the row it was called - # with (the task ids only exist on their own env, so a misplacement - # would fail the rollout). - job = await Taskset("zoo", rows).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(path)) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - assert [run.prompt for run in job.runs] == ["alpha:1:2", "beta:3:4"] - - -async def test_rollout_threads_job_and_group_ids(env_file: Path) -> None: - run = await rollout( - _add_task(1, 1), - _FnAgent(_solve_add), - runtime=SubprocessRuntime(env_file), - job_id="j1", - group_id="g1", - ) - - assert run.reward == 1.0 - assert run.job_id == "j1" - assert run.group_id == "g1" - - -@pytest.mark.parametrize( - ("ambient_trace_id", "trace_id", "expected_parent_trace_id"), - [ - ("parent-trace", "child-trace", "parent-trace"), - ( - "00000000-0000-0000-0000-000000000001", - "00000000000000000000000000000001", - None, - ), - ], -) -async def test_nested_rollout_binds_child_and_parent_trace_context( - monkeypatch: pytest.MonkeyPatch, - ambient_trace_id: str, - trace_id: str, - expected_parent_trace_id: str | None, -) -> None: - entered: dict[str, Any] = {} - active: dict[str, str | None] = {} - env = Environment("nested") - - @env.template(id="add") - async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - - async def capture_enter(trace_id: str, **kwargs: Any) -> None: - entered["trace_id"] = trace_id - entered.update(kwargs) - - class ContextAgent(Agent): - async def __call__(self, run: Run) -> None: - active["trace_id"] = get_current_trace_id() - active["parent_trace_id"] = get_trace_headers().get("X-HUD-Parent-Trace-Id") - run.trace.content = "2" - - monkeypatch.setattr(run_module, "trace_enter", capture_enter) - - with set_trace_context(ambient_trace_id): - run = await rollout( - Task(env="nested", id="add", args={"a": 1, "b": 1}), - ContextAgent(), - runtime=LocalRuntime(env), - job_id="job-1", - trace_id=trace_id, - ) - - assert run.trace_id == trace_id - assert active == { - "trace_id": trace_id, - "parent_trace_id": expected_parent_trace_id, - } - assert entered["trace_id"] == trace_id - assert entered["parent_trace_id"] == expected_parent_trace_id - - -# ─── Run prompt views (what agents consume) ─────────────────────────── - - -def _run_with_prompt(prompt: Any) -> Run: - run = Run(None, "t", {}) - run.prompt = prompt - return run - - -def test_prompt_messages_wraps_plain_text_as_one_user_turn() -> None: - (msg,) = _run_with_prompt("hello").prompt_messages - assert msg.role == "user" - assert isinstance(msg.content, mcp_types.TextContent) - assert msg.content.text == "hello" - - -def test_prompt_messages_no_prompt_is_one_empty_user_turn() -> None: - (msg,) = _run_with_prompt(None).prompt_messages - assert isinstance(msg.content, mcp_types.TextContent) - assert msg.content.text == "" - - -def test_prompt_messages_normalizes_chat_dicts_and_passes_through() -> None: - existing = mcp_types.PromptMessage( - role="assistant", content=mcp_types.TextContent(type="text", text="prior") - ) - msgs = _run_with_prompt( - [ - {"role": "user", "content": {"type": "text", "text": "hi"}}, - {"role": "system", "content": "be nice"}, # outside MCP vocab → user - existing, - ] - ).prompt_messages - assert [m.role for m in msgs] == ["user", "user", "assistant"] - assert msgs[2] is existing - - -def test_prompt_text_flattens_text_turns_and_drops_non_text() -> None: - image = mcp_types.PromptMessage( - role="user", - content=mcp_types.ImageContent(type="image", data="aGk=", mimeType="image/png"), - ) - run = _run_with_prompt([{"role": "user", "content": "first"}, image, "second"]) - assert run.prompt_text == "first\n\nsecond" - - -@pytest.mark.parametrize("character", ["x", "é"]) -async def test_oversized_task_args_become_trace_error_before_setup(character: str) -> None: - encoded_character_bytes = len(json.dumps(character)) - 2 - data = character * (16 * 1024 * 1024 // encoded_character_bytes + 1) - env = Environment("large-task") - started = False - - @env.template() - async def task(criteria: str): - nonlocal started - started = True - yield "ready" - yield 1.0 - - run = await rollout( - Task(env=env.name, id="task", args={"criteria": data}), - _FnAgent(lambda _: pytest.fail("agent must not launch")), - runtime=LocalRuntime(env), - ) - assert not started - assert run.trace.is_error - assert "[starting task]" in (run.trace.error or "") - assert "'tasks.start' request" in (run.trace.error or "") - assert "limit is 16777216 bytes" in (run.trace.error or "") - assert "file ID" in (run.trace.error or "") - assert "EOFError" not in (run.trace.error or "") + assert (run.reward, run.trace.status, run.trace.stop_reason) == (1.0, "error", "timeout") From 5eeba96c27a79b7f521ca7776b34807e19a68f7b Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 089/248] fix: export the failure step of a rollout that never launched Run.failed built its trace by hand, bypassing Trace.record, so a run that failed in provisioning, at task start or on a deadline before launch left no step span: the local span file showed nothing for it. It now records the step like every other failure. --- hud/eval/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hud/eval/run.py b/hud/eval/run.py index 151f20fba..464a41459 100644 --- a/hud/eval/run.py +++ b/hud/eval/run.py @@ -395,7 +395,8 @@ def failed(cls, error: str) -> Run: runtime, partial trace) with the error recorded on the trace. """ run = cls(None, "", {}) - run.trace = Trace(status="error", steps=[Step(source="system", error=error)]) + run.trace.status = "error" + run.record(Step(source="system", error=error)) return run From 3dd69217af1d9ad402b78624ee59cb4b4ec61934 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 090/248] test: rebuild local runtime sources as scenario tables Every source form (file, directory, sibling import, package, shared constructor, separate providers, a live env in a child) is a row asserting rewards and the env's own import and lifecycle log; provider construction and acquisition errors are tables of the messages users see. The patched create_process_group_exec goes: a real child closes its stdout or exits. The child-exit row is flaky under load: stdout reaches EOF before the child is reaped, so the error says 'closed stdout' instead of its exit code. --- tests/eval/test_local_runtime.py | 805 ++++++++++++++++--------------- 1 file changed, 405 insertions(+), 400 deletions(-) diff --git a/tests/eval/test_local_runtime.py b/tests/eval/test_local_runtime.py index 5fa4fc682..6df689f5d 100644 --- a/tests/eval/test_local_runtime.py +++ b/tests/eval/test_local_runtime.py @@ -1,197 +1,206 @@ -"""LocalRuntime serves environments from live instances and source recipes.""" +"""Local providers serve an environment from the source a user has: a file, a directory, +a package, a live instance, or a constructor, in this process or in a child process.""" from __future__ import annotations import asyncio +import os +import signal +import subprocess import sys -from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime -from pathlib import Path -from typing import Any, cast +import textwrap +from typing import TYPE_CHECKING, Any, cast import pytest -import hud.eval.runtime.local as local_runtime_module -from hud.agents.base import Agent from hud.environment import Environment, load_environment from hud.eval import LocalRuntime, RuntimeConfig, SubprocessRuntime, Task, Taskset +from tests.eval.envs import eventually, lab, solve +from tests.harness import ScriptedAgent -_SUMS_ENV = """\ -from hud import Environment +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path -env = Environment("{name}") + from hud.eval import Provider +LOGGED_ENV = """ +import asyncio +from pathlib import Path -@env.template(id="add") -async def add(a: int, b: int): - answer = yield f"add:{{a}}:{{b}}" - yield 1.0 if answer == str(a + b) else 0.0 -""" +from hud import Environment +LOG = Path({log!r}) +with LOG.open("a") as log: + log.write("import\\n") -def _sums_env(name: str = "sums") -> Environment: - env = Environment(name) +env = Environment("{name}") - @env.template(id="add") - async def add(a: int, b: int) -> AsyncGenerator[Any, Any]: - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 - return env +@env.initialize +async def start(): + with LOG.open("a") as log: + log.write("start\\n") + await asyncio.sleep(0) -class _FnAgent(Agent): - """Stateless agent: answers each run by applying ``fn`` to ``run.prompt``.""" +@env.shutdown +async def stop(): + with LOG.open("a") as log: + log.write("stop\\n") - def __init__(self, fn: Any) -> None: - super().__init__() - self._fn = fn - async def __call__(self, run: Any) -> None: - run.trace.content = self._fn(run.prompt) +@env.template() +async def add(a: int, b: int): + answer = yield f"add {{a}} {{b}}" + yield 1.0 if answer == str(a + b) else 0.0 +""" -def _solve_add(prompt: str) -> str: - _, a, b = prompt.split(":") - return str(int(a) + int(b)) +class Project: + """A source tree on disk whose environment logs its imports and lifecycle.""" + def __init__(self, root: Path, request: pytest.FixtureRequest) -> None: + self.root = root + self.log = root / "lifecycle.log" + self.package = root / f"project_{abs(hash(root)):x}" + self.package.mkdir() + request.addfinalizer(self.forget) -# ─── LocalRuntime sources ────────────────────────────────────────────── + def module(self, name: str = "sums", *, env: str = "lab") -> Path: + path = self.package / f"{self.package.name}_{name}.py" + path.write_text(LOGGED_ENV.format(log=str(self.log), name=env)) + return path + def entry(self, text: str) -> Path: + path = self.package / "env.py" + path.write_text(text) + return path -async def test_source_path_serves_a_fresh_env_per_rollout(tmp_path) -> None: - # LOADS is module state: a fresh throwaway import per acquisition means - # every rollout sees exactly one load. - (tmp_path / "env.py").write_text( - "from hud import Environment\n\n" - "LOADS = []\n" - 'env = Environment("sums")\n\n\n' - '@env.template(id="add")\nasync def add(a: int, b: int):\n' - " LOADS.append(1)\n" - ' answer = yield f"add:{a}:{b}:{len(LOADS)}"\n' - " yield 1.0 if answer == str(a + b) else 0.0\n", - encoding="utf-8", - ) + def lines(self) -> list[str]: + return self.log.read_text().splitlines() if self.log.exists() else [] - def _solve(prompt: str) -> str: - _, a, b, loads = prompt.split(":") - assert loads == "1" - return str(int(a) + int(b)) + def forget(self) -> None: + for name, module in tuple(sys.modules.items()): + file = getattr(module, "__file__", None) + if file and str(file).startswith(str(self.package)): + del sys.modules[name] - job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run( - _FnAgent(_solve), - runtime=LocalRuntime(tmp_path / "env.py"), - group=2, - ) - assert [run.reward for run in job.runs] == [1.0, 1.0] +def file_source(project: Project) -> Provider: + return LocalRuntime(project.module()) -async def test_subprocess_runtime_streams_environment_output( - tmp_path, - capsys: pytest.CaptureFixture[str], -) -> None: - source = tmp_path / "env.py" - source.write_text( - "import sys\n\n" - "from hud import Environment\n\n" - 'env = Environment("sums")\n\n' - "@env.initialize\n" - "async def start():\n" - ' print("x" * 100_000, flush=True)\n' - ' print("environment booted", flush=True)\n' - ' print("y" * 100_000, file=sys.stderr, flush=True)\n' - ' print("environment warning", file=sys.stderr, flush=True)\n', - encoding="utf-8", - ) +def directory_source(project: Project) -> Provider: + project.module() + return LocalRuntime(project.package) - async with SubprocessRuntime(source)(Task(env="sums", id="add")): - pass - captured = capsys.readouterr() - assert "x" * 100_000 in captured.out - assert "environment booted" in captured.out - assert "y" * 100_000 in captured.err - assert "environment warning" in captured.err +def split_file_source(project: Project) -> Provider: + module = project.module().stem + return LocalRuntime(project.entry(f"from {module} import env\n")) -def test_subprocess_runtime_serves_a_live_env_from_its_template_file(tmp_path, request) -> None: - module_name = f"sums_env_{request.node.name}" - env_py = tmp_path / f"{module_name}.py" - env_py.write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8") - tasks_py = tmp_path / "tasks.py" - tasks_py.write_text(f"from {module_name} import add\n\ntasks = [add(a=2, b=3)]\n") - request.addfinalizer(lambda: sys.modules.pop(module_name, None)) +def package_source(project: Project) -> Provider: + module = project.module().stem + (project.package / "__init__.py").write_text("") + return LocalRuntime(project.entry(f"from .{module} import env\n")) - task = next(iter(Taskset.from_module(tasks_py))) - assert task._env is not None - provider = SubprocessRuntime(task._env) - assert provider.source == env_py.resolve() - assert provider.env == "sums" +def shared_constructor(project: Project) -> Provider: + env = load_environment(project.module()) + return LocalRuntime(lambda task: env) -def test_subprocess_runtime_rejects_env_without_templates() -> None: - with pytest.raises(ValueError, match="exactly one source file"): - SubprocessRuntime(Environment("demo")) +def separate_providers(project: Project) -> Provider: + env = load_environment(project.module()) + return lambda task: LocalRuntime(env)(task) -def test_subprocess_runtime_rejects_env_pin_for_live_env() -> None: - with pytest.raises(TypeError, match="env= applies only to source paths"): - SubprocessRuntime(_sums_env(), env="sums") +def subprocess_live_env(project: Project) -> Provider: + return SubprocessRuntime(load_environment(project.module())) -async def test_subprocess_runtime_fails_when_stdout_closes_before_serving( - tmp_path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - source = tmp_path / "env.py" - source.write_text("", encoding="utf-8") - output = asyncio.StreamReader() - output.feed_eof() - error = asyncio.StreamReader() - error.feed_data(b"stdout closed\n") - error.feed_eof() +SERIAL = ["start", "stop", "start", "stop"] +SOURCES = { + "a file is imported per acquisition": (file_source, 1, ["import"] * 2, SERIAL), + "a directory is imported once": (directory_source, 2, ["import"], SERIAL), + "a file importing its env from a sibling shares it": ( + split_file_source, + 2, + ["import"], + SERIAL, + ), + "a package entry with a relative import shares its env": ( + package_source, + 2, + ["import"], + SERIAL, + ), + "a constructor returning one instance shares it": (shared_constructor, 2, ["import"], SERIAL), + "separate providers over one instance share it": (separate_providers, 2, ["import"], SERIAL), + "a child process serves a live env from its template file": ( + subprocess_live_env, + 1, + ["import"] * 3, + SERIAL, + ), +} - class Process: - stdout = output - stderr = error - returncode = None - terminated = False - async def wait(self) -> int: - await asyncio.Event().wait() - return 0 +@pytest.mark.parametrize( + ("source", "concurrency", "imports", "lifecycle"), SOURCES.values(), ids=SOURCES.keys() +) +async def test_a_source_serves_its_rows_one_acquisition_at_a_time_per_instance( + source: Callable[[Project], Provider], + concurrency: int, + imports: list[str], + lifecycle: list[str], + tmp_path: Path, + request: pytest.FixtureRequest, +) -> None: + project = Project(tmp_path, request) - async def terminate(self) -> None: - self.terminated = True + job = await Task(env="lab", id="add", args={"a": 2, "b": 3}).run( + ScriptedAgent(solve), runtime=source(project), group=2, max_concurrent=concurrency + ) - process = Process() + assert [run.reward for run in job.runs] == [1.0, 1.0] + assert [line for line in project.lines() if line == "import"] == imports + assert [line for line in project.lines() if line != "import"] == lifecycle - async def create_process(*args: Any, **kwargs: Any) -> Any: - return process - monkeypatch.setattr(local_runtime_module, "create_process_group_exec", create_process) +async def test_one_source_serves_each_rows_own_environment( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + project = Project(tmp_path, request) + alpha, beta = project.module("alpha", env="alpha"), project.module("beta", env="beta") + source = project.entry( + f"from {alpha.stem} import env as alpha\nfrom {beta.stem} import env as beta\n" + ) + rows = [ + Task(env="alpha", id="add", args={"a": 1, "b": 2}), + Task(env="beta", id="add", args={"a": 3, "b": 4}), + ] + agent = ScriptedAgent(solve) - async def run() -> None: - async with SubprocessRuntime(source, ready_timeout=30)(Task(env="closed", id="noop")): - pass + job = await Taskset("mixed", rows).run(agent, runtime=LocalRuntime(source)) - with pytest.raises(RuntimeError, match=r"(?s)closed stdout.*stdout closed"): - await asyncio.wait_for(run(), timeout=1) - assert process.terminated is True + assert [(run.slug, run.reward) for run in job.runs] == [(row.slug, 1.0) for row in rows] + assert sorted(agent.prompts) == ["add 1 2", "add 3 4"] -async def test_constructor_builds_fresh_per_rollout_from_the_row() -> None: +async def test_distinct_instances_from_a_constructor_run_concurrently() -> None: built: list[str] = [] started = 0 all_started = asyncio.Event() - def env_for(task: Task) -> Environment: - built.append(task.env) - env = _sums_env(task.env) + def build(task: Task) -> Environment: + built.append(task.slug) + env = lab() @env.initialize - async def start() -> None: + async def barrier() -> None: nonlocal started started += 1 if started == 3: @@ -200,336 +209,332 @@ async def start() -> None: return env - job = await Task(env="sums", id="add", args={"a": 1, "b": 2}).run( - _FnAgent(_solve_add), - runtime=LocalRuntime(env_for, ready_timeout=2), - group=3, - max_concurrent=3, - ) + row = Task(env="lab", id="add", args={"a": 1, "b": 2}) + job = await row.run(ScriptedAgent(solve), runtime=LocalRuntime(build, ready_timeout=5), group=3) - assert all(run.reward == 1.0 for run in job.runs) - assert built == ["sums", "sums", "sums"] + assert [run.reward for run in job.runs] == [1.0, 1.0, 1.0] + assert built == [row.slug] * 3 -async def test_live_environment_is_served_serially() -> None: - env = Environment("sums") - active = 0 +async def test_a_template_can_import_a_sibling_module_while_it_runs(tmp_path: Path) -> None: + (tmp_path / "lazy_helper.py").write_text("SUFFIX = ' ok'\n") + source = tmp_path / "env.py" + source.write_text( + textwrap.dedent( + """ + from hud import Environment - @env.template(id="add") - async def add(a: int, b: int): - answer = yield f"add:{a}:{b}" - yield 1.0 if answer == str(a + b) else 0.0 + env = Environment("lab") - @env.initialize - async def _start() -> None: - nonlocal active - active += 1 - assert active == 1 - await asyncio.sleep(0) - @env.shutdown - async def _stop() -> None: - nonlocal active - active -= 1 + @env.template() + async def add(a: int, b: int): + import lazy_helper - job = await add(a=2, b=3).run( - _FnAgent(_solve_add), - group=2, - max_concurrent=2, + answer = yield f"add {a} {b}{lazy_helper.SUFFIX}" + yield 1.0 if answer == str(a + b) else 0.0 + """ + ) ) + agent = ScriptedAgent(lambda prompt: solve(prompt.removesuffix(" ok"))) - assert [run.reward for run in job.runs] == [1.0, 1.0] - assert active == 0 - - -@pytest.mark.parametrize( - "source", ["directory", "split_file", "package", "factory", "separate_providers"] -) -async def test_cached_environment_lifecycle_is_serialized(tmp_path, request, source) -> None: - module_name = f"cached_lifecycle_{source}" - project = tmp_path / module_name - project.mkdir() - if source == "package": - (project / "__init__.py").write_text("from .env import env\n") - events = tmp_path / "lifecycle.txt" - (project / f"{module_name}.py").write_text( - _SUMS_ENV.format(name="sums") - + f""" -import asyncio -from pathlib import Path -events = Path({str(events)!r}) - -@env.initialize -async def start(): - with events.open("a") as log: - log.write("start\\n") - await asyncio.sleep(0) - -@env.shutdown -async def stop(): - with events.open("a") as log: - log.write("stop\\n") -""" + job = await Task(env="lab", id="add", args={"a": 2, "b": 3}).run( + agent, runtime=LocalRuntime(source) ) - prefix = "." if source == "package" else "" - entrypoint = project / "env.py" - entrypoint.write_text(f"from {prefix}{module_name} import env\n") - - def cleanup() -> None: - for name, module in tuple(sys.modules.items()): - file = getattr(module, "__file__", None) - if file and Path(file).is_relative_to(project): - del sys.modules[name] - request.addfinalizer(cleanup) - if source in {"factory", "separate_providers"}: - env = load_environment(entrypoint) - runtime = LocalRuntime(lambda _task: env) - if source == "separate_providers": - runtime = lambda task: LocalRuntime(env)(task) - else: - runtime = LocalRuntime(project if source == "directory" else entrypoint) - job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run( - _FnAgent(_solve_add), runtime=runtime, group=2, max_concurrent=2 - ) + assert (job.reward, agent.prompts) == (1.0, ["add 2 3 ok"]) - assert [run.reward for run in job.runs] == [1.0, 1.0] - assert events.read_text().splitlines() == ["start", "stop", "start", "stop"] +def test_a_live_environment_serves_across_event_loops() -> None: + env = lab() + row = Task(env="lab", id="add", args={"a": 2, "b": 3}) -async def test_cancelling_a_waiting_acquisition_preserves_the_running_environment() -> None: - env = _sums_env() - active = False + rewards = [ + asyncio.run(row.run(ScriptedAgent(solve), runtime=LocalRuntime(env), group=2)).reward + for _ in range(2) + ] - @env.initialize - async def start() -> None: - nonlocal active - active = True + assert rewards == [1.0, 1.0] - @env.shutdown - async def stop() -> None: - nonlocal active - active = False - runtime = LocalRuntime(lambda _task: env) - task = Task(env="sums", id="add") +async def test_cancelling_a_waiting_acquisition_leaves_the_running_one_alone() -> None: + events: list[str] = [] + runtime = LocalRuntime(lab(events)) + row = Task(env="lab", id="add") async def acquire() -> None: - async with runtime(task): - pytest.fail("the environment already has an active acquisition") + async with runtime(row): + events.append("second acquisition") - async with runtime(task): + async with runtime(row): waiter = asyncio.create_task(acquire()) await asyncio.sleep(0) waiter.cancel() with pytest.raises(asyncio.CancelledError): await waiter - assert active - - assert not active - + assert events == ["initialize"] + + assert events == ["initialize", "shutdown"] + + +CONSTRUCTION_ERRORS = { + "an unsupported source": ( + lambda: LocalRuntime(cast("Any", 42)), + TypeError, + "LocalRuntime: expected a source path, an Environment, or a", + ), + "env= with a live env": ( + lambda: LocalRuntime(lab(), env="lab"), + TypeError, + "LocalRuntime: env= applies only to source paths", + ), + "env= with a constructor": ( + lambda: LocalRuntime(lambda task: lab(), env="lab"), + TypeError, + "LocalRuntime: env= applies only to source paths", + ), + "env= with a live env in a child": ( + lambda: SubprocessRuntime(lab(), env="lab"), + TypeError, + "SubprocessRuntime: env= applies only to source paths", + ), + "a live env without templates in a child": ( + lambda: SubprocessRuntime(Environment("bare")), + ValueError, + "SubprocessRuntime: Environment 'bare' must declare its @env.template tasks in " + "exactly one source file", + ), +} -def test_local_environment_can_run_in_successive_event_loops() -> None: - env = _sums_env() - task = Task(env="sums", id="add", args={"a": 2, "b": 3}) - for _ in range(2): - job = asyncio.run( - task.run(_FnAgent(_solve_add), runtime=LocalRuntime(env), group=2, max_concurrent=2) - ) - assert [run.reward for run in job.runs] == [1.0, 1.0] - - -async def test_serialized_task_does_not_retain_local_placement() -> None: - env = Environment("sums") +@pytest.mark.parametrize( + ("build", "error", "message"), CONSTRUCTION_ERRORS.values(), ids=CONSTRUCTION_ERRORS.keys() +) +def test_a_provider_rejects_a_source_it_cannot_serve( + build: Callable[[], object], error: type[Exception], message: str +) -> None: + with pytest.raises(error) as raised: + build() - @env.template(id="add") - async def add(a: int, b: int): - yield f"add:{a}:{b}" - yield 1.0 + assert str(raised.value).startswith(message) - task = add(a=2, b=3) - portable = Task.model_validate(task.model_dump()) - with pytest.raises(ValueError, match="no placement: pass runtime="): - await portable.run(_FnAgent(_solve_add)) +def write(path: Path, source: str) -> Path: + path.write_text(textwrap.dedent(source)) + return path -async def test_source_missing_env_name_fails_loudly(tmp_path) -> None: - (tmp_path / "env.py").write_text( - 'from hud import Environment\n\nenv = Environment("sums")\n', - encoding="utf-8", - ) - provider = LocalRuntime(tmp_path / "env.py") +ADD = Task(env="lab", id="add", args={"a": 2, "b": 3}) +IMAGE_ROW = ADD.model_copy(update={"runtime_config": RuntimeConfig(image="lab")}) - with pytest.raises(ValueError, match="no Environment named 'other'"): - async with provider(Task(env="other", id="add")): - pass +def failing_initialize(events: list[str]) -> Environment: + env = lab(events) -async def test_module_taskset_uses_factory_environments(tmp_path) -> None: - source = tmp_path / "tasks.py" - source.write_text( - _SUMS_ENV.format(name="sums") + "\ntasks = [add(a=2, b=3), add(a=4, b=5)]\n", - encoding="utf-8", - ) + @env.initialize + async def explode() -> None: + raise RuntimeError("daemon failed to start") - job = await Taskset.from_module(source).run(_FnAgent(_solve_add)) + return env - assert [run.reward for run in job.runs] == [1.0, 1.0] +def slow_initialize(events: list[str]) -> Environment: + env = lab(events) -async def test_tasks_module_uses_factory_environment(tmp_path, request) -> None: - module_name = f"sums_env_{request.node.name}" - (tmp_path / f"{module_name}.py").write_text( - _SUMS_ENV.format(name="sums"), - encoding="utf-8", - ) - tasks = tmp_path / "tasks.py" - tasks.write_text( - f"from {module_name} import add\n\ntasks = [add(a=2, b=3)]\n", - encoding="utf-8", - ) - request.addfinalizer(lambda: sys.modules.pop(module_name, None)) + @env.initialize + async def hang() -> None: + await asyncio.Event().wait() - job = await Taskset.from_module(tasks).run(_FnAgent(_solve_add)) + return env - assert job.reward == 1.0 +ACQUISITION_ERRORS: dict[ + str, tuple[Callable[[Path, list[str]], Provider], Task, Any, str, list] +] = { + "a source without the row's env": ( + lambda tmp, events: LocalRuntime(write(tmp / "env.py", "from hud import Environment\n")), + Task(env="other", id="add"), + ValueError, + r"no Environment named 'other' found in .*env\.py", + [], + ), + "a constructor returning something else": ( + lambda tmp, events: LocalRuntime(lambda task: cast("Any", 42)), + ADD, + TypeError, + r"LocalRuntime: constructor returned 42, not an Environment", + [], + ), + "a source running an event loop at import": ( + lambda tmp, events: LocalRuntime( + write(tmp / "env.py", "import asyncio\n\nasyncio.run(asyncio.sleep(0))\n") + ), + ADD, + RuntimeError, + r"the env source ran async code while being imported to place a rollout — guard " + r'top-level run calls with `if __name__ == "__main__":`', + [], + ), + "a row with runtime_config in process": ( + lambda tmp, events: LocalRuntime(lab(events)), + IMAGE_ROW, + ValueError, + r"LocalRuntime does not support task runtime_config", + [], + ), + "a row with runtime_config in a child": ( + lambda tmp, events: SubprocessRuntime(write(tmp / "env.py", "")), + IMAGE_ROW, + ValueError, + r"SubprocessRuntime does not support task runtime_config", + [], + ), + "a missing source for a child": ( + lambda tmp, events: SubprocessRuntime(tmp / "missing.py"), + ADD, + FileNotFoundError, + r"SubprocessRuntime: source not found: .*missing\.py", + [], + ), + "an initialize hook that raises still shuts down": ( + lambda tmp, events: LocalRuntime(failing_initialize(events)), + ADD, + RuntimeError, + r"daemon failed to start", + ["initialize", "shutdown"], + ), + "an initialize hook slower than ready_timeout still shuts down": ( + lambda tmp, events: LocalRuntime(slow_initialize(events), ready_timeout=0.1), + ADD, + TimeoutError, + None, + ["initialize", "shutdown"], + ), +} + + +@pytest.mark.filterwarnings("ignore:coroutine 'sleep' was never awaited:RuntimeWarning") +@pytest.mark.parametrize( + ("provider", "row", "error", "message", "lifecycle"), + ACQUISITION_ERRORS.values(), + ids=ACQUISITION_ERRORS.keys(), +) +async def test_a_failed_acquisition_names_the_mistake( + provider: Callable[[Path, list[str]], Provider], + row: Task, + error: type[Exception], + message: str | None, + lifecycle: list[str], + tmp_path: Path, +) -> None: + events: list[str] = [] -async def test_ad_hoc_taskset_requires_explicit_placement() -> None: - with pytest.raises(ValueError, match="no placement: pass runtime="): - await Taskset("sums", [Task(env="sums", id="add")]).run(_FnAgent(_solve_add)) + with pytest.raises(error, match=message): + async with provider(tmp_path, events)(row): + pytest.fail("the acquisition should not yield a runtime") + assert events == lifecycle -async def test_container_rows_start_their_image_by_default(monkeypatch) -> None: - """A row that names an image is placeable without ``runtime=``: it gets DockerRuntime.""" - import hud.eval.taskset as taskset_module - live = LocalRuntime(_sums_env()) - monkeypatch.setattr( - taskset_module, - "DockerRuntime", - lambda: lambda task: live(task.model_copy(update={"runtime_config": None})), - ) - portable = Task(env="sums", id="add", args={"a": 2, "b": 3}) - container = Task( - env="sums", id="add", args={"a": 4, "b": 5}, runtime_config=RuntimeConfig(image="sums") +def process_is_gone(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + status = subprocess.run( + ["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True, check=False ) - - job = await Taskset("sums", [container]).run(_FnAgent(_solve_add)) - assert [run.reward for run in job.runs] == [1.0] - - with pytest.raises(ValueError, match="no placement"): - await Taskset("sums", [container, portable]).run(_FnAgent(_solve_add)) + return status.stdout.strip().startswith("Z") -async def test_container_row_with_shared_verifier_is_placeable(monkeypatch) -> None: - """A verifier on the same env with no runtime of its own rides the actor's container.""" - import hud.eval.taskset as taskset_module - - env = _sums_env() +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX process groups") +async def test_a_child_that_dies_before_serving_reports_its_output_and_takes_its_children( + tmp_path: Path, +) -> None: + source = write( + tmp_path / "env.py", + """ + import asyncio + import sys + from pathlib import Path - @env.template(id="verify") - async def verify() -> AsyncGenerator[Any, Any]: - actor_grade = yield "" - yield 1.0 if actor_grade["score"] == 1.0 else 0.0 + from hud import Environment - live = LocalRuntime(env) - starts: list[str] = [] + env = Environment("leaky") - def docker(task: Task) -> Any: - starts.append(task.id) - return live(task.model_copy(update={"runtime_config": None})) - monkeypatch.setattr(taskset_module, "DockerRuntime", lambda: docker) - container = Task( - env="sums", - id="add", - args={"a": 4, "b": 5}, - runtime_config=RuntimeConfig(image="sums"), - verifier=Task(env="sums", id="verify"), + @env.initialize + async def start(): + child = await asyncio.create_subprocess_exec("sleep", "120") + Path("child.pid").write_text(str(child.pid)) + print("daemon starting", file=sys.stderr, flush=True) + raise SystemExit(3) + """, ) - job = await Taskset("sums", [container]).run(_FnAgent(_solve_add)) - assert [run.reward for run in job.runs] == [1.0] - assert starts == ["add"] # one container for actor and verifier - - -async def test_empty_taskset_needs_no_placement() -> None: - job = await Taskset("empty", []).run(_FnAgent(_solve_add)) - - assert job.runs == [] - - -def test_rejects_a_non_pointer_argument() -> None: - with pytest.raises(TypeError, match="expected a source path"): - LocalRuntime(cast("Any", 42)) - - -async def test_failed_startup_still_runs_shutdown_hooks() -> None: - env = _sums_env() - lifecycle: list[str] = [] - - @env.initialize - async def _up() -> None: - lifecycle.append("up") - - @env.shutdown - async def _down() -> None: - lifecycle.append("down") - - @env.initialize - async def _boom() -> None: - raise RuntimeError("daemon failed to start") - - with pytest.raises(RuntimeError, match="daemon failed to start"): - async with LocalRuntime(env)(Task(env="sums", id="add")): - pass - - assert lifecycle == ["up", "down"] - + with pytest.raises(RuntimeError) as raised: + async with SubprocessRuntime(source)(Task(env="leaky", id="noop")): + pytest.fail("the child should not serve") -async def test_template_can_lazily_import_a_sibling_module(tmp_path) -> None: - (tmp_path / "lazy_helper.py").write_text("ANSWER_SUFFIX = ':ok'\n", encoding="utf-8") - (tmp_path / "env.py").write_text( - "from hud import Environment\n\n" - 'env = Environment("sums")\n\n\n' - '@env.template(id="add")\nasync def add(a: int, b: int):\n' - " import lazy_helper\n" - ' answer = yield f"add:{a}:{b}{lazy_helper.ANSWER_SUFFIX}"\n' - " yield 1.0 if answer == str(a + b) else 0.0\n", - encoding="utf-8", + assert str(raised.value).startswith( + f"spawned env exited with code 3 before serving (source: {source}):\n" + ) + assert "daemon starting" in str(raised.value) + pid = int((tmp_path / "child.pid").read_text()) + try: + await eventually(lambda: process_is_gone(pid), within=5) + finally: + if not process_is_gone(pid): + os.kill(pid, signal.SIGKILL) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX file descriptors") +async def test_a_child_that_closes_stdout_before_serving_is_reported(tmp_path: Path) -> None: + source = write( + tmp_path / "env.py", + """ + import os + import sys + import time + + print("stdout closed", file=sys.stderr, flush=True) + os.close(1) + time.sleep(60) + """, ) - def _solve(prompt: str) -> str: - _, a, b, ok = prompt.split(":") - assert ok == "ok" - return str(int(a) + int(b)) + with pytest.raises( + RuntimeError, match=r"(?s)spawned env closed stdout before serving.*stdout closed" + ): + async with SubprocessRuntime(source)(Task(env="lab", id="add")): + pytest.fail("the child should not serve") - job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run( - _FnAgent(_solve), - runtime=LocalRuntime(tmp_path / "env.py"), - group=2, - max_concurrent=2, - ) - assert [run.reward for run in job.runs] == [1.0, 1.0] +async def test_a_child_streams_its_output_to_the_terminal( + tmp_path: Path, capfd: pytest.CaptureFixture[str] +) -> None: + source = write( + tmp_path / "env.py", + """ + import sys + from hud import Environment -# ─── seam defenses ───────────────────────────────────────────────────── + env = Environment("lab") -@pytest.mark.filterwarnings("ignore::RuntimeWarning") # the broken source leaks its coroutine -async def test_unguarded_run_call_in_source_names_the_mistake(tmp_path) -> None: - (tmp_path / "env.py").write_text( - "import asyncio\n\nfrom hud import Environment\n\n" - 'env = Environment("sums")\n\n' - "asyncio.run(asyncio.sleep(0))\n", - encoding="utf-8", + @env.initialize + async def start(): + print("x" * 100_000, flush=True) + print("environment booted", flush=True) + print("y" * 100_000, file=sys.stderr, flush=True) + print("environment warning", file=sys.stderr, flush=True) + """, ) - provider = LocalRuntime(tmp_path / "env.py") - with pytest.raises(RuntimeError, match='if __name__ == "__main__"'): - async with provider(Task(env="sums", id="add")): - pass + async with SubprocessRuntime(source)(Task(env="lab", id="add")): + pass + + out, err = capfd.readouterr() + assert "x" * 100_000 + "\nenvironment booted\n" in out + assert "y" * 100_000 + "\nenvironment warning\n" in err From 6049a07d0b2fc4d0a952568ce19df46dae9bfe3f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 091/248] fix: report a child's exit code when it dies before serving SubprocessRuntime checked the return code the moment stdout hit EOF, before the exited child was reaped, so a child that exited during startup was often reported as having closed stdout. It now waits briefly for the exit first. --- hud/eval/runtime/local.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hud/eval/runtime/local.py b/hud/eval/runtime/local.py index a383a732d..29a094bd2 100644 --- a/hud/eval/runtime/local.py +++ b/hud/eval/runtime/local.py @@ -251,6 +251,11 @@ def capture_error(text: str) -> None: finally: finish_stdout() if port is None: + # Stdout ends as the child exits; give it a moment to be reaped + # so the error reports its exit code rather than a closed pipe. + with contextlib.suppress(TimeoutError): + async with asyncio.timeout(1.0): + await proc.wait() await finish_output(stderr_drain) tail = "\n".join((*output_tail, *error_tail)).strip() detail = f":\n{tail}" if tail else " (no output captured)" From e60ed622975e4424241f88874aae19b785ae9589 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 092/248] test: rebuild verifier placement and failures as one scenario table Rows cover a verifier on its own env, on the actor's env with and without a runtime of its own, across two fake-docker containers whose session files really transfer, and through a one-wide pool; then each failure of the actor or the verifier. Each asserts the grade of record, the placement log, the evaluate step spans, Job.errors and the fallback warning. HudClient.grade is no longer patched: the fixture environments raise or yield scoreless frames. --- tests/eval/envs.py | 59 ++++-- tests/eval/test_verifier.py | 352 ++++++++++++++++++++++++++++++++++++ 2 files changed, 398 insertions(+), 13 deletions(-) create mode 100644 tests/eval/test_verifier.py diff --git a/tests/eval/envs.py b/tests/eval/envs.py index 300e8093b..ccd99d4b1 100644 --- a/tests/eval/envs.py +++ b/tests/eval/envs.py @@ -8,13 +8,19 @@ from __future__ import annotations import asyncio +import contextlib +from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Any from hud import Environment from hud.environment.env import current_session_id +from hud.eval import LocalRuntime, Task if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import AsyncIterator, Callable + from pathlib import Path + + from tests.harness import FakeDocker SUMS_SOURCE = """ from hud import Environment @@ -120,19 +126,21 @@ async def large(criteria: str): return env -def actor(answers: list[str], *, grade: str = "score", sessions: Any = None) -> Environment: - """The ``actor`` side of a verifier pair: ``solve`` grades 0.25 and carries the answer. +def actor( + env: Environment | None = None, *, grade: str = "score", sessions: Path | None = None +) -> Environment: + """Declare the actor side of a verifier pair: ``solve`` grades 0.25 and carries the answer. - ``grade="raise"`` makes its grading raise and ``grade="scoreless"`` yields a - frame without a score. With ``sessions``, the template writes the answer into - its control session's directory under that root, as a container would. + It goes on ``env``, or on a new ``actor`` environment. ``grade="raise"`` makes + its grading raise and ``grade="scoreless"`` yields a frame without a score. + With ``sessions``, the template writes the answer into its control session's + directory under that root, as an environment in a container would. """ - env = Environment("actor") + env = env or Environment("actor") @env.template() async def solve(): answer = yield "answer secret" - answers.append(str(answer)) if sessions is not None: session = sessions / "runtime" / "sessions" / str(current_session_id.get()) session.mkdir(parents=True) @@ -144,13 +152,16 @@ async def solve(): return env -def judge(*, verdict: str = "check", sessions: Any = None) -> Environment: - """The ``judge`` side: ``verify`` pays 1.0 when the actor's answer was ``secret``. +def judge( + env: Environment | None = None, *, verdict: str = "check", sessions: Path | None = None +) -> Environment: + """Declare the judge side: ``verify`` pays 1.0 when the actor's answer was ``secret``. - ``verdict`` picks a failure instead: ``"raise"``, ``"scoreless"`` or ``"zero"``. - With ``sessions``, it reads the answer from its restored session directory. + It goes on ``env``, or on a new ``judge`` environment. ``verdict`` picks a + failure instead: ``"raise"``, ``"scoreless"`` or ``"zero"``. With + ``sessions``, it reads the answer from its restored session directory. """ - env = Environment("judge") + env = env or Environment("judge") @env.template() async def verify(): @@ -168,3 +179,25 @@ async def verify(): yield 1.0 if result["answer"] == "secret" else 0.0 return env + + +@asynccontextmanager +async def containers( + fake_docker: FakeDocker, rootfs: Path, images: dict[str, Environment] +) -> AsyncIterator[None]: + """Make each image a container the fake docker starts, served in this process. + + ``docker run `` answers a container named after the image, ``docker + port`` answers the address its environment serves on here, and ``docker + cp``/``exec`` work on ``/``. + """ + fake_docker.on(r"^(cp|exec) ", rootfs=rootfs) + async with contextlib.AsyncExitStack() as stack: + for image, env in images.items(): + runtime = await stack.enter_async_context( + LocalRuntime(env)(Task(env=env.name, id="serve")) + ) + address = runtime.url.removeprefix("tcp://") + fake_docker.on(rf"^run .* {image}$", stdout=f"{image}\n") + fake_docker.on(rf"^port {image} 8765$", stdout=f"{address}\n") + yield diff --git a/tests/eval/test_verifier.py b/tests/eval/test_verifier.py new file mode 100644 index 000000000..6cc95c24a --- /dev/null +++ b/tests/eval/test_verifier.py @@ -0,0 +1,352 @@ +"""Verifier tasks: the grade of record comes from a second, agent-less task. + +The actor (``solve``) grades 0.25 and hands its answer on; the judge (``verify``) +pays 1.0 only when that answer was ``secret``. A reward of 1.0 therefore proves +the actor's result reached the verifier, through its grade frame or, across +containers, through the session files the runtime transferred. +""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import pytest +from dirty_equals import IsStr +from pydantic import BaseModel + +from hud import Environment +from hud.eval import ( + DockerRuntime, + LocalRuntime, + Runtime, + RuntimeConfig, + RuntimeLimits, + Shared, + Task, +) +from tests.eval.envs import actor, containers, eventually, judge +from tests.harness import RecordingProvider, ScriptedAgent, steps + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + from pathlib import Path + + from hud.eval import Provider + from tests.harness import FakeDocker + +OWN_LIMITS = RuntimeConfig(limits=RuntimeLimits(run_timeout_s=20)) + + +def by_env(providers: dict[str, Provider]) -> Provider: + """Place each row with the provider for its environment.""" + + def place(task: Task) -> Any: + return providers[task.env](task) + + return place + + +def portable(provider: Provider) -> Provider: + """Place rows without their runtime_config, as a provider that cannot size them would.""" + + def place(task: Task) -> Any: + return provider(task.model_copy(update={"runtime_config": None})) + + return place + + +@asynccontextmanager +async def unavailable(task: Task) -> AsyncIterator[Runtime]: + raise RuntimeError("verifier unavailable") + yield Runtime("tcp://127.0.0.1:1") + + +@dataclass +class Case: + placement: str + expected: dict[str, Any] + agent: ScriptedAgent + grade: str = "score" + verdict: str = "check" + + +SEPARATE = ["start:solve", "stop:solve", "start:verify", "stop:verify"] +SHARED = ["start:solve", "stop:solve"] +EVALUATED = ["solve", "verify"] + + +def graded(**changes: Any) -> dict[str, Any]: + return { + "reward": 1.0, + "raw": {"score": 1.0}, + "status": "completed", + "error": None, + "placements": SEPARATE, + "evaluated": EVALUATED, + "in_errors": False, + "fallback_warning": False, + } | changes + + +def ungraded(error: Any, **changes: Any) -> dict[str, Any]: + return graded(reward=0.0, raw={}, status="error", error=error, in_errors=True) | changes + + +ANSWER = ScriptedAgent("secret") +CASES = { + "a verifier on its own env is placed after the actor is torn down": Case( + "separate", graded(), ANSWER + ), + "a verifier on the actor's env without a runtime shares its substrate": Case( + "same env", graded(placements=SHARED), ANSWER + ), + "a verifier with its own runtime runs beside an actor that cannot transport its session": ( + Case( + "same env, own runtime", + graded(placements=SHARED, fallback_warning=True), + ANSWER, + ) + ), + "session files reach a verifier in another container": Case("containers", graded(), ANSWER), + "session files reach a verifier in a fresh container of the same env": Case( + "same env, own container", graded(), ANSWER + ), + "a one-wide pool releases the actor's lease before the verifier takes one": Case( + "pooled container", graded(placements=["start:solve", "stop:solve"]), ANSWER + ), + "the verifier stays authoritative when the agent raised after answering": Case( + "separate", + graded(status="error", error="[agent loop] RuntimeError: agent exploded"), + ScriptedAgent("secret", fail_after=RuntimeError("agent exploded")), + ), + "the verifier stays authoritative when actor grading raised": Case( + "separate", + graded( + status="error", + error=IsStr(regex=r"\[grading\] .*actor grade exploded"), + evaluated=["verify"], + ), + ANSWER, + grade="raise", + ), + "the verifier stays authoritative over a scoreless actor frame": Case( + "separate", + graded( + status="error", + error=IsStr(regex=r"\[grading\] .*numeric 'score'.*"), + evaluated=["verify"], + ), + ANSWER, + grade="scoreless", + ), + "a verifier that cannot be placed leaves the run ungraded": Case( + "verifier unavailable", + ungraded( + "[provisioning verifier] RuntimeError: verifier unavailable", + placements=["start:solve", "stop:solve", "start:verify", "stop:verify"], + evaluated=["solve"], + ), + ANSWER, + ), + "a verifier that raises leaves the run ungraded": Case( + "separate", + ungraded(IsStr(regex=r"\[verifying\] .*verifier exploded"), evaluated=["solve"]), + ANSWER, + verdict="raise", + ), + "a scoreless verifier frame leaves the run ungraded": Case( + "separate", + ungraded(IsStr(regex=r"\[verifying\] .*numeric 'score'.*"), evaluated=["solve"]), + ANSWER, + verdict="scoreless", + ), + "a zero verdict on an errored run still counts toward the job": Case( + "separate", + graded( + reward=0.0, + raw={"score": 0.0}, + status="error", + error="[agent loop] RuntimeError: agent exploded", + ), + ScriptedAgent("secret", fail_after=RuntimeError("agent exploded")), + verdict="zero", + ), +} + + +def arrange(case: Case, rootfs: Path) -> tuple[Task, Provider, dict[str, Environment]]: + """The row, the provider placing it, and the container images it needs.""" + if case.placement in {"separate", "verifier unavailable"}: + row = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) + judge_provider = ( + unavailable + if case.placement == "verifier unavailable" + else LocalRuntime(judge(verdict=case.verdict)) + ) + provider = by_env({"actor": LocalRuntime(actor(grade=case.grade)), "judge": judge_provider}) + return row, provider, {} + if case.placement in {"same env", "same env, own runtime"}: + env = judge(actor(Environment("reviewed"), grade=case.grade), verdict=case.verdict) + own = OWN_LIMITS if case.placement == "same env, own runtime" else None + row = Task( + env="reviewed", + id="solve", + verifier=Task(env="reviewed", id="verify", runtime_config=own), + ) + return row, portable(LocalRuntime(env)), {} + if case.placement == "containers": + row = Task( + env="actor", + id="solve", + runtime_config=RuntimeConfig(image="actor"), + verifier=Task(env="judge", id="verify", runtime_config=RuntimeConfig(image="judge")), + ) + images = { + "actor": actor(sessions=rootfs / "actor"), + "judge": judge(sessions=rootfs / "judge"), + } + return row, DockerRuntime(), images + image = RuntimeConfig(image="reviewed") + row = Task( + env="reviewed", + id="solve", + runtime_config=image, + verifier=Task(env="reviewed", id="verify", runtime_config=image), + ) + sessions = rootfs / "reviewed" + env = judge(actor(Environment("reviewed"), sessions=sessions), sessions=sessions) + return row, DockerRuntime(), {"reviewed": env} + + +@pytest.mark.parametrize("case", CASES.values(), ids=CASES.keys()) +async def test_the_verifier_grade_is_the_grade_of_record( + case: Case, + tmp_path: Path, + fake_docker: FakeDocker, + caplog: pytest.LogCaptureFixture, +) -> None: + rootfs = tmp_path / "rootfs" + row, inner, images = arrange(case, rootfs) + recording = RecordingProvider(inner) + provider: Provider = ( + Shared(recording, width=1) if case.placement == "pooled container" else recording + ) + + async with containers(fake_docker, rootfs, images): + job = await row.run(case.agent, runtime=provider, rollout_timeout=30) + (run,) = job.runs + + assert { + "reward": run.reward, + "raw": run.grade.raw, + "status": run.trace.status, + "error": run.trace.error, + "placements": [f"{event}:{slug}" for event, slug in recording.events], + "evaluated": [ + step["task_call"]["name"] + for step in steps(run.trace_id) + if step["source"] == "task" and step["task_call"]["phase"] == "evaluate" + ], + "in_errors": run in job.errors, + "fallback_warning": "verifier runtime_config ignored" in caplog.text, + } == case.expected + + +class ActorResult(BaseModel): + score: float + answer: str + + +async def test_the_verifier_receives_the_actor_grade_as_its_declared_type() -> None: + received: list[Any] = [] + env = actor(Environment("reviewed")) + + @env.template(returns=ActorResult) + async def verify(): + result = yield "" + received.append(result.content) + yield 1.0 + + row = Task(env="reviewed", id="solve", verifier=Task(env="reviewed", id="verify")) + job = await row.run(ScriptedAgent("secret"), runtime=LocalRuntime(env)) + + assert received == [ActorResult(score=0.25, answer="secret")] + assert job.reward == 1.0 + + +def held_teardown( + providers: dict[str, Provider], held: str, release: asyncio.Event, log: list[str] +) -> Provider: + """Places rows by env; teardown of the ``held`` env waits for ``release``.""" + + @asynccontextmanager + async def place(task: Task) -> AsyncIterator[Runtime]: + log.append(f"start:{task.env}") + try: + async with providers[task.env](task) as runtime: + yield runtime + finally: + if task.env == held: + await release.wait() + log.append(f"stop:{task.env}") + + return place + + +DEADLINES: dict[str, tuple[str, float, str, list[str]]] = { + "a deadline in the actor's teardown never starts the verifier": ( + "actor", + 0.0, + "rollout timed out after 0.2s during actor cleanup", + ["start:actor", "stop:actor"], + ), + "a deadline in the verifier's teardown keeps the verdict": ( + "judge", + 1.0, + "rollout timed out after 0.2s during cleanup", + ["start:actor", "stop:actor", "start:judge", "stop:judge"], + ), +} + + +@pytest.mark.parametrize( + ("held", "reward", "error", "placements"), DEADLINES.values(), ids=DEADLINES.keys() +) +async def test_a_deadline_during_teardown_returns_and_lets_teardown_finish( + held: str, reward: float, error: str, placements: list[str] +) -> None: + release = asyncio.Event() + log: list[str] = [] + provider = held_teardown( + {"actor": LocalRuntime(actor()), "judge": LocalRuntime(judge())}, held, release, log + ) + row = Task(env="actor", id="solve", verifier=Task(env="judge", id="verify")) + + job = await row.run(ScriptedAgent("secret"), runtime=provider, rollout_timeout=0.2) + assert log[-1] == f"start:{held}" + release.set() + await eventually(lambda: log[-1] == f"stop:{held}") + await asyncio.sleep(0) + + (run,) = job.runs + assert (run.reward, run.trace.stop_reason, run.trace.error) == (reward, "timeout", error) + assert log == placements + + +SESSION_IDS = ["", ".", "..", "../actor", "a/b", "a\\b"] + + +@pytest.mark.parametrize("session_id", SESSION_IDS) +async def test_a_runtime_rejects_a_session_id_that_is_not_one_path_component( + session_id: str, tmp_path: Path +) -> None: + runtime = Runtime("tcp://127.0.0.1:1") + + with pytest.raises(ValueError, match="runtime session id must be a single path component"): + async with runtime.snapshot_session(session_id): + pass + with pytest.raises(ValueError, match="runtime session id must be a single path component"): + await runtime.restore_session(session_id, tmp_path) From 43728038de88af68954fb70c8ac80b9a3a8314dd Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 093/248] test: rebuild scheduling and pooling as scenario tables Task.run and Taskset.run are driven over plain, concurrency-bounded and pooled placements, asserting run order, group ids, job fields, placement counts and the peak number of agents in flight; one table holds every schedule the scheduler rejects before a rollout starts. Shared's lease API keeps its own rows: warm scopes, failed boots, run-timeout reboots, keying, teardown errors. Default placement runs live-env, module and container rows, the latter through the fake docker. test_shared.py's rows move here. --- tests/eval/test_scheduler.py | 417 +++++++++++++++++++++++++++++++++++ tests/eval/test_shared.py | 244 -------------------- 2 files changed, 417 insertions(+), 244 deletions(-) create mode 100644 tests/eval/test_scheduler.py delete mode 100644 tests/eval/test_shared.py diff --git a/tests/eval/test_scheduler.py b/tests/eval/test_scheduler.py new file mode 100644 index 000000000..71c8c9bd9 --- /dev/null +++ b/tests/eval/test_scheduler.py @@ -0,0 +1,417 @@ +"""The scheduler: ``Task.run`` and ``Taskset.run`` expand rows into rollouts, place them, +bound their concurrency and collect the runs into a ``Job``.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +import pytest + +from hud.agents.base import Agent +from hud.eval import ( + Job, + LocalRuntime, + ModalRuntime, + Runtime, + RuntimeConfig, + RuntimeLimits, + Shared, + Task, + Taskset, +) +from tests.eval.envs import containers, lab, solve +from tests.harness import RecordingProvider, ScriptedAgent + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + from pathlib import Path + + from hud.eval import Provider, Run + from tests.harness import FakeDocker, FakeServices + + +class Counting(Agent): + """Solves ``add`` rows after a pause, recording how many run at once.""" + + def __init__(self) -> None: + super().__init__() + self.live = 0 + self.peak = 0 + + async def __call__(self, run: Run) -> None: + self.live += 1 + self.peak = max(self.peak, self.live) + await asyncio.sleep(0.05) + run.trace.content = solve(run.prompt_text) + self.live -= 1 + + +ROWS = [ + Task(env="lab", id="add", args={"a": 1, "b": 2}), + Task(env="lab", id="add", args={"a": 3, "b": 4}), +] + + +def placements(events: list[tuple[str, str]]) -> tuple[int, int]: + """How many placements started, and the most open at once.""" + open_now = peak = 0 + for event, _slug in events: + open_now += 1 if event == "start" else -1 + peak = max(peak, open_now) + return sum(event == "start" for event, _ in events), peak + + +SCHEDULES: dict[str, tuple[Callable[[RecordingProvider], Provider], int | None, int, int, int]] = { + "every rollout gets its own placement": (lambda inner: inner, None, 6, 6, 6), + "max_concurrent bounds the rollouts in flight": (lambda inner: inner, 2, 6, 2, 2), + "a pool boots once and bounds occupancy by its width": ( + lambda inner: Shared(inner, width=2), + None, + 1, + 1, + 2, + ), +} + + +@pytest.mark.parametrize( + ("wrap", "max_concurrent", "boots", "open_at_once", "peak"), + SCHEDULES.values(), + ids=SCHEDULES.keys(), +) +async def test_a_taskset_runs_each_row_group_times_in_task_major_order( + wrap: Callable[[RecordingProvider], Provider], + max_concurrent: int | None, + boots: int, + open_at_once: int, + peak: int, + services: FakeServices, +) -> None: + recording = RecordingProvider(LocalRuntime(lambda task: lab())) + agent = Counting() + + job = await Taskset("pair", ROWS).run( + agent, runtime=wrap(recording), group=3, max_concurrent=max_concurrent + ) + + slugs = [row.slug for row in ROWS] + assert [run.slug for run in job.runs] == [slugs[0]] * 3 + [slugs[1]] * 3 + assert [run.reward for run in job.runs] == [1.0] * 6 + assert {run.job_id for run in job.runs} == {job.id} + first, second = ({run.group_id for run in job.results[slug]} for slug in slugs) + assert len(first) == len(second) == 1 and first != second + assert (job.name, job.group, job.reward) == ("pair (2 tasks) (3 times)", 3, 1.0) + assert placements(recording.events) == (boots, open_at_once) + assert recording.events[-1][0] == "stop" + assert agent.peak == peak + assert services.requests() == [] + + +async def test_an_open_job_collects_the_runs_of_several_calls() -> None: + session = await Job.start("training", group=2) + row = ROWS[0] + runtime = LocalRuntime(lab()) + + first = await row.run(ScriptedAgent(solve), runtime=runtime, job=session) + second = await row.run(ScriptedAgent(solve), runtime=runtime, job=session) + + assert first is second is session + assert [run.job_id for run in session.runs] == [session.id] * 4 + assert (session.name, session.group, session.reward) == ("training", 2, 1.0) + + +async def test_an_open_pool_stays_warm_across_calls_and_closes_with_its_scope() -> None: + recording = RecordingProvider(LocalRuntime(lab())) + + async with Shared(recording, width=2) as pool: + await Taskset("one", ROWS[:1]).run(ScriptedAgent(solve), runtime=pool) + await Taskset("two", ROWS[1:]).run(ScriptedAgent(solve), runtime=pool) + assert recording.events == [("start", ROWS[0].slug)] + + assert recording.events == [("start", ROWS[0].slug), ("stop", ROWS[0].slug)] + + +def flaky(inner: Provider) -> tuple[Provider, list[int]]: + """A provider whose first boot fails.""" + attempts: list[int] = [] + + @asynccontextmanager + async def place(task: Task) -> AsyncIterator[Runtime]: + attempts.append(len(attempts) + 1) + if len(attempts) == 1: + raise RuntimeError("boot failed") + async with inner(task) as runtime: + yield runtime + + return place, attempts + + +async def test_a_failed_boot_fails_one_lease_and_the_next_lease_retries() -> None: + provider, attempts = flaky(LocalRuntime(lab())) + + job = await ROWS[0].run( + ScriptedAgent(solve), runtime=Shared(provider, width=1), group=2, max_concurrent=1 + ) + + assert [(run.reward, run.trace.error) for run in job.runs] == [ + (0.0, "[provisioning] RuntimeError: boot failed"), + (1.0, None), + ] + assert attempts == [1, 2] + assert (job.reward, job.errors) == (1.0, [job.runs[0]]) + + +async def test_a_pool_reboots_a_substrate_past_its_run_timeout() -> None: + recording = RecordingProvider(LocalRuntime(lab())) + limited = RuntimeConfig(limits=RuntimeLimits(run_timeout_s=1)) + row = ROWS[0].model_copy(update={"runtime_config": limited}) + + def strip(task: Task) -> Any: + return recording(task.model_copy(update={"runtime_config": None})) + + async with Shared(strip, width=1) as pool: + async with pool(row): + pass + await asyncio.sleep(1.05) + async with pool(row): + pass + + assert [event for event, _ in recording.events] == ["start", "stop", "start", "stop"] + + +async def test_a_pool_keys_substrates_by_env_and_config_and_tears_them_down_in_reverse() -> None: + log: list[str] = [] + + @asynccontextmanager + async def provider(task: Task) -> AsyncIterator[Runtime]: + name = f"{task.env}:{task.runtime_config is not None}" + log.append(f"start {name}") + yield Runtime(f"tcp://127.0.0.1:1/{name}") + log.append(f"stop {name}") + + sized = RuntimeConfig(limits=RuntimeLimits(startup_timeout_s=5)) + async with Shared(provider, width=4) as pool: + urls = [] + for row in ( + Task(env="actor", id="solve"), + Task(env="judge", id="verify"), + Task(env="actor", id="solve", runtime_config=sized), + Task(env="actor", id="other"), + ): + async with pool(row) as runtime: + urls.append(runtime.url) + + assert urls == [ + "tcp://127.0.0.1:1/actor:False", + "tcp://127.0.0.1:1/judge:False", + "tcp://127.0.0.1:1/actor:True", + "tcp://127.0.0.1:1/actor:False", + ] + assert log == [ + "start actor:False", + "start judge:False", + "start actor:True", + "stop actor:True", + "stop judge:False", + "stop actor:False", + ] + + +async def test_a_pool_scope_raises_what_its_teardown_raised() -> None: + @asynccontextmanager + async def provider(task: Task) -> AsyncIterator[Runtime]: + yield Runtime("tcp://127.0.0.1:1") + raise RuntimeError("teardown boom") + + with pytest.raises(RuntimeError, match="teardown boom"): + async with Shared(provider, width=1) as pool, pool(ROWS[0]): + pass + + +async def test_a_pool_leased_outside_its_scope_says_so() -> None: + pool = Shared(LocalRuntime(lab()), width=1) + + with pytest.raises(RuntimeError, match="lease inside the scope"): + async with pool(ROWS[0]): + pass + + +def limited(**limits: int) -> RuntimeConfig: + return RuntimeConfig(limits=RuntimeLimits(**limits)) + + +def with_verifier(config: RuntimeConfig) -> Task: + return ROWS[0].model_copy( + update={"verifier": Task(env="judge", id="verify", runtime_config=config)} + ) + + +def timed(seconds: float, **row: Any) -> Task: + return ROWS[0].model_copy(update={"agent_config": {"timeout_seconds": seconds}, **row}) + + +INVALID: dict[str, tuple[Callable[[Provider], Any], str]] = { + "group below one": ( + lambda runtime: Taskset("t", ROWS).run(ScriptedAgent(), runtime=runtime, group=0), + "group must be >= 1", + ), + "max_concurrent below one": ( + lambda runtime: Taskset("t", ROWS).run(ScriptedAgent(), runtime=runtime, max_concurrent=0), + "max_concurrent must be >= 1", + ), + "a zero rollout_timeout": ( + lambda runtime: ROWS[0].run(ScriptedAgent(), runtime=runtime, rollout_timeout=0), + "rollout_timeout must be greater than 0", + ), + "a negative rollout_timeout": ( + lambda runtime: ROWS[0].run(ScriptedAgent(), runtime=runtime, rollout_timeout=-1), + "rollout_timeout must be greater than 0", + ), + "an agent timeout at the rollout_timeout": ( + lambda runtime: timed(10).run(ScriptedAgent(), runtime=runtime, rollout_timeout=10), + "agent timeout (10s) must be less than rollout_timeout (10s)", + ), + "an agent timeout past the row's run limit": ( + lambda runtime: timed(5000, runtime_config=limited(run_timeout_s=3600)).run( + ScriptedAgent(), runtime=runtime + ), + "agent timeout (5000s) must be less than runtime_config.limits.run_timeout_s (3600s)", + ), + "an agent timeout past the provider's run limit": ( + lambda runtime: timed(900).run( + ScriptedAgent(), + runtime=ModalRuntime("img", runtime_config=limited(run_timeout_s=600)), + ), + "agent timeout (900s) must be less than runtime_config.limits.run_timeout_s (600s)", + ), + "an actor run limit at the rollout_timeout": ( + lambda runtime: ( + ROWS[0] + .model_copy(update={"runtime_config": limited(run_timeout_s=3600)}) + .run(ScriptedAgent(), runtime=runtime, rollout_timeout=3600) + ), + "actor runtime_config.limits.run_timeout_s (3600s) must be less than " + "rollout_timeout (3600s)", + ), + "a provider startup limit at the rollout_timeout": ( + lambda runtime: ROWS[0].run( + ScriptedAgent(), + runtime=ModalRuntime("img", runtime_config=limited(startup_timeout_s=600)), + rollout_timeout=600, + ), + "actor runtime_config.limits.startup_timeout_s (600s) must be less than " + "rollout_timeout (600s)", + ), + "a verifier startup limit at the rollout_timeout": ( + lambda runtime: with_verifier(limited(startup_timeout_s=90)).run( + ScriptedAgent(), runtime=runtime, rollout_timeout=90 + ), + "verifier runtime_config.limits.startup_timeout_s (90s) must be less than " + "rollout_timeout (90s)", + ), + "duplicate slugs": ( + lambda runtime: Taskset("t", [ROWS[0], ROWS[0]]), + f"duplicate task slugs: {ROWS[0].slug}", + ), + "a pool narrower than one": ( + lambda runtime: Shared(runtime, width=0), + "Shared width must be >= 1", + ), + "a portable row without a runtime": ( + lambda runtime: Taskset("t", ROWS).run(ScriptedAgent()), + "no placement: pass runtime=", + ), + "a row minted by a template, serialized and reloaded": ( + lambda runtime: Task.model_validate(lab().tasks["add"](a=1, b=2).model_dump()).run( + ScriptedAgent() + ), + "no placement: pass runtime=", + ), + "container rows mixed with portable rows": ( + lambda runtime: Taskset( + "t", [ROWS[0], ROWS[1].model_copy(update={"runtime_config": RuntimeConfig(image="x")})] + ).run(ScriptedAgent()), + "no placement: pass runtime=", + ), +} + + +@pytest.mark.parametrize(("start", "message"), INVALID.values(), ids=INVALID.keys()) +async def test_an_invalid_schedule_raises_before_any_rollout_starts( + start: Callable[[Provider], Any], message: str, services: FakeServices +) -> None: + recording = RecordingProvider(LocalRuntime(lab())) + + with pytest.raises(ValueError) as raised: + outcome = start(recording) + if asyncio.iscoroutine(outcome): + await outcome + + assert str(raised.value).startswith(message) + assert recording.events == [] + assert services.requests() == [] + + +async def test_an_empty_taskset_needs_no_placement() -> None: + job = await Taskset("empty", []).run(ScriptedAgent()) + + assert (job.runs, job.reward, job.name) == ([], 0.0, "empty (0 tasks)") + + +async def test_rows_minted_by_a_live_environment_run_against_it() -> None: + events: list[str] = [] + env = lab(events) + row = env.tasks["add"](a=2, b=3) + + job = await Taskset("live", [row]).run(ScriptedAgent(solve), group=2, max_concurrent=2) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + assert events.count("initialize") == events.count("shutdown") == 2 + assert events[-1] == "shutdown" + + +async def test_a_module_taskset_runs_against_the_environments_it_defines(tmp_path: Path) -> None: + source = tmp_path / "tasks.py" + source.write_text( + "from hud import Environment\n\n" + 'env = Environment("lab")\n\n\n' + "@env.template()\n" + "async def add(a: int, b: int):\n" + ' answer = yield f"add {a} {b}"\n' + " yield 1.0 if answer == str(a + b) else 0.0\n\n\n" + "tasks = [add(a=2, b=3), add(a=4, b=5)]\n" + ) + + job = await Taskset.from_module(source).run(ScriptedAgent(solve)) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + + +CONTAINER_ROWS = { + "an image row starts its image": ( + ROWS[0].model_copy(update={"runtime_config": RuntimeConfig(image="lab")}), + 1.0, + ), + "a same-env verifier rides the actor's container": ( + ROWS[0].model_copy( + update={ + "runtime_config": RuntimeConfig(image="lab"), + "verifier": Task(env="lab", id="frame", args={"result": {"score": 0.5}}), + } + ), + 0.5, + ), +} + + +@pytest.mark.parametrize(("row", "reward"), CONTAINER_ROWS.values(), ids=CONTAINER_ROWS.keys()) +async def test_container_rows_start_their_image_without_a_runtime( + row: Task, reward: float, fake_docker: FakeDocker, tmp_path: Path +) -> None: + async with containers(fake_docker, tmp_path / "rootfs", {"lab": lab()}): + job = await Taskset("images", [row]).run(ScriptedAgent(solve)) + + assert job.reward == reward + assert [command.split()[-1] for command in fake_docker.commands("run ")] == ["lab"] diff --git a/tests/eval/test_shared.py b/tests/eval/test_shared.py deleted file mode 100644 index 71d744347..000000000 --- a/tests/eval/test_shared.py +++ /dev/null @@ -1,244 +0,0 @@ -"""``Shared``: one substrate leased to many concurrent rollouts. - -The contract: the substrate boots lazily on the first lease and lives for the -enclosing ``async with`` scope (one boot, deterministic teardown); ``width`` -bounds concurrent occupancy by making the next lease wait, not error; and -``Taskset.run`` scopes a context-manager placement to the call, so a bare -``runtime=Shared(...)`` needs no ceremony. -""" - -from __future__ import annotations - -import asyncio -import contextlib -from contextlib import asynccontextmanager -from typing import TYPE_CHECKING, Any - -import pytest - -from hud.agents.base import Agent -from hud.environment import Environment -from hud.eval import LocalRuntime, RuntimeConfig, Shared, Task, Taskset -from hud.eval.runtime import Runtime - -if TYPE_CHECKING: - from collections.abc import AsyncIterator - - from hud.eval.task import Task as TaskRow - - -class _CountingProvider: - """Wraps one local env, counting provisions and teardowns.""" - - def __init__(self, env: Environment) -> None: - self.runtime = LocalRuntime(env) - self.provisions = 0 - self.teardowns = 0 - - @asynccontextmanager - async def __call__(self, task: TaskRow) -> AsyncIterator[Runtime]: - self.provisions += 1 - try: - async with self.runtime(task) as runtime: - yield runtime - finally: - self.teardowns += 1 - - -def _echo_env() -> Environment: - env = Environment("pool") - - @env.template() - async def echo(tag: str = "x"): - answer = yield f"go {tag}" - yield 1.0 if answer == "ok" else 0.0 - - return env - - -class _OkAgent(Agent): - async def __call__(self, run: Any) -> None: - run.trace.content = "ok" - - -async def test_one_boot_serves_a_whole_grouped_run_and_closes_after() -> None: - inner = _CountingProvider(_echo_env()) - shared = Shared(inner, width=2) - - job = await Taskset("pool", [Task(env="pool", id="echo")]).run( - _OkAgent(), runtime=shared, group=3 - ) - - assert [run.reward for run in job.runs] == [1.0, 1.0, 1.0] - # All repeats shared one substrate, and one group_id (ordinary semantics). - assert inner.provisions == 1 - assert inner.teardowns == 1 # scoped to the run() call - assert len({run.group_id for run in job.runs}) == 1 - - -async def test_an_open_scope_keeps_the_substrate_warm_across_calls() -> None: - inner = _CountingProvider(_echo_env()) - taskset = Taskset("pool", [Task(env="pool", id="echo")]) - - async with Shared(inner, width=2) as shared: - await taskset.run(_OkAgent(), runtime=shared) - await taskset.run(_OkAgent(), runtime=shared) - assert inner.teardowns == 0 # still warm between calls - assert inner.provisions == 1 - assert inner.teardowns == 1 - - -async def test_distinct_task_placements_get_distinct_substrates() -> None: - provisions: list[str] = [] - teardowns: list[str] = [] - - @asynccontextmanager - async def provider(task: Task) -> AsyncIterator[Runtime]: - provisions.append(task.env) - try: - yield Runtime(f"tcp://{task.env}") - finally: - teardowns.append(task.env) - - async with Shared(provider, width=1) as shared: - async with shared(Task(env="actor", id="solve")) as actor: - assert actor.url == "tcp://actor" - async with shared(Task(env="verifier", id="verify")) as verifier: - assert verifier.url == "tcp://verifier" - - assert provisions == ["actor", "verifier"] - assert teardowns == ["verifier", "actor"] - - -async def test_width_one_releases_the_actor_lease_before_verifying() -> None: - env = Environment("reviewed") - - @env.template() - async def solve(): - answer = yield "answer" - yield {"score": 0.0, "answer": answer} - - @env.template() - async def verify(): - result = yield "" - yield 1.0 if result["answer"] == "ok" else 0.0 - - provisions = 0 - - @asynccontextmanager - async def provider(task: Task) -> AsyncIterator[Runtime]: - nonlocal provisions - provisions += 1 - async with LocalRuntime(env)(task.model_copy(update={"runtime_config": None})) as runtime: - yield runtime - - config = RuntimeConfig() - task = Task( - env="reviewed", - id="solve", - runtime_config=config, - verifier=Task( - env="reviewed", - id="verify", - runtime_config=config, - ), - ) - - job = await task.run( - _OkAgent(), - runtime=Shared(provider, width=1), - rollout_timeout=1, - ) - - assert job.reward == 1.0 - assert provisions == 1 - - -async def test_width_bounds_occupancy_by_waiting_not_erroring() -> None: - live = 0 - peak = 0 - - class _SlowAgent(Agent): - async def __call__(self, run: Any) -> None: - nonlocal live, peak - live += 1 - peak = max(peak, live) - await asyncio.sleep(0.05) - live -= 1 - run.trace.content = "ok" - - job = await Taskset("pool", [Task(env="pool", id="echo")]).run( - _SlowAgent(), runtime=Shared(_CountingProvider(_echo_env()), width=2), group=5 - ) - - assert [run.reward for run in job.runs] == [1.0] * 5 - assert peak <= 2 # lease 3 waited for a slot instead of raising - - -async def test_a_failed_boot_fails_one_lease_and_the_next_retries() -> None: - attempts = 0 - env = _echo_env() - - @asynccontextmanager - async def flaky(task: TaskRow) -> AsyncIterator[Runtime]: - nonlocal attempts - attempts += 1 - if attempts == 1: - raise RuntimeError("boot boom") - async with LocalRuntime(env)(task) as runtime: - yield runtime - - job = await Taskset("pool", [Task(env="pool", id="echo")]).run( - _OkAgent(), runtime=Shared(flaky, width=1), group=2 - ) - - # One rollout absorbed the boot failure; the retry served the other. - assert attempts == 2 - assert sorted(run.reward for run in job.runs) == [0.0, 1.0] - assert any(run.trace.is_error for run in job.runs) - - -async def test_leasing_outside_any_scope_errors_loudly() -> None: - shared = Shared(_CountingProvider(_echo_env()), width=1) - with pytest.raises(RuntimeError, match="scope"): - async with shared(Task(env="pool", id="echo")): - pass - - -async def test_scope_exit_without_any_lease_is_clean() -> None: - inner = _CountingProvider(_echo_env()) - async with Shared(inner, width=1): - pass - assert (inner.provisions, inner.teardowns) == (0, 0) - - -async def test_teardown_error_surfaces_at_scope_exit() -> None: - @asynccontextmanager - async def leaky(task: TaskRow) -> AsyncIterator[Runtime]: - try: - yield Runtime("tcp://127.0.0.1:1") - finally: - raise RuntimeError("teardown boom") - - with pytest.raises(RuntimeError, match="teardown boom"): - async with Shared(leaky, width=1) as shared: - async with shared(Task(env="pool", id="echo")): - pass - - -async def test_width_must_be_positive() -> None: - with pytest.raises(ValueError, match="width"): - Shared(_CountingProvider(_echo_env()), width=0) - - -async def test_non_pooling_placements_are_untouched_by_the_scope_hook() -> None: - # A plain provider is not a context manager; run() must not try to enter it. - inner = _CountingProvider(_echo_env()) - assert not isinstance(inner, contextlib.AbstractAsyncContextManager) - - job = await Taskset("pool", [Task(env="pool", id="echo")]).run( - _OkAgent(), runtime=inner, group=2 - ) - - assert [run.reward for run in job.runs] == [1.0, 1.0] - assert inner.provisions == 2 # one substrate per rollout, as before From e7166df39548a6101a40f6891bfc39bea5975adc Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 094/248] test: pin platform reporting on the wire against the fake services Job, trace-enter and trace-exit bodies are asserted as the HUD API receives them, through the real platform client: the job name for each way of running, the model the agent samples, parent traces, every outcome's exit body (errors, deadlines, metadata, summarized subscores, runs that never launched), the one retry and its warning, the telemetry-and-key gate, and a cancelled rollout reporting no outcome. test_job.py's patched PlatformClient.from_settings and hand-built runs go. --- tests/eval/test_job.py | 135 ------------- tests/eval/test_reporting.py | 360 +++++++++++++++++++++++++++++++++++ 2 files changed, 360 insertions(+), 135 deletions(-) delete mode 100644 tests/eval/test_job.py create mode 100644 tests/eval/test_reporting.py diff --git a/tests/eval/test_job.py b/tests/eval/test_job.py deleted file mode 100644 index eb0d2e8b0..000000000 --- a/tests/eval/test_job.py +++ /dev/null @@ -1,135 +0,0 @@ -"""``hud.eval.job`` reporting — the trace-exit payload sent to the platform. - -No network: the platform client is replaced with a recorder. -""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING, Any - -import pytest - -from hud.eval import job as job_mod -from hud.eval.run import Run - -if TYPE_CHECKING: - from collections.abc import Iterator - - -class _Recorder: - """Stand-in platform client that captures the last reported body.""" - - def __init__(self) -> None: - self.calls: list[tuple[str, dict[str, Any]]] = [] - - async def apost(self, path: str, *, json: dict[str, Any]) -> dict[str, Any]: - self.calls.append((path, json)) - return {} - - -@pytest.fixture -def recorder(monkeypatch: pytest.MonkeyPatch) -> Iterator[_Recorder]: - from hud.settings import settings - - monkeypatch.setattr(settings, "telemetry_enabled", True) - monkeypatch.setattr(settings, "api_key", "sk-hud-test") - rec = _Recorder() - monkeypatch.setattr(job_mod.PlatformClient, "from_settings", classmethod(lambda cls: rec)) - yield rec - - -def _run_with(trace_id: str, *, extra: dict[str, Any]) -> Run: - run = Run(None, "task", {}) - run.trace.trace_id = trace_id - run.trace.status = "completed" - run.trace.extra = extra - return run - - -async def test_trace_enter_reports_task_and_group_identity(recorder: _Recorder) -> None: - await job_mod.trace_enter( - "abc", - job_id="job-1", - group_id="group-1", - task_slug="fix-bug-3", - model="test-model", - parent_trace_id="parent-1", - ) - - assert recorder.calls == [ - ( - "/trace/abc/enter", - { - "job_id": "job-1", - "group_id": "group-1", - "task_slug": "fix-bug-3", - "model": "test-model", - "parent_trace_id": "parent-1", - }, - ) - ] - - -async def test_job_enter_logs_canonical_web_uuid( - recorder: _Recorder, - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - from hud.settings import settings - - compact_id = "03dd2a73d3df4d10a54ae3d87c2d530d" - canonical_id = "03dd2a73-d3df-4d10-a54a-e3d87c2d530d" - monkeypatch.setattr(settings, "hud_web_url", "https://hud.test") - caplog.set_level(logging.INFO, logger="hud.eval.job") - - await job_mod.job_enter(compact_id, name="test", group=1) - - assert recorder.calls[0][0] == f"/trace/job/{compact_id}/enter" - assert f"job: https://hud.test/jobs/{canonical_id}" in caplog.text - - -async def test_trace_exit_propagates_stop_reason(recorder: _Recorder) -> None: - run = _run_with("abc", extra={}) - run.trace.stop_reason = "max_steps" - await job_mod.trace_exit(run) - - assert len(recorder.calls) == 1 - path, body = recorder.calls[0] - assert path == "/trace/abc/exit" - assert body["stop_reason"] == "max_steps" - assert "metadata" not in body - - -async def test_trace_exit_omits_metadata_when_extra_empty(recorder: _Recorder) -> None: - await job_mod.trace_exit(_run_with("abc", extra={})) - - assert len(recorder.calls) == 1 - _, body = recorder.calls[0] - assert "metadata" not in body - - -def test_errored_runs_do_not_deflate_the_job_reward() -> None: - """Infrastructure failure is never a score: a run that errored (a launch - failure, or a hosted trace that ended in error) carries no verdict and - must not drag the job mean down as a silent zero.""" - from hud.eval.job import Job - from hud.eval.run import Grade - - graded = _run_with("t1", extra={}) - graded.grade = Grade(reward=1.0) - failed = Run.failed("provisioning never finished") - - job = Job(id="j1", name="test", runs=[graded, failed]) - - assert job.reward == 1.0 - assert job.errors == [failed] - - -def test_job_with_only_errors_reports_zero_reward() -> None: - from hud.eval.job import Job - - job = Job(id="j2", name="test", runs=[Run.failed("boom")]) - - assert job.reward == 0.0 - assert job.errors and job.errors[0].trace.is_error diff --git a/tests/eval/test_reporting.py b/tests/eval/test_reporting.py new file mode 100644 index 000000000..bacb2c845 --- /dev/null +++ b/tests/eval/test_reporting.py @@ -0,0 +1,360 @@ +"""Platform reporting: every rollout reports under a job, through the HUD API's job +and trace enter/exit endpoints, and only while telemetry is on with an API key.""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +import pytest +from dirty_equals import IsStr +from inline_snapshot import snapshot + +from hud.agents.base import Agent +from hud.agents.types import AgentConfig +from hud.eval import Job, LocalRuntime, Runtime, Task, Taskset, rollout +from hud.telemetry import flush +from hud.telemetry.context import get_trace_headers, set_trace_context +from tests.eval.envs import eventually, lab, solve +from tests.harness import Reply, ScriptedAgent + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable, Iterator + + from hud.eval import Run + from tests.harness import FakeServices, HudEnv + +HEX_ID = IsStr(regex=r"[0-9a-f]{32}") +ADD = Task(env="lab", id="add", args={"a": 2, "b": 3}) +JOB_ENTER = "/v2/trace/job/{id}/enter" +TRACE_ENTER = "/v2/trace/{id}/enter" +TRACE_EXIT = "/v2/trace/{id}/exit" + + +@pytest.fixture +def platform(services: FakeServices, hud_env: HudEnv) -> Iterator[FakeServices]: + """The fake platform accepting every report, with telemetry on and a key set. + + Span uploads drain before the test ends, so none reaches a later test's platform. + """ + hud_env.set(HUD_API_KEY="k", HUD_TELEMETRY_ENABLED="1", HUD_WEB_URL="https://hud.test") + for path in (JOB_ENTER, TRACE_ENTER, TRACE_EXIT): + services.route("api", "POST", path, json={}) + services.route("telemetry", "POST", "/trace/{id}/telemetry-upload", json={}) + yield services + assert flush(timeout=10) + + +class Modelled(Agent): + """Answers with ``answer(prompt)`` while sampling a named model.""" + + def __init__(self, answer: Callable[[str], str] = solve, **extra: Any) -> None: + super().__init__(AgentConfig(model="claude-test")) + self.answer = answer + self.extra = extra + self.headers: list[dict[str, str]] = [] + + async def __call__(self, run: Run) -> None: + self.headers.append(get_trace_headers()) + run.trace.content = self.answer(run.prompt_text) + run.trace.extra.update(self.extra) + + +async def test_a_taskset_reports_one_job_and_each_rollout_under_it( + platform: FakeServices, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level(logging.INFO, logger="hud.eval.job") + rows = [ADD, Task(env="lab", id="add", args={"a": 1, "b": 1})] + + job = await Taskset("pair", rows, taskset_id="ts-1").run( + Modelled(), runtime=LocalRuntime(lab()), group=2 + ) + + (job_enter,) = platform.requests("api", "POST", JOB_ENTER) + assert (job_enter.params["id"], job_enter.bearer) == (job.id, "k") + assert job_enter.json == snapshot( + {"name": "pair (2 tasks) (2 times)", "group": 2, "taskset_id": "ts-1"} + ) + assert f"job: https://hud.test/jobs/{uuid.UUID(job.id)}" in caplog.messages + enters = platform.requests("api", "POST", TRACE_ENTER) + assert sorted(request.params["id"] for request in enters) == sorted( + run.trace_id for run in job.runs + ) + assert sorted( + (request.json for request in enters), key=lambda body: body["task_slug"] + ) == sorted( + ( + {"job_id": job.id, "group_id": HEX_ID, "task_slug": row.slug, "model": "claude-test"} + for row in rows + for _ in range(2) + ), + key=lambda body: body["task_slug"], + ) + assert {body["job_id"] for body in platform.bodies("api", "POST", TRACE_ENTER)} == {job.id} + exits = platform.requests("api", "POST", TRACE_EXIT) + assert sorted(request.params["id"] for request in exits) == sorted( + run.trace_id for run in job.runs + ) + assert [request.json for request in exits] == [ + {"status": "completed", "reward": 1.0, "evaluation_result": {"score": 1.0}} + ] * 4 + + +async def bare(runtime: LocalRuntime) -> Any: + return await rollout(ADD, ScriptedAgent(solve), runtime=runtime) + + +async def threaded(runtime: LocalRuntime) -> Any: + return await rollout(ADD, ScriptedAgent(solve), runtime=runtime, job_id="j" * 32) + + +async def single(runtime: LocalRuntime) -> Any: + return await ADD.run(ScriptedAgent(solve), runtime=runtime) + + +async def grouped(runtime: LocalRuntime) -> Any: + return await ADD.run(ScriptedAgent(solve), runtime=runtime, group=2) + + +async def session(runtime: LocalRuntime) -> Any: + job = await Job.start("training", group=2) + await ADD.run(ScriptedAgent(solve), runtime=runtime, job=job) + return await ADD.run(ScriptedAgent(solve), runtime=runtime, job=job) + + +JOBS: dict[str, tuple[Callable[[LocalRuntime], Awaitable[Any]], list[dict[str, Any]]]] = { + "a bare rollout registers a job of one named after its task": ( + bare, + [{"name": "add", "group": 1}], + ), + "a rollout given a job registers none": (threaded, []), + "a single task's job is named after the task": (single, [{"name": "add", "group": 1}]), + "a grouped task's job counts its repeats": ( + grouped, + [{"name": "add (2 times)", "group": 2}], + ), + "an open job registers once for all its calls": ( + session, + [{"name": "training", "group": 2}], + ), +} + + +@pytest.mark.parametrize(("start", "bodies"), JOBS.values(), ids=JOBS.keys()) +async def test_each_execution_registers_the_job_its_rollouts_report_under( + start: Callable[[LocalRuntime], Awaitable[Any]], + bodies: list[dict[str, Any]], + platform: FakeServices, +) -> None: + await start(LocalRuntime(lab())) + + assert platform.bodies("api", "POST", JOB_ENTER) == bodies + job_ids = {request.params["id"] for request in platform.requests("api", "POST", JOB_ENTER)} + assert {body["job_id"] for body in platform.bodies("api", "POST", TRACE_ENTER)} == ( + job_ids or {"j" * 32} + ) + + +@asynccontextmanager +async def unavailable(task: Task) -> AsyncIterator[Runtime]: + raise RuntimeError("no substrate") + yield Runtime("tcp://127.0.0.1:1") + + +def subscored(score: float) -> Task: + subscores = [ + {"name": "format", "weight": 0.5, "value": 1.0, "info": {"seen": True}}, + {"name": "content", "weight": 0.5, "value": 0.0}, + ] + return Task(env="lab", id="frame", args={"result": {"score": score, "subscores": subscores}}) + + +EXITS: dict[str, tuple[Task, Agent, bool, dict[str, Any]]] = { + "a graded run": ( + ADD, + Modelled(), + True, + {"status": "completed", "reward": 1.0, "evaluation_result": {"score": 1.0}}, + ), + "a run whose agent raised": ( + ADD, + ScriptedAgent(solve, fail_after=RuntimeError("agent exploded")), + True, + { + "status": "error", + "reward": 1.0, + "error": "[agent loop] RuntimeError: agent exploded", + "evaluation_result": {"score": 1.0}, + }, + ), + "a run cut off by its agent deadline": ( + ADD.model_copy(update={"agent_config": {"timeout_seconds": 0.05}}), + ScriptedAgent(solve, linger=30), + True, + { + "status": "error", + "reward": 1.0, + "error": "agent timed out after 0.05s", + "evaluation_result": {"score": 1.0}, + "stop_reason": "timeout", + }, + ), + "a run carrying trajectory metadata": ( + ADD, + Modelled(turns=3), + True, + { + "status": "completed", + "reward": 1.0, + "evaluation_result": {"score": 1.0}, + "metadata": {"turns": 3}, + }, + ), + "a run graded with subscores": ( + subscored(0.5), + Modelled(lambda prompt: "x"), + True, + { + "status": "completed", + "reward": 0.5, + "evaluation_result": { + "score": 0.5, + "subscores": [ + {"name": "format", "weight": 0.5, "value": 1.0, "children": None}, + {"name": "content", "weight": 0.5, "value": 0.0, "children": None}, + ], + }, + }, + ), + "a run whose grading failed": ( + Task(env="lab", id="grade_raises"), + Modelled(lambda prompt: "x"), + True, + { + "status": "error", + "reward": 0.0, + "error": IsStr(regex=r"\[grading\] .*grader exploded"), + }, + ), + "a run that never launched": ( + ADD, + Modelled(), + False, + {"status": "error", "reward": 0.0, "error": "[provisioning] RuntimeError: no substrate"}, + ), +} + + +@pytest.mark.parametrize(("row", "agent", "placed", "body"), EXITS.values(), ids=EXITS.keys()) +async def test_a_finished_rollout_reports_its_outcome( + row: Task, agent: Agent, placed: bool, body: dict[str, Any], platform: FakeServices +) -> None: + run = await rollout(row, agent, runtime=LocalRuntime(lab()) if placed else unavailable) + + (exit_request,) = platform.requests("api", "POST", TRACE_EXIT) + assert exit_request.params["id"] == run.trace_id + assert exit_request.json == body + + +PARENTS = { + "a rollout inside another trace names it as its parent": ( + "a" * 32, + "b" * 32, + {"Trace-Id": "b" * 32, "X-HUD-Parent-Trace-Id": "a" * 32}, + "a" * 32, + ), + "an ambient trace that is the rollout's own is not its parent": ( + str(uuid.UUID("b" * 32)), + "b" * 32, + {"Trace-Id": "b" * 32}, + None, + ), +} + + +@pytest.mark.parametrize( + ("ambient", "trace_id", "headers", "parent"), PARENTS.values(), ids=PARENTS.keys() +) +async def test_a_nested_rollout_reports_and_propagates_its_parent_trace( + ambient: str, + trace_id: str, + headers: dict[str, str], + parent: str | None, + platform: FakeServices, +) -> None: + agent = Modelled() + + with set_trace_context(ambient): + run = await rollout(ADD, agent, runtime=LocalRuntime(lab()), trace_id=trace_id) + + (enter,) = platform.bodies("api", "POST", TRACE_ENTER) + assert (run.trace_id, agent.headers) == (trace_id, [headers]) + assert enter.get("parent_trace_id") == parent + + +RETRIES = { + "a rejected report is retried once": ([Reply(status=500), Reply(json={})], 2, []), + "a report rejected twice is given up with a warning": ( + [Reply(status=500)], + 2, + [IsStr(regex=r"platform report /trace/job/[0-9a-f]{32}/enter failed: .*500.*")], + ), + "an accepted report is sent once": ([Reply(json={})], 1, []), +} + + +@pytest.mark.parametrize(("replies", "attempts", "warnings"), RETRIES.values(), ids=RETRIES.keys()) +async def test_reporting_never_fails_the_run( + replies: list[Reply], + attempts: int, + warnings: list[Any], + platform: FakeServices, + caplog: pytest.LogCaptureFixture, +) -> None: + platform.route("api", "POST", JOB_ENTER, *replies) + + job = await Job.start("flaky") + + assert len(platform.requests("api", "POST", JOB_ENTER)) == attempts + assert [ + record.getMessage() + for record in caplog.records + if record.name == "hud.eval.job" and record.levelno == logging.WARNING + ] == warnings + assert job.name == "flaky" + + +GATES = { + "telemetry on without a key": {"HUD_TELEMETRY_ENABLED": "1", "HUD_API_KEY": None}, + "a key with telemetry off": {"HUD_TELEMETRY_ENABLED": "0", "HUD_API_KEY": "k"}, +} + + +@pytest.mark.parametrize("variables", GATES.values(), ids=GATES.keys()) +async def test_nothing_is_reported_unless_telemetry_is_on_with_a_key( + variables: dict[str, str | None], services: FakeServices, hud_env: HudEnv +) -> None: + hud_env.set(**variables) + + job = await ADD.run(ScriptedAgent(solve), runtime=LocalRuntime(lab())) + + assert job.reward == 1.0 + assert services.requests() == [] + + +async def test_a_cancelled_rollout_reports_its_start_but_no_outcome( + platform: FakeServices, +) -> None: + agent = ScriptedAgent(solve, delay=30) + pending = asyncio.create_task(rollout(ADD, agent, runtime=LocalRuntime(lab()))) + await eventually(lambda: agent.prompts != []) + + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + + assert len(platform.requests("api", "POST", TRACE_ENTER)) == 1 + assert platform.requests("api", "POST", TRACE_EXIT) == [] From ed45136a106070193b53f11c1c68c2b134e1e00a Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 095/248] test: drive hosted rollouts against the fake platform The submit body is snapshotted as the platform receives it through the real client (canonical ids, the agent spec without credentials, the inlined Compose document, explicit nulls, the verifier); parent traces, every terminal trace state and how it folds into the run and job, deadlines on a stuck poll and a stuck submit, local cancellation, the deprecated constructor timeout, and runs that cannot be submitted are rows. The patched PlatformClient.from_settings goes; Taskset timeout validation moved to the scheduler table, tunnel rows to their own file, and Agent.dump rows to the agents area. The deadline rows hang intermittently: httpx can swallow the cancellation asyncio.timeout delivers, and the poll loop never notices. --- tests/eval/test_hosted.py | 1254 +++++++++---------------------------- 1 file changed, 310 insertions(+), 944 deletions(-) diff --git a/tests/eval/test_hosted.py b/tests/eval/test_hosted.py index f5b3abe36..8988a72f9 100644 --- a/tests/eval/test_hosted.py +++ b/tests/eval/test_hosted.py @@ -1,1052 +1,418 @@ -"""HUD-hosted placement: agent spec, submission/polling, and scheduler dispatch. - -The hosted path never opens a local connection — :class:`HostedRuntime` submits the -rollout to the platform, polls the trace until terminal, and folds the result -into a ``Run``. The scheduler (:meth:`Taskset.run`) chooses between ``HostedRuntime`` -and a local provider. These tests fake the platform client at the -``PlatformClient`` seam, so they cover everything local: spec serialization, -payload shape, id canonicalization, terminal detection, timeout cancel, the -Run the caller gets back, and the dispatch. -""" +"""Hosted execution: the platform runs the whole rollout; this process submits it, +polls its trace to a terminal state, and folds the result into a ``Run``.""" from __future__ import annotations import asyncio -import json import uuid -from typing import TYPE_CHECKING, Any, ClassVar, cast +from typing import TYPE_CHECKING, Any import pytest +from dirty_equals import IsStr +from inline_snapshot import snapshot -from hud.agents.base import Agent from hud.agents.claude import ClaudeCLIAgent, ClaudeCLIConfig from hud.agents.openai_compatible import OpenAIChatAgent from hud.agents.types import OpenAIChatConfig -from hud.eval.job import Job -from hud.eval.run import Run -from hud.eval.runtime import ( +from hud.eval import ( ComposeProject, HostedRuntime, - HUDRuntime, - ModalRuntime, - Runtime, RuntimeConfig, RuntimeGPU, RuntimeLimits, RuntimeResources, + Task, + Taskset, ) -from hud.eval.runtime.core import resolve_runtime_config -from hud.eval.runtime.hud import _splice_websocket -from hud.eval.task import Task -from hud.eval.taskset import Taskset +from hud.telemetry.context import set_trace_context +from tests.eval.envs import eventually +from tests.harness import Reply, ScriptedAgent if TYPE_CHECKING: from pathlib import Path -from hud.settings import settings -from hud.telemetry.context import set_trace_context - - -class _FakePlatform: - """Scripted PlatformClient: records posts, serves trace states in order.""" - - api_key = "test-key" - - def __init__(self, states: list[dict[str, Any]]) -> None: - self.states = states - self.posts: list[tuple[str, dict[str, Any]]] = [] - self.polled = 0 - - async def apost(self, path: str, *, json: Any | None = None) -> Any: - self.posts.append((path, json or {})) - return {"status": "queued"} + from hud.agents.base import Agent + from tests.harness import FakeServices, HudEnv - async def aget(self, path: str, *, params: dict[str, Any] | None = None) -> Any: - assert path.startswith("/trace/") and path.count("/") == 2 - state = self.states[min(self.polled, len(self.states) - 1)] - self.polled += 1 - return state +SUBMIT = "/v2/rollouts/submit" +CANCEL = "/v2/rollouts/cancel" +POLL = "/v2/trace/{id}" +TRACE_ID = "1" * 32 +JOB_ID = "2" * 32 +ROW = Task(env="sums", id="add", args={"a": 1, "b": 2}) -class _FakeResponse: - def __init__(self, body: dict[str, Any]) -> None: - self.body = body +@pytest.fixture +def platform(services: FakeServices, hud_env: HudEnv) -> FakeServices: + """The fake platform accepting submissions, with a key set.""" + hud_env.set(HUD_API_KEY="k") + services.route("api", "POST", SUBMIT, json={"status": "queued"}) + services.route("api", "POST", CANCEL, json={}) + return services - def raise_for_status(self) -> None: - return None - def json(self) -> dict[str, Any]: - return self.body - - -def _agent() -> OpenAIChatAgent: +def chat_agent() -> OpenAIChatAgent: return OpenAIChatAgent( - OpenAIChatConfig(model="test-model", api_key="k", base_url="http://localhost") + OpenAIChatConfig(model="test-model", api_key="secret", base_url="http://model.test") ) -@pytest.mark.parametrize("runtime_type", [HostedRuntime, HUDRuntime]) -def test_runtime_constructor_timeout_is_a_deprecated_alias(runtime_type: type[Any]) -> None: - with pytest.warns(DeprecationWarning, match="rollout_timeout"): - runtime = runtime_type(run_timeout=90.0) - - assert runtime.run_timeout == 90.0 - - -def test_dump_serializes_full_config() -> None: - agent = _agent() - agent.config.system_prompt = "be brief" - agent.config.max_steps = 7 - agent.config.timeout_seconds = 3600 - agent.config.tool_timeout_seconds = 1800 - - config = agent.dump() - - # The full config travels, so every knob is preserved... - assert config["model"] == "test-model" - assert config["max_steps"] == 7 - assert config["system_prompt"] == "be brief" - assert config["timeout_seconds"] == 3600 - assert config["tool_timeout_seconds"] == 1800 - # ...minus what can't or shouldn't cross the wire. - assert "model_client" not in config - assert "api_key" not in config - assert "base_url" not in config - assert "hosted_tools" not in config +def submitted(platform: FakeServices) -> dict[str, Any]: + (request,) = platform.requests("api", "POST", SUBMIT) + assert request.bearer == "k" + return request.json -def test_dump_preserves_training_config( - monkeypatch: pytest.MonkeyPatch, +async def test_a_hosted_rollout_submits_the_row_and_agent_then_polls_to_completion( + platform: FakeServices, tmp_path: Path ) -> None: - """The constructor builds the runtime client without putting it in config.""" - from hud.agents import create_agent - from hud.utils.gateway import GatewayModelInfo, GatewayProviderInfo - - class _GatewayStub: - pass - - client = _GatewayStub() - model = GatewayModelInfo( - id="arith-rl", - model_name="arith-rl", - sdk_agent_type="openai_compatible", - provider=GatewayProviderInfo(name="openai"), - ) - monkeypatch.setattr("hud.utils.gateway.list_gateway_models", lambda *_: [model]) - monkeypatch.setattr("hud.agents.settings.api_key", "test-key") - monkeypatch.setattr("hud.utils.gateway.build_gateway_client", lambda _provider: client) - - agent = create_agent( - "arith-rl", - system_prompt="/no_think", - completion_kwargs={ - "extra_body": { - "return_token_ids": True, - "chat_template_kwargs": {"enable_thinking": False}, - } - }, - ) - assert isinstance(agent, OpenAIChatAgent) - assert agent.config.model_client is None - assert agent.oai is client - - config = agent.dump() - assert config["model"] == "arith-rl" - assert config["system_prompt"] == "/no_think" - assert config["completion_kwargs"] == { - "extra_body": { - "return_token_ids": True, - "chat_template_kwargs": {"enable_thinking": False}, - } - } - assert "model_client" not in config - - -def test_dump_rejects_custom_model_client() -> None: - agent = _agent() - agent.config = OpenAIChatConfig(model="m", model_client=object()) - with pytest.raises(ValueError, match=r"custom model_client.*HUDRuntime"): - agent.dump() - - -@pytest.mark.asyncio -async def test_run_rejects_unregistered_agent() -> None: - """An agent that can't serialize its identity yields a failed Run, not a crash.""" - - class _LocalOnlyAgent(Agent): - async def __call__(self, run: Run) -> None: - run.trace.content = "done" - - run = await HostedRuntime(poll_interval=0.0).run( - Task(env="e", id="x"), - _LocalOnlyAgent(), - job_id="j", - ) - assert run.trace.is_error - assert "registered agent types" in (run.trace.error or "") - assert "_LocalOnlyAgent" in (run.trace.error or "") - - -@pytest.mark.asyncio -async def test_run_submits_and_polls_to_terminal(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform( - [ - {"status": "pending"}, - {"status": "running"}, - {"status": "completed", "reward": 0.5}, - ] - ) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - hosted = HostedRuntime(poll_interval=0.0) - trace_id = uuid.uuid4().hex - job_id = uuid.uuid4().hex - parent_trace_id = uuid.uuid4().hex - task = Task( + compose = tmp_path / "compose.yaml" + compose.write_text("services:\n main:\n image: sums:latest\n") + platform.route( + "api", + "GET", + POLL, + Reply(json={"status": "pending"}), + Reply(json={"status": "running"}), + Reply(json={"status": "completed", "reward": 0.5}), + ) + row = Task( env="sums", id="add", slug="sums-add", args={"a": 1, "b": 2}, agent_config={"timeout_seconds": 45.0}, runtime_config=RuntimeConfig( - image="registry.example/sums:latest", - resources=RuntimeResources(cpu=2, gpu=RuntimeGPU(type="L4", count=1)), + compose=ComposeProject(document=compose, service_access=True), + resources=RuntimeResources(cpu=2, gpu=RuntimeGPU(type="L4"), memory_mb=None), limits=RuntimeLimits(startup_timeout_s=120, run_timeout_s=900), ), verifier=Task( env="judge", id="verify", args={"expected": 3}, - runtime_config=RuntimeConfig(resources=RuntimeResources(memory_mb=4096)), + runtime_config=RuntimeConfig(resources=None), ), ) - with set_trace_context(parent_trace_id): - run = await hosted.run(task, _agent(), job_id=job_id, group_id="g1", trace_id=trace_id) - - assert run.reward == 0.5 - assert run.trace.status == "completed" - assert run.trace.trace_id == trace_id - assert run.job_id == job_id - assert run.group_id == "g1" - assert run.slug == task.slug - assert Job(id="job", name="test", runs=[run]).results == {"sums-add": [run]} - assert platform.polled == 3 - (path, payload) = platform.posts[0] - assert path == "/rollouts/submit" - # Hex ids travel as canonical UUID strings. - assert payload["trace_id"] == str(uuid.UUID(trace_id)) - assert payload["job_id"] == str(uuid.UUID(job_id)) - assert payload["parent_trace_id"] == str(uuid.UUID(parent_trace_id)) - assert payload["env"] == "sums" - assert payload["task"] == "add" - assert payload["slug"] == "sums-add" - assert payload["args"] == {"a": 1, "b": 2} - assert payload["runtime_config"] == { - "image": "registry.example/sums:latest", - "resources": {"cpu": 2.0, "gpu": {"type": "L4", "count": 1}}, - "limits": {"startup_timeout_s": 120, "run_timeout_s": 900}, - } - assert payload["verifier"] == { - "env": "judge", - "id": "verify", - "args": {"expected": 3}, - "slug": "verify-5579a3e5", - "runtime_config": {"resources": {"memory_mb": 4096}}, - } - assert payload["group_id"] == "g1" - assert payload["agent"]["type"] == "openai_compatible" - assert payload["agent"]["config"]["model"] == "test-model" - assert payload["agent"]["config"]["timeout_seconds"] == 45.0 - - -@pytest.mark.asyncio -async def test_run_submits_registered_cli_agent(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - agent = ClaudeCLIAgent( - ClaudeCLIConfig( - model="claude-sonnet-4-6", - max_steps=23, - gateway=True, + with set_trace_context("3" * 32): + run = await HostedRuntime(poll_interval=0).run( + row, chat_agent(), job_id=JOB_ID, group_id="g1", trace_id=TRACE_ID ) - ) - - run = await HostedRuntime(poll_interval=0.0).run( - Task(env="coding", id="solve"), - agent, - job_id=uuid.uuid4().hex, - trace_id=uuid.uuid4().hex, - ) - - assert run.reward == 1.0 - submitted = platform.posts[0][1]["agent"] - assert submitted["type"] == "claude_cli" - assert submitted["config"]["model"] == "claude-sonnet-4-6" - assert submitted["config"]["max_steps"] == 23 - assert submitted["config"]["gateway"] is True - -@pytest.mark.asyncio -async def test_run_preserves_runtime_config_null_override( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 0.5}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - await HostedRuntime(poll_interval=0.0).run( - Task(env="sums", id="add", runtime_config=RuntimeConfig(resources=None)), - _agent(), - job_id=uuid.uuid4().hex, - trace_id=uuid.uuid4().hex, - ) - - assert platform.posts[0][1]["runtime_config"] == {"resources": None} - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("parent_trace_id", "trace_id"), - [ - ("external-run-id", "00000000000000000000000000000001"), - ( - "00000000-0000-0000-0000-000000000001", - "00000000000000000000000000000001", - ), - ( - "{00000000-0000-0000-0000-000000000001}", - "00000000000000000000000000000001", - ), - ( - "urn:uuid:00000000-0000-0000-0000-000000000001", - "00000000000000000000000000000001", - ), - ], -) -async def test_run_omits_unusable_parent_trace_id( - monkeypatch: pytest.MonkeyPatch, - parent_trace_id: str, - trace_id: str, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 0.5}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) + assert submitted(platform) == snapshot( + { + "trace_id": "11111111-1111-1111-1111-111111111111", + "job_id": "22222222-2222-2222-2222-222222222222", + "env": "sums", + "task": "add", + "slug": "sums-add", + "args": {"a": 1, "b": 2}, + "agent": { + "type": "openai_compatible", + "config": { + "timeout_seconds": 45.0, + "model_name": "OpenAI Chat", + "model": "test-model", + "gateway": False, + "auto_respond": False, + "max_steps": 10, + "tool_timeout_seconds": None, + "system_prompt": None, + "citations_enabled": False, + "stop_on": [], + "screenshot_encoding": {"mime_type": "image/png"}, + "checkpoint": None, + "completion_kwargs": {}, + }, + }, + "group_id": "g1", + "parent_trace_id": "33333333-3333-3333-3333-333333333333", + "runtime_config": { + "compose": { + "document": { + "services": { + "main": { + "image": "sums:latest", + "environment": {}, + "expose": [], + "ports": [], + "volumes": [], + } + }, + "networks": {}, + }, + "service_access": True, + }, + "resources": {"cpu": 2.0, "gpu": {"type": "L4"}}, + "limits": {"startup_timeout_s": 120, "run_timeout_s": 900}, + }, + "verifier": { + "env": "judge", + "id": "verify", + "args": {"expected": 3}, + "slug": "verify-5579a3e5", + "runtime_config": {"resources": None}, + }, + } ) - - with set_trace_context(parent_trace_id): - run = await HostedRuntime(poll_interval=0.0).run( - Task(env="sums", id="add"), - _agent(), - job_id=uuid.uuid4().hex, - trace_id=trace_id, - ) - - assert run.trace.status == "completed" - assert "parent_trace_id" not in platform.posts[0][1] + polls = platform.requests("api", "GET", POLL) + assert [poll.params["id"] for poll in polls] == [str(uuid.UUID(TRACE_ID))] * 3 + assert (run.reward, run.trace.status, run.trace_id, run.job_id, run.group_id, run.slug) == ( + 0.5, + "completed", + TRACE_ID, + JOB_ID, + "g1", + "sums-add", + ) + assert run.runtime == f"hud://trace/{TRACE_ID}" + with pytest.raises(RuntimeError, match="no live client"): + run.client -@pytest.mark.asyncio -async def test_run_submits_compose_document( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - compose = tmp_path / "compose.json" - compose.write_text( - json.dumps( +AGENTS: dict[str, tuple[Agent, dict[str, Any]]] = { + "a chat agent travels without its credentials": ( + chat_agent(), + snapshot( { - "services": { - "main": {"image": "ghcr.io/hud-evals/harbor-main:latest"}, - "database": {"image": "postgres:16"}, - } + "type": "openai_compatible", + "config": { + "timeout_seconds": None, + "model_name": "OpenAI Chat", + "model": "test-model", + "gateway": False, + "auto_respond": False, + "max_steps": 10, + "tool_timeout_seconds": None, + "system_prompt": None, + "citations_enabled": False, + "stop_on": [], + "screenshot_encoding": {"mime_type": "image/png"}, + "checkpoint": None, + "completion_kwargs": {}, + }, } ), - encoding="utf-8", - ) - platform = _FakePlatform([{"status": "completed", "reward": 0.5}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - await HostedRuntime(poll_interval=0.0).run( - Task( - env="harbor", - id="solve", - runtime_config=RuntimeConfig( - compose=ComposeProject(document=compose, service_access=True) - ), + ), + "a CLI agent travels as its registered type": ( + ClaudeCLIAgent(ClaudeCLIConfig(model="claude-sonnet-4-6", max_steps=23, gateway=True)), + snapshot( + { + "type": "claude_cli", + "config": { + "timeout_seconds": None, + "model_name": "Claude CLI", + "model": "claude-sonnet-4-6", + "gateway": True, + "system_prompt": None, + "permission_mode": "bypassPermissions", + "max_steps": 23, + "reasoning_effort": None, + "screenshot_encoding": {"mime_type": "image/webp", "quality": 85}, + "allowed_tools": [ + "Read", + "Write", + "Edit", + "Bash", + "Glob", + "Grep", + "WebSearch", + "WebFetch", + ], + }, + } ), - _agent(), - job_id=uuid.uuid4().hex, - trace_id=uuid.uuid4().hex, - ) - - runtime_config = platform.posts[0][1]["runtime_config"] - assert runtime_config["compose"]["document"]["services"]["database"]["image"] == "postgres:16" - assert runtime_config["compose"]["service_access"] is True - assert str(compose) not in json.dumps(runtime_config) - - -@pytest.mark.asyncio -async def test_run_timeout_requests_platform_cancel(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "running"}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - hosted = HostedRuntime(poll_interval=0.0) - task = Task(env="sums", id="add", args={}) - - run = await hosted.run( - task, - _agent(), - job_id=uuid.uuid4().hex, - rollout_timeout=0.001, - ) - await asyncio.sleep(0) - - cancel_posts = [(p, b) for p, b in platform.posts if p == "/rollouts/cancel"] - assert len(cancel_posts) == 1 - assert run.trace.status == "error" - assert run.trace.stop_reason == "timeout" - - -@pytest.mark.asyncio -async def test_omitted_rollout_timeout_allows_long_environment_timeout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=18_000)), - ) - - run = await HostedRuntime(poll_interval=0.0).run( - task, - _agent(), - job_id=uuid.uuid4().hex, - ) - - assert run.trace.status == "completed" - assert platform.posts[0][0] == "/rollouts/submit" + ), +} -@pytest.mark.asyncio -@pytest.mark.parametrize("environment_timeout", [3_600, 18_000]) -async def test_explicit_rollout_timeout_rejects_environment_timeout_at_or_beyond_it( - monkeypatch: pytest.MonkeyPatch, - environment_timeout: int, +@pytest.mark.parametrize(("agent", "spec"), AGENTS.values(), ids=AGENTS.keys()) +async def test_a_registered_agent_travels_as_its_type_and_config( + agent: Agent, spec: dict[str, Any], platform: FakeServices ) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=environment_timeout)), - ) + platform.route("api", "GET", POLL, json={"status": "completed", "reward": 1.0}) - with pytest.raises(ValueError, match=r"actor runtime_config\.limits\.run_timeout_s"): - await HostedRuntime().run( - task, - _agent(), - job_id=uuid.uuid4().hex, - rollout_timeout=3_600, - ) + await HostedRuntime(poll_interval=0).run(ROW, agent, job_id=JOB_ID, trace_id=TRACE_ID) - assert platform.posts == [] + assert submitted(platform)["agent"] == spec -@pytest.mark.asyncio -async def test_taskset_rollout_timeout_reaches_hosted_runtime( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=18_000)), - ) +PARENTS = { + "a uuid ambient trace is the parent": ("3" * 32, str(uuid.UUID("3" * 32))), + "a non-uuid ambient trace is dropped": ("external-run-id", None), + "the rollout's own id is not its parent": (str(uuid.UUID(TRACE_ID)), None), + "the rollout's own id in braces is not its parent": ( + "{" + str(uuid.UUID(TRACE_ID)) + "}", + None, + ), + "the rollout's own id as a urn is not its parent": (f"urn:uuid:{uuid.UUID(TRACE_ID)}", None), +} - job = await Taskset("sums", [task]).run( - _agent(), - runtime=HostedRuntime(poll_interval=0.0), - rollout_timeout=18_600, - ) - assert job.runs[0].trace.status == "completed" - assert platform.posts[0][0] == "/rollouts/submit" - - -@pytest.mark.asyncio -async def test_agent_timeout_must_fit_explicit_environment_timeout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - agent_config={"timeout_seconds": 5_000}, - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=3_600)), - ) - - with pytest.raises(ValueError, match=r"agent timeout \(5000s\).+run_timeout_s \(3600s\)"): - await HostedRuntime().run(task, _agent(), job_id=uuid.uuid4().hex) - - assert platform.posts == [] - - -@pytest.mark.asyncio -async def test_agent_timeout_must_fit_explicit_rollout_timeout( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize(("ambient", "parent"), PARENTS.values(), ids=PARENTS.keys()) +async def test_a_hosted_rollout_names_its_parent_trace_only_when_it_is_another_trace( + ambient: str, parent: str | None, platform: FakeServices ) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task( - env="sums", - id="add", - agent_config={"timeout_seconds": 5_000}, - ) + platform.route("api", "GET", POLL, json={"status": "completed", "reward": 1.0}) - with pytest.raises(ValueError, match=r"agent timeout \(5000s\).+rollout_timeout \(3600s\)"): - await HostedRuntime().run( - task, - _agent(), - job_id=uuid.uuid4().hex, - rollout_timeout=3_600, + with set_trace_context(ambient): + await HostedRuntime(poll_interval=0).run( + ROW, chat_agent(), job_id=JOB_ID, trace_id=TRACE_ID ) - assert platform.posts == [] - - -@pytest.mark.asyncio -async def test_agent_timeout_must_fit_provider_runtime_timeout() -> None: - task = Task( - env="sums", - id="add", - agent_config={"timeout_seconds": 900}, - ) - runtime = ModalRuntime(runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=600))) - - with pytest.raises(ValueError, match=r"agent timeout \(900s\).+run_timeout_s \(600s\)"): - await Taskset("sums", [task]).run( - _agent(), - runtime=runtime, - rollout_timeout=1_200, - ) - - -def test_task_runtime_timeout_overrides_provider_runtime_timeout() -> None: - runtime = ModalRuntime(runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=600))) - task = Task( - env="sums", - id="add", - runtime_config=RuntimeConfig(limits=RuntimeLimits(run_timeout_s=1_000)), - ) - - config = resolve_runtime_config(runtime, task) - - assert config is not None - assert config.limits == RuntimeLimits(run_timeout_s=1_000) + assert submitted(platform).get("parent_trace_id") == parent -@pytest.mark.asyncio -async def test_provider_startup_timeout_must_fit_rollout_timeout() -> None: - task = Task( - env="sums", - id="add", - agent_config={"timeout_seconds": 30}, - ) - runtime = ModalRuntime( - runtime_config=RuntimeConfig(limits=RuntimeLimits(startup_timeout_s=600)) - ) - - with pytest.raises( - ValueError, - match=r"actor runtime_config\.limits\.startup_timeout_s \(600s\)", - ): - await Taskset("sums", [task]).run( - _agent(), - runtime=runtime, - rollout_timeout=600, - ) - - -@pytest.mark.asyncio -async def test_rollout_timeout_validates_separable_verifier_limits() -> None: - task = Task( - env="actor", - id="solve", - verifier=Task( - env="judge", - id="verify", - runtime_config=RuntimeConfig(limits=RuntimeLimits(startup_timeout_s=90)), +TERMINAL = { + "a completed trace": ( + {"status": "completed", "reward": 1.0, "error": None}, + (1.0, "completed", None, {"score": 1.0}, False), + ), + "a completed trace with its evaluation": ( + { + "status": "completed", + "reward": 0.5, + "evaluation_result": {"score": 0.5, "content": "half", "info": {"k": 1}}, + }, + (0.5, "completed", None, {"score": 0.5, "content": "half", "info": {"k": 1}}, False), + ), + "an errored trace with an errored evaluation": ( + {"status": "error", "evaluation_result": {"score": 0.0, "isError": True}}, + (0.0, "error", None, {"score": 0.0, "isError": True}, True), + ), + "an errored trace without a reward": ( + {"status": "error", "reward": None, "error": "env exploded"}, + (0.0, "error", "env exploded", {}, True), + ), + "an errored trace that was still graded": ( + {"status": "error", "reward": 0.75, "error": "agent exploded"}, + (0.75, "error", "agent exploded", {"score": 0.75}, False), + ), + "a trace cancelled before grading": ( + {"status": "cancelled", "reward": None}, + (0.0, "cancelled", None, {}, True), + ), + "a trace whose evaluation has no numeric score": ( + {"status": "completed", "evaluation_result": {"score": "high"}}, + ( + 0.0, + "error", + "hud rpc error -32603: tasks.grade: result must include a numeric 'score'", + {}, + True, ), - ) + ), +} - with pytest.raises( - ValueError, - match=r"verifier runtime_config\.limits\.startup_timeout_s \(90s\)", - ): - await Taskset("separable", [task]).run( - _agent(), - runtime=Runtime("tcp://127.0.0.1:1"), - rollout_timeout=90, - ) - - -@pytest.mark.asyncio -async def test_submit_timeout_requests_platform_cancel(monkeypatch: pytest.MonkeyPatch) -> None: - never = asyncio.Event() - - class _StuckSubmitPlatform(_FakePlatform): - async def apost(self, path: str, *, json: Any | None = None) -> Any: - self.posts.append((path, json or {})) - if path == "/rollouts/submit": - await never.wait() - return {"status": "queued"} - - platform = _StuckSubmitPlatform([]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - run = await HostedRuntime().run( - Task(env="sums", id="add"), - _agent(), - job_id=uuid.uuid4().hex, - rollout_timeout=0.001, - ) - await asyncio.sleep(0) - - assert run.trace.stop_reason == "timeout" - assert any(path == "/rollouts/cancel" for path, _ in platform.posts) - - -@pytest.mark.asyncio -async def test_run_folds_completed_receipt(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "completed", "reward": 1.0, "error": None}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - - task = Task(env="sums", id="add", args={"a": 2, "b": 3}) - run = await HostedRuntime(poll_interval=0.0).run(task, _agent(), job_id=uuid.uuid4().hex) - - assert run.reward == 1.0 - assert run.trace.status == "completed" - assert not run.trace.is_error - assert run.runtime == f"hud://trace/{run.trace.trace_id}" - # The platform owns the trace lifecycle: no local client ever existed. - with pytest.raises(RuntimeError, match="no live client"): - _ = run.client - -@pytest.mark.asyncio -@pytest.mark.parametrize("is_error", [False, True]) -async def test_run_preserves_structured_grade( - monkeypatch: pytest.MonkeyPatch, is_error: bool +@pytest.mark.parametrize(("state", "folded"), TERMINAL.values(), ids=TERMINAL.keys()) +async def test_a_terminal_trace_folds_into_the_run_and_its_job( + state: dict[str, Any], folded: tuple[Any, ...], platform: FakeServices ) -> None: - result = { - "score": 0.5, - "info": {"passed": 3}, - "content": "verifier failed" if is_error else "partial credit", - "isError": is_error, - "subscores": [{"name": "accuracy", "weight": 1.0, "value": 0.5}], - } - platform = _FakePlatform( - [ - { - "status": "error" if is_error else "completed", - "reward": 0.5, - "evaluation_result": result, - } - ] - ) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - run = await HostedRuntime(poll_interval=0.0).run( - Task(env="sums", id="add"), _agent(), job_id=uuid.uuid4().hex - ) - assert run.reward == 0.5 - assert run.grade.raw == result - assert run.grade.info == {"passed": 3} - assert run.grade.is_error is is_error - assert run.grade.content == result["content"] - assert platform.polled == 1 - - -@pytest.mark.asyncio -async def test_run_reports_malformed_grade_as_a_failed_run(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "completed", "evaluation_result": {"score": "bad"}}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) - task = Task(env="sums", id="add") - run = await HostedRuntime(poll_interval=0.0).run(task, _agent(), job_id=uuid.uuid4().hex) - assert run.trace.is_error - assert "numeric 'score'" in (run.trace.error or "") - assert run.slug == task.slug + platform.route("api", "GET", POLL, json=state) + job = await Taskset("hosted", [ROW]).run(chat_agent(), runtime=HostedRuntime(poll_interval=0)) -@pytest.mark.asyncio -async def test_run_folds_error_receipt(monkeypatch: pytest.MonkeyPatch) -> None: - platform = _FakePlatform([{"status": "error", "reward": None, "error": "env exploded"}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) + (run,) = job.runs + assert (run.reward, run.trace.status, run.trace.error, run.grade.raw, run in job.errors) == ( + folded ) + assert run.slug == ROW.slug + body = submitted(platform) + assert (body["job_id"], body["group_id"]) == (str(uuid.UUID(job.id)), run.group_id) - task = Task(env="sums", id="add", args={}) - run = await HostedRuntime(poll_interval=0.0).run(task, _agent(), job_id=uuid.uuid4().hex) - assert run.reward == 0.0 - assert run.trace.is_error - assert "env exploded" in (run.trace.error or "") +STALLS = { + "a trace that never finishes": ( + Reply(json={"status": "queued"}), + Reply(json={"status": "running"}), + ), + "a submission that never returns": (Reply(json={"status": "queued"}, delay=5), None), +} -@pytest.mark.asyncio -async def test_run_keeps_a_grade_from_an_errored_hosted_trace( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize(("submit", "poll"), STALLS.values(), ids=STALLS.keys()) +async def test_a_hosted_rollout_past_its_deadline_is_cancelled_remotely( + submit: Reply, poll: Reply | None, platform: FakeServices ) -> None: - platform = _FakePlatform([{"status": "error", "reward": 0.75, "error": "agent timed out"}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) - ) + platform.route("api", "POST", SUBMIT, submit) + if poll is not None: + platform.route("api", "GET", POLL, poll) - task = Task(env="sums", id="add", args={}) - run = await HostedRuntime(poll_interval=0.0).run( - task, - _agent(), - job_id=uuid.uuid4().hex, - ) - job = Job(id="job", name="test", runs=[run]) + job = await ROW.run(chat_agent(), runtime=HostedRuntime(poll_interval=0), rollout_timeout=0.2) + await eventually(lambda: platform.requests("api", "POST", CANCEL) != []) - assert run.trace.is_error - assert not run.grade.is_error - assert run.evaluation == {"score": 0.75} - assert job.reward == 0.75 - assert job.errors == [] - - -@pytest.mark.asyncio -async def test_run_folds_ungraded_cancellation_as_an_error( - monkeypatch: pytest.MonkeyPatch, -) -> None: - platform = _FakePlatform([{"status": "cancelled", "reward": None, "error": None}]) - monkeypatch.setattr( - "hud.eval.runtime.hosted.PlatformClient.from_settings", classmethod(lambda cls: platform) + (run,) = job.runs + assert (run.trace.status, run.trace.stop_reason, run.trace.error) == ( + "error", + "timeout", + IsStr(regex=r"hosted rollout [0-9a-f]{32} did not finish within 0\.2s"), ) + assert platform.bodies("api", "POST", CANCEL) == [{"trace_id": str(uuid.UUID(run.trace_id))}] - task = Task(env="sums", id="add", args={}) - run = await HostedRuntime(poll_interval=0.0).run( - task, - _agent(), - job_id=uuid.uuid4().hex, - ) - job = Job(id="job", name="test", runs=[run]) - - assert run.trace.status == "cancelled" - assert run.grade.is_error - assert job.reward == 0.0 - assert job.errors == [run] - - -@pytest.mark.asyncio -async def test_scheduler_drives_provider_locally(monkeypatch: pytest.MonkeyPatch) -> None: - """A Provider placement goes through the local rollout atom, not HostedRuntime.""" - import hud.eval.taskset as taskset_mod - - seen: dict[str, Any] = {} - - async def fake_rollout(task: Task, agent: Any, **kwargs: Any) -> Run: - seen.update(kwargs) - run = Run(None, task.id, {}) - run.trace.status = "completed" - return run - - monkeypatch.setattr(taskset_mod, "rollout", fake_rollout) - - job = await Taskset("t", [Task(env="e", id="x")]).run( - _agent(), runtime=Runtime("tcp://127.0.0.1:1") - ) - - assert len(job.runs) == 1 - assert isinstance(seen["runtime"], Runtime) - assert "job_id" in seen and "group_id" in seen - - -@pytest.mark.asyncio -async def test_scheduler_delegates_hosted(monkeypatch: pytest.MonkeyPatch) -> None: - """A HostedRuntime placement is delegated to via HostedRuntime.run, not the local atom.""" - seen: dict[str, Any] = {} - - class _RecordingHostedRuntime(HostedRuntime): - async def run(self, task: Task, agent: Agent, **kwargs: Any) -> Run: - seen.update(kwargs) - run = Run(None, task.id, {}) - run.trace.status = "completed" - return run - - job = await Taskset("t", [Task(env="e", id="x")]).run( - _agent(), runtime=_RecordingHostedRuntime(), rollout_timeout=90.0 - ) - assert len(job.runs) == 1 - assert "job_id" in seen and "group_id" in seen - assert seen["rollout_timeout"] == 90.0 - - -@pytest.mark.asyncio -async def test_hud_runtime_drives_local_rollout(monkeypatch: pytest.MonkeyPatch) -> None: - seen: dict[str, Any] = {} - - async def fake_rollout(task: Task, agent: Any, **kwargs: Any) -> Run: - seen.update(kwargs) - run = Run(None, task.id, {}) - run.trace.status = "completed" - return run - - monkeypatch.setattr("hud.eval.runtime.hud.rollout", fake_rollout) - - runtime = HUDRuntime() - job_id = uuid.uuid4().hex - trace_id = uuid.uuid4().hex - run = await runtime.run( - Task(env="e", id="x"), - _agent(), - job_id=job_id, - group_id="g1", - trace_id=trace_id, - rollout_timeout=90.0, - ) - - assert run.trace.status == "completed" - assert seen["runtime"] is runtime - assert seen["job_id"] == job_id - assert seen["group_id"] == "g1" - assert seen["trace_id"] == trace_id - assert seen["rollout_timeout"] == 90.0 - - with pytest.raises(ValueError, match="placement requirements"): - async with runtime( - Task( - env="e", - id="x", - runtime_config=RuntimeConfig(resources=RuntimeResources(gpu=RuntimeGPU())), - ) - ): - pass - - -@pytest.mark.asyncio -async def test_runtime_session_create_payload_omits_trace_id( - monkeypatch: pytest.MonkeyPatch, +async def test_the_deprecated_constructor_timeout_is_the_default_deadline( + platform: FakeServices, ) -> None: - posts: list[dict[str, Any]] = [] - session_id = str(uuid.uuid4()) - - class _RecordingAsyncClient: - def __init__(self, *args: Any, **kwargs: Any) -> None: - pass - - async def __aenter__(self) -> _RecordingAsyncClient: - return self - - async def __aexit__(self, *args: Any) -> None: - return None - - async def post( - self, - path: str, - *, - headers: dict[str, str], - json: dict[str, Any], - ) -> _FakeResponse: - posts.append({"path": path, "headers": headers, "json": json}) - return _FakeResponse({"id": session_id}) + platform.route("api", "GET", POLL, json={"status": "running"}) + with pytest.warns(DeprecationWarning, match="rollout_timeout=... to Task.run"): + runtime = HostedRuntime(poll_interval=0, run_timeout=0.2) - monkeypatch.setattr("hud.eval.runtime.hud.httpx.AsyncClient", _RecordingAsyncClient) + job = await ROW.run(chat_agent(), runtime=runtime) - created = await HUDRuntime()._create_runtime_session( - "https://mcp.hud.ai", - "sk-hud-test", - Task(env="e", id="x"), - ) - - assert created == session_id - assert posts == [ - { - "path": "https://mcp.hud.ai/runtime/sessions", - "headers": {"Authorization": "Bearer sk-hud-test"}, - "json": {"environment": "e"}, - } - ] + assert job.runs[0].trace.stop_reason == "timeout" + await eventually(lambda: platform.requests("api", "POST", CANCEL) != []) -@pytest.mark.asyncio -async def test_runtime_session_create_payload_includes_current_trace_id( - monkeypatch: pytest.MonkeyPatch, -) -> None: - posts: list[dict[str, Any]] = [] - session_id = str(uuid.uuid4()) - trace_id = uuid.uuid4().hex - - class _RecordingAsyncClient: - def __init__(self, *args: Any, **kwargs: Any) -> None: - pass +async def test_cancelling_a_hosted_rollout_cancels_it_remotely(platform: FakeServices) -> None: + platform.route("api", "GET", POLL, json={"status": "running"}) + pending = asyncio.create_task(ROW.run(chat_agent(), runtime=HostedRuntime(poll_interval=0))) + await eventually(lambda: platform.requests("api", "GET", POLL) != []) - async def __aenter__(self) -> _RecordingAsyncClient: - return self + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + await eventually(lambda: platform.requests("api", "POST", CANCEL) != []) - async def __aexit__(self, *args: Any) -> None: - return None + trace_id = submitted(platform)["trace_id"] + assert platform.bodies("api", "POST", CANCEL) == [{"trace_id": trace_id}] - async def post( - self, - path: str, - *, - headers: dict[str, str], - json: dict[str, Any], - ) -> _FakeResponse: - posts.append({"path": path, "headers": headers, "json": json}) - return _FakeResponse({"id": session_id}) - monkeypatch.setattr("hud.eval.runtime.hud.httpx.AsyncClient", _RecordingAsyncClient) +REFUSALS: dict[str, tuple[Agent, dict[str, str | None], str]] = { + "an agent the platform cannot run": ( + ScriptedAgent("3"), + {"HUD_API_KEY": "k"}, + "hosted execution supports the registered agent types (claude, claude_cli, codex_cli, " + "openai, gemini, openai_compatible); got ScriptedAgent", + ), + "no API key": (chat_agent(), {"HUD_API_KEY": None}, "HUD_API_KEY is required"), +} - with set_trace_context(trace_id): - created = await HUDRuntime()._create_runtime_session( - "https://mcp.hud.ai", - "sk-hud-test", - Task(env="e", id="x"), - ) - assert created == session_id - assert posts == [ - { - "path": "https://mcp.hud.ai/runtime/sessions", - "headers": {"Authorization": "Bearer sk-hud-test"}, - "json": {"environment": "e", "trace_id": str(uuid.UUID(trace_id))}, - } - ] - - -@pytest.mark.asyncio -async def test_runtime_session_sets_runtime_connection_params( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize(("agent", "variables", "error"), REFUSALS.values(), ids=REFUSALS.keys()) +async def test_a_hosted_rollout_that_cannot_be_submitted_is_a_failed_run( + agent: Agent, + variables: dict[str, str | None], + error: str, + services: FakeServices, + hud_env: HudEnv, ) -> None: - session_id = str(uuid.uuid4()) - deleted: list[tuple[str, str, str]] = [] - - class _Socket: - def getsockname(self) -> tuple[str, int]: - return ("127.0.0.1", 4321) - - class _Server: - sockets: ClassVar[list[_Socket]] = [_Socket()] - - def __init__(self) -> None: - self.closed = False - self.waited = False - - def close(self) -> None: - self.closed = True + hud_env.set(**variables) - async def wait_closed(self) -> None: - self.waited = True + job = await ROW.run(agent, runtime=HostedRuntime(poll_interval=0)) - server = _Server() + (run,) = job.runs + assert (run.trace.status, run.trace.error, run in job.errors) == ("error", error, True) + assert services.requests() == [] - async def fake_start_server(*args: Any, **kwargs: Any) -> _Server: - return server - async def fake_create_runtime_session( - self: HUDRuntime, - runtime_url: str, - api_key: str, - task: Task, - ) -> str: - assert runtime_url == "https://mcp.hud.ai" - assert api_key == "sk-hud-test" - assert task.env == "e" - return session_id +async def test_a_rejected_submission_is_a_failed_run(platform: FakeServices) -> None: + platform.route("api", "POST", SUBMIT, json={"detail": "unknown env 'sums'"}, status=422) - async def fake_delete_runtime_session( - self: HUDRuntime, - runtime_url: str, - api_key: str, - session: str, - ) -> None: - deleted.append((runtime_url, api_key, session)) + job = await ROW.run(chat_agent(), runtime=HostedRuntime(poll_interval=0)) - monkeypatch.setattr(settings, "api_key", "sk-hud-test") - monkeypatch.setattr("hud.eval.runtime.hud.asyncio.start_server", fake_start_server) - monkeypatch.setattr(HUDRuntime, "_create_runtime_session", fake_create_runtime_session) - monkeypatch.setattr(HUDRuntime, "_delete_runtime_session", fake_delete_runtime_session) - - with pytest.warns(DeprecationWarning, match="rollout_timeout"): - cloud = HUDRuntime(runtime_url="https://mcp.hud.ai/", run_timeout=30.0) - async with cloud._runtime_session(Task(env="e", id="x")) as runtime: - assert runtime.url == "tcp://127.0.0.1:4321" - assert runtime.params == { - "session_id": session_id, - "gateway_url": "https://mcp.hud.ai", - "ready_timeout": 300.0, - } - - assert deleted == [("https://mcp.hud.ai", "sk-hud-test", session_id)] - assert server.closed - assert server.waited - - -@pytest.mark.asyncio -async def test_splice_websocket_propagates_relay_errors() -> None: - class _Reader: - def __init__(self) -> None: - self.reads = [b"payload", b""] - - async def read(self, _limit: int) -> bytes: - return self.reads.pop(0) - - class _Writer: - def write(self, _data: bytes) -> None: - pass - - async def drain(self) -> None: - pass - - class _WebSocket: - async def send(self, _data: bytes) -> None: - raise RuntimeError("relay failed") - - def __aiter__(self) -> _WebSocket: - return self - - async def __anext__(self) -> bytes: - await asyncio.sleep(60.0) - raise StopAsyncIteration - - with pytest.raises(RuntimeError, match="relay failed"): - await _splice_websocket( - cast("asyncio.StreamReader", _Reader()), - cast("asyncio.StreamWriter", _Writer()), - _WebSocket(), - ) + (run,) = job.runs + assert run.trace.status == "error" + assert "unknown env 'sums'" in str(run.trace.error) + assert platform.requests("api", "GET", POLL) == [] From d108f478eb1e8d4a00db3d15cc6d00446477d4cf Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 096/248] fix: stop polling a hosted rollout once its cancellation is requested asyncio.timeout bounds a hosted rollout by cancelling it once. When that cancellation lands mid-request, httpx (through anyio) can swallow it, and the poll loop carried on forever: a rollout past rollout_timeout, or one cancelled with Ctrl-C, never returned. The loop now checks for a pending cancellation before every poll. A scripted loop of 600 timed-out hosted rollouts hung within the first hundred before this change and passes after. --- hud/eval/runtime/hosted.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hud/eval/runtime/hosted.py b/hud/eval/runtime/hosted.py index a98efe720..43db75bbe 100644 --- a/hud/eval/runtime/hosted.py +++ b/hud/eval/runtime/hosted.py @@ -211,7 +211,12 @@ def _fold(state: dict[str, Any], trace_id: str) -> Run: return run async def _await_terminal(self, platform: PlatformClient, trace_id: str) -> dict[str, Any]: + task = asyncio.current_task() while True: + # httpx can swallow a cancellation that lands mid-request, which + # would leave a timed-out or cancelled rollout polling forever. + if task is not None and task.cancelling(): + raise asyncio.CancelledError state: dict[str, Any] = await platform.aget(f"/trace/{trace_id}") if state.get("status") in _TERMINAL_TRACE_STATUSES: return state From 41524431fd03a33db4cf4aed08728f398995be63 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 097/248] test: run tunneled rollouts end to end through the fake runtime service The fake runtime service relays each tunnel WebSocket to an environment served in-process, so a platform taskset run without runtime= really completes over HUDRuntime. Rows pin the session body with its trace id, the bearer on the POST and the WebSocket, the acquired runtime's params, the session released on every path (including after a deadline), refusals made before any session, the one-time warning for ignored resources, and the deprecated constructor timeout. The httpx.AsyncClient and asyncio.start_server patches and the private splice test go. --- tests/eval/test_hud_runtime.py | 247 +++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 tests/eval/test_hud_runtime.py diff --git a/tests/eval/test_hud_runtime.py b/tests/eval/test_hud_runtime.py new file mode 100644 index 000000000..6a125ec4a --- /dev/null +++ b/tests/eval/test_hud_runtime.py @@ -0,0 +1,247 @@ +"""The HUD runtime tunnel: the agent loop runs here against an environment the +platform hosts, reached through a runtime session and a WebSocket tunnel. + +The fake runtime service relays each tunnel's bytes to an environment served in +this process, so a tunneled rollout really completes and its reward proves it. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import uuid +from typing import TYPE_CHECKING, Any + +import pytest +from dirty_equals import IsStr + +from hud.eval import ( + HUDRuntime, + LocalRuntime, + RuntimeConfig, + RuntimeGPU, + RuntimeLimits, + RuntimeResources, + Task, + Taskset, +) +from tests.eval.envs import eventually, lab, solve +from tests.harness import Reply, ScriptedAgent + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable + + from starlette.websockets import WebSocket + + from tests.harness import FakeServices, HudEnv + +SESSIONS = "/runtime/sessions" +SESSION = "/runtime/sessions/{id}" +TUNNEL = "/runtime/tunnels/{id}" +ADD = Task(env="lab", id="add", args={"a": 2, "b": 3}) + + +def relay(port: int) -> Callable[[WebSocket, dict[str, str]], Awaitable[None]]: + """A tunnel endpoint splicing each WebSocket to the control channel on ``port``.""" + + async def handler(websocket: WebSocket, params: dict[str, str]) -> None: + del params + await websocket.accept() + reader, writer = await asyncio.open_connection("127.0.0.1", port) + + async def upstream() -> None: + while (message := await websocket.receive())["type"] != "websocket.disconnect": + writer.write(message.get("bytes") or message.get("text", "").encode()) + await writer.drain() + + async def downstream() -> None: + while data := await reader.read(65536): + await websocket.send_bytes(data) + + tasks = [asyncio.create_task(upstream()), asyncio.create_task(downstream())] + await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + writer.close() + with contextlib.suppress(Exception): + await websocket.close() + + return handler + + +async def dropped(websocket: WebSocket, params: dict[str, str]) -> None: + del params + await websocket.accept() + await websocket.close() + + +@contextlib.asynccontextmanager +async def hosted_env(services: FakeServices, hud_env: HudEnv) -> AsyncIterator[None]: + """Serve ``lab`` here and let the fake runtime service tunnel to it.""" + hud_env.set(HUD_API_KEY="k") + services.route("runtime", "POST", SESSIONS, json={"id": "sess-1"}) + services.route("runtime", "DELETE", SESSION, json={}) + async with LocalRuntime(lab())(Task(env="lab", id="serve")) as served: + services.websocket("runtime", TUNNEL, relay(int(served.url.rsplit(":", 1)[1]))) + yield + + +async def test_a_platform_taskset_runs_over_the_tunnel_by_default( + services: FakeServices, hud_env: HudEnv +) -> None: + async with hosted_env(services, hud_env): + job = await Taskset("remote", [ADD], taskset_id="ts-1").run(ScriptedAgent(solve)) + + (run,) = job.runs + assert run.reward == 1.0 + (create,) = services.requests("runtime", "POST", SESSIONS) + assert (create.bearer, create.json) == ( + "k", + {"environment": "lab", "trace_id": str(uuid.UUID(run.trace_id))}, + ) + tunnels = services.requests("runtime", "WS", TUNNEL) + assert tunnels != [] + assert {(tunnel.params["id"], tunnel.bearer) for tunnel in tunnels} == {("sess-1", "k")} + assert [request.path for request in services.requests("runtime", "DELETE")] == [ + "/runtime/sessions/sess-1" + ] + + +async def test_an_acquired_tunnel_is_a_local_address_for_the_session( + services: FakeServices, hud_env: HudEnv +) -> None: + async with hosted_env(services, hud_env), HUDRuntime()(ADD) as runtime: + assert services.requests("runtime", "DELETE") == [] + + assert (runtime.url, runtime.params) == ( + IsStr(regex=r"tcp://127\.0\.0\.1:\d+"), + { + "session_id": "sess-1", + "gateway_url": services.url("runtime"), + "ready_timeout": 300.0, + }, + ) + assert services.bodies("runtime", "POST", SESSIONS) == [{"environment": "lab"}] + assert len(services.requests("runtime", "DELETE", SESSION)) == 1 + + +GPU = RuntimeConfig(resources=RuntimeResources(gpu=RuntimeGPU(type="H100"))) +REFUSED = ( + "[provisioning] ValueError: HUDRuntime cannot honor this task's declared placement " + "requirements or limits on an already-deployed env; run it on a placement that provisions them" +) +FAILURES: dict[str, tuple[Task, dict[str, Any], Any, list[str]]] = { + "a GPU it cannot provision is refused before a session": ( + ADD.model_copy(update={"runtime_config": GPU}), + {}, + REFUSED, + [], + ), + "explicit limits are refused before a session": ( + ADD.model_copy( + update={"runtime_config": RuntimeConfig(limits=RuntimeLimits(run_timeout_s=60))} + ), + {}, + REFUSED, + [], + ), + "no API key": ( + ADD, + {"key": None}, + "[provisioning] RuntimeError: HUD runtime tunnel requires HUD_API_KEY", + [], + ), + "a rejected session": ( + ADD, + {"session": Reply(status=503, json={"detail": "no capacity"})}, + IsStr(regex=r"(?s)\[provisioning\] httpx\.HTTPStatusError: Server error '503 .*"), + ["POST"], + ), + "a session without an id": ( + ADD, + {"session": Reply(json={"status": "ok"})}, + "[provisioning] RuntimeError: Runtime gateway did not return a session id", + ["POST"], + ), + "a tunnel the gateway keeps dropping, until the deadline": ( + ADD, + {"tunnel": dropped, "rollout_timeout": 0.5}, + "rollout timed out after 0.5s during starting task", + ["POST", "WS", "DELETE"], + ), + "an agent that raises": ( + ADD, + {"agent": ScriptedAgent(fail_before=RuntimeError("agent exploded"))}, + "[agent loop] RuntimeError: agent exploded", + ["POST", "WS", "DELETE"], + ), +} + + +@pytest.mark.parametrize( + ("row", "change", "error", "calls"), FAILURES.values(), ids=FAILURES.keys() +) +async def test_a_failed_tunnel_rollout_is_an_errored_run_and_its_session_is_released( + row: Task, + change: dict[str, Any], + error: Any, + calls: list[str], + services: FakeServices, + hud_env: HudEnv, +) -> None: + async with hosted_env(services, hud_env): + hud_env.set(HUD_API_KEY=change.get("key", "k")) + if "session" in change: + services.route("runtime", "POST", SESSIONS, change["session"]) + if "tunnel" in change: + services.websocket("runtime", TUNNEL, change["tunnel"]) + job = await row.run( + change.get("agent", ScriptedAgent(solve)), + runtime=HUDRuntime(), + rollout_timeout=change.get("rollout_timeout"), + ) + + (run,) = job.runs + assert (run.trace.status, run.trace.error) == ("error", error) + + # A deadline returns before teardown finishes; the session is released after. + def methods() -> list[str]: + return list(dict.fromkeys(request.method for request in services.requests())) + + await eventually(lambda: methods() == calls, within=5) + + +async def test_resources_the_deployed_env_ignores_are_warned_about_once( + services: FakeServices, hud_env: HudEnv, caplog: pytest.LogCaptureFixture +) -> None: + sized = RuntimeConfig(image="lab:1", resources=RuntimeResources(cpu=4, memory_mb=8192)) + rows = [Task(env="lab", id="add", args={"a": a, "b": 1}, runtime_config=sized) for a in (1, 2)] + + async with hosted_env(services, hud_env): + job = await Taskset("sized", rows).run(ScriptedAgent(solve), runtime=HUDRuntime()) + + assert [run.reward for run in job.runs] == [1.0, 1.0] + assert [ + record.getMessage() for record in caplog.records if record.levelno == logging.WARNING + ] == [ + "HUDRuntime cannot honor task runtime_config ['resources'] on an already-deployed env; " + "rollouts proceed on the platform's defaults" + ] + + +async def test_the_deprecated_constructor_timeout_is_the_default_deadline( + services: FakeServices, hud_env: HudEnv +) -> None: + with pytest.warns(DeprecationWarning, match="rollout_timeout=... to Task.run"): + runtime = HUDRuntime(run_timeout=0.3) + + async with hosted_env(services, hud_env): + job = await ADD.run(ScriptedAgent(solve, linger=30), runtime=runtime) + + (run,) = job.runs + assert (run.trace.stop_reason, run.trace.error) == ( + "timeout", + "rollout timed out after 0.3s during agent loop", + ) From 55bb3b5acd3bb6b803b4b078226913d890bf5450 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:52 -0700 Subject: [PATCH 098/248] test: pin DockerRuntime through the fake docker executable Image and Compose rows snapshot the docker transcript and the staged compose, override and ports files, and each started container serves a real rollout through an environment in this process. Failure rows cover every refusal and every teardown path; session transfer runs end to end through container filesystems the fake docker emulates, including the symbolic-link guard and the restore-side archive filter. The _docker patches, the private security-argument constants and the hidden real 'docker logs' go. --- tests/eval/envs.py | 48 +- tests/eval/test_docker.py | 1152 +++++++++++++++++++++++++++++++++++++ 2 files changed, 1188 insertions(+), 12 deletions(-) create mode 100644 tests/eval/test_docker.py diff --git a/tests/eval/envs.py b/tests/eval/envs.py index ccd99d4b1..4bb48259b 100644 --- a/tests/eval/envs.py +++ b/tests/eval/envs.py @@ -9,6 +9,7 @@ import asyncio import contextlib +import re from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Any @@ -127,14 +128,19 @@ async def large(criteria: str): def actor( - env: Environment | None = None, *, grade: str = "score", sessions: Path | None = None + env: Environment | None = None, + *, + grade: str = "score", + sessions: Path | None = None, + link: bool = False, ) -> Environment: """Declare the actor side of a verifier pair: ``solve`` grades 0.25 and carries the answer. It goes on ``env``, or on a new ``actor`` environment. ``grade="raise"`` makes its grading raise and ``grade="scoreless"`` yields a frame without a score. With ``sessions``, the template writes the answer into its control session's - directory under that root, as an environment in a container would. + directory under that root, as an environment in a container would, plus a + symbolic link when ``link`` is set. """ env = env or Environment("actor") @@ -145,6 +151,8 @@ async def solve(): session = sessions / "runtime" / "sessions" / str(current_session_id.get()) session.mkdir(parents=True) (session / "work.txt").write_text(str(answer)) + if link: + (session / "escape").symlink_to("/etc/passwd") if grade == "raise": raise RuntimeError("actor grade exploded") yield {"score": 0.25, "answer": answer} if grade == "score" else {"answer": answer} @@ -159,7 +167,8 @@ def judge( It goes on ``env``, or on a new ``judge`` environment. ``verdict`` picks a failure instead: ``"raise"``, ``"scoreless"`` or ``"zero"``. With - ``sessions``, it reads the answer from its restored session directory. + ``sessions``, it reads the answer from its restored session directory and + pays 0.0 when nothing was restored. """ env = env or Environment("judge") @@ -174,30 +183,45 @@ async def verify(): yield 0.0 elif sessions is not None: session = sessions / "runtime" / "sessions" / str(current_session_id.get()) - yield 1.0 if (session / "work.txt").read_text() == "secret" else 0.0 + work = session / "work.txt" + yield 1.0 if work.exists() and work.read_text() == "secret" else 0.0 else: yield 1.0 if result["answer"] == "secret" else 0.0 return env +DF_FREE = ( + "Filesystem 1024-blocks Used Available Capacity Mounted on\n" + "overlay 104857600 0 104857600 0% /\n" +) + + +def container(image: str) -> str: + """The name the fake docker gives a container started from ``image``.""" + return re.sub(r"[^\w.-]", "-", image) + + @asynccontextmanager async def containers( fake_docker: FakeDocker, rootfs: Path, images: dict[str, Environment] -) -> AsyncIterator[None]: +) -> AsyncIterator[dict[str, str]]: """Make each image a container the fake docker starts, served in this process. - ``docker run `` answers a container named after the image, ``docker - port`` answers the address its environment serves on here, and ``docker - cp``/``exec`` work on ``/``. + ``docker run `` answers the container :func:`container` names, + ``docker port`` the address its environment serves on here, and ``docker + cp``/``exec`` work on ``/``. Yields each image's address. """ fake_docker.on(r"^(cp|exec) ", rootfs=rootfs) + addresses: dict[str, str] = {} async with contextlib.AsyncExitStack() as stack: for image, env in images.items(): runtime = await stack.enter_async_context( LocalRuntime(env)(Task(env=env.name, id="serve")) ) - address = runtime.url.removeprefix("tcp://") - fake_docker.on(rf"^run .* {image}$", stdout=f"{image}\n") - fake_docker.on(rf"^port {image} 8765$", stdout=f"{address}\n") - yield + name = container(image) + (rootfs / name).mkdir(parents=True, exist_ok=True) + addresses[image] = runtime.url.removeprefix("tcp://") + fake_docker.on(rf"^run .* {re.escape(image)}$", stdout=f"{name}\n") + fake_docker.on(rf"^port {re.escape(name)} 8765$", stdout=f"{addresses[image]}\n") + yield addresses diff --git a/tests/eval/test_docker.py b/tests/eval/test_docker.py new file mode 100644 index 000000000..a6833b21a --- /dev/null +++ b/tests/eval/test_docker.py @@ -0,0 +1,1152 @@ +"""DockerRuntime: what it asks the docker CLI to do, and the runtime it yields. + +The fake ``docker`` on ``PATH`` records every invocation; its ``port`` answer +points at an environment served in this process, so each started container +really serves a rollout whose reward proves the placement worked. +""" + +# The snapshots hold whole docker command lines. +# ruff: noqa: E501 + +from __future__ import annotations + +import asyncio +import io +import json +import re +import tarfile +import textwrap +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest +from dirty_equals import IsStr +from inline_snapshot import snapshot + +import hud +from hud import Environment +from hud.eval import ( + ComposeProject, + DockerRuntime, + RuntimeConfig, + RuntimeGPU, + RuntimeLimits, + RuntimeResources, + Task, + rollout, +) +from hud.eval.runtime import DockerBindMount +from tests.eval.envs import DF_FREE, actor, containers, eventually, judge, lab, solve +from tests.harness import ScriptedAgent + +if TYPE_CHECKING: + from hud.eval import Runtime + from tests.harness import FakeDocker + +SECCOMP = Path(hud.__file__).parent / "eval" / "docker-seccomp.json" +ADD = Task(env="lab", id="add", args={"a": 2, "b": 3}) + + +def scrub(text: str, tmp_path: Path) -> str: + """Docker arguments with their run-specific names replaced by placeholders.""" + for pattern, placeholder in ( + (r"hud-runtime-sessions-[0-9a-f]{32}", ""), + (r"hud-[0-9a-f]{12}\b", ""), + (r"[^\s\"]*/hud-compose-[^/\s\"]+", ""), + (r"[^\s\"]*/hud-session(-import)?-[^/\s\"]+", ""), + (r"/runtime/session-export-[0-9a-f]{32}", ""), + (r"sess-[0-9a-f]{8}", ""), + (r"127\.0\.0\.1:\d{4,5}\b", "127.0.0.1:"), + ): + text = re.sub(pattern, placeholder, text) + return text.replace(str(SECCOMP), "").replace(str(tmp_path), "") + + +def transcript(fake_docker: FakeDocker, tmp_path: Path) -> list[str]: + """The docker commands run, less the log followers, which may be stopped before they log.""" + return [ + scrub(" ".join("