[v2] Add blackbox tests

.github/workflows/update-lockfiles.yml

@@ -62,7 +62,10 @@ jobs:
       - name: Regenerate lock files
         if: runner.os != 'Windows'
         run: |
-          python scripts/regenerate-lock-files --show-files --include-sdist
+          python scripts/regenerate-lock-files \
+            --show-files \
+            --include-sdist \
+            --include-blackbox
           echo "CHANGES<<EOF" >> $GITHUB_ENV
           echo "CHANGES=$(git status --porcelain=v1)" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV

requirements-test-blackbox.txt

@@ -0,0 +1,6 @@
+pytest==8.4.0
+pytest-xdist==3.6.1
+pip-tools==7.6.1
+# localstub is a required dependency of
+# blackbox tests.
+localstub==0.0.3

scripts/regenerate-lock-files

@@ -103,7 +103,7 @@ class LockFileBuilder:
             f.write(content)
 
 
-def show_files(build_directory: Path, include_sdist: bool, include_base: bool):
+def show_files(build_directory: Path, include_sdist: bool, include_base: bool, include_blackbox: bool):
     if include_sdist:
         for root, dirs, files in os.walk(build_directory / 'requirements'):
             for filename in files:
@@ -115,6 +115,11 @@ def show_files(build_directory: Path, include_sdist: bool, include_base: bool):
             stemmed_filename = Path(filename).stem
             if stemmed_filename.endswith('-lock'):
                 show_file(filename)
+    if include_blackbox:
+        for filename in Path.iterdir(build_directory):
+            stemmed_filename = Path(filename).stem
+            if stemmed_filename.endswith('-lock') and 'blackbox' in stemmed_filename:
+                show_file(filename)
 
 
 def show_file(path: Path):
@@ -128,6 +133,7 @@ def main(
     should_show_files: bool,
     include_sdist: bool,
     include_base: bool,
+    include_blackbox: bool,
 ):
     builder = LockFileBuilder(
         source_directory=ROOT,
@@ -191,8 +197,16 @@ def main(
             output=Path("requirements-docs"),
             allow_unsafe=True,
         )
+    if include_blackbox:
+        builder.build_lock_file(
+            sources=[
+                Path("requirements-test-blackbox.txt"),
+            ],
+            output=Path("requirements-test-blackbox"),
+            allow_unsafe=True,
+        )
     if should_show_files:
-        show_files(build_directory, include_sdist, include_base)
+        show_files(build_directory, include_sdist, include_base, include_blackbox)
 
 
 if __name__ == "__main__":
@@ -218,10 +232,16 @@ if __name__ == "__main__":
         '--no-include-base', action='store_false', dest='include_base'
     )
     parser.set_defaults(include_base=False)
+    parser.add_argument('--include-blackbox', action='store_true')
+    parser.add_argument(
+        '--no-include-blackbox', action='store_false', dest='include_blackbox'
+    )
+    parser.set_defaults(include_blackbox=False)
     args = parser.parse_args()
     main(
         args.output_directory,
         args.show_files,
         args.include_sdist,
         args.include_base,
+        args.include_blackbox,
     )

tests/blackbox/conftest.py

@@ -0,0 +1,82 @@
+"""Shared fixtures for blackbox tests."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import shutil
+
+import pytest
+
+from tests.blackbox.utils import cli_env, mock_server, run_cli
+from localstub.server import HTTPResponse
+
+
+def pytest_configure(config):
+    """Heuristic to verify if the CLI binary routes traffic through
+    HTTPS_PROXY before any test runs.
+
+    If the proxy receives no request, the binary under test does not support
+    HTTPS_PROXY and all tests would be invalid — fail the session immediately.
+    """
+    command = os.environ.get("AWS_TEST_COMMAND") or shutil.which("aws")
+    if command is None:
+        pytest.exit(
+            "No AWS CLI binary found: set AWS_TEST_COMMAND to its path or "
+            "make `aws` available on PATH.",
+            returncode=1,
+        )
+
+    async def _verify():
+
+        async with mock_server() as (server, proxy):
+            server.set_response_sequence([
+                HTTPResponse.raw(
+                    b'<?xml version="1.0" ?>'
+                    b"<Error><Code>AccessDenied</Code>"
+                    b"<Message>Test</Message></Error>",
+                    status=403,
+                    headers={"Content-Type": "application/xml"},
+                ),
+            ])
+            env = cli_env(proxy)
+            await run_cli(command, ["s3", "ls"], env)
+            return len(server.requests) > 0
+
+    proxy_works = asyncio.run(_verify())
+    if not proxy_works:
+        pytest.exit(
+            f"CLI binary {command!r} does not route traffic through "
+            f"HTTPS_PROXY. All blackbox tests require proxy support to "
+            f"prevent accidental calls to production AWS endpoints.",
+            returncode=1,
+        )
+
+
+@pytest.fixture
+def aws_cli() -> str:
+    command = os.environ.get("AWS_TEST_COMMAND") or shutil.which("aws")
+    if command is None:
+        pytest.fail(
+            "No AWS CLI binary found: set AWS_TEST_COMMAND to its path or "
+            "make `aws` available on PATH."
+        )
+    return command
+
+
+@pytest.fixture
+def aws_config(tmp_path):
+    """Factory fixture that writes an AWS config file and returns its path."""
+
+    def _make(config_dict: dict[str, dict[str, str]]) -> str:
+        lines = []
+        for section, values in config_dict.items():
+            lines.append(f"[{section}]")
+            for key, val in values.items():
+                lines.append(f"{key} = {val}")
+            lines.append("")
+        path = tmp_path / "config"
+        path.write_text("\n".join(lines))
+        return str(path)
+
+    return _make