diff --git a/.github/scripts/build_lambda_layer.py b/.github/scripts/build_lambda_layer.py index ff903405..56a719c3 100644 --- a/.github/scripts/build_lambda_layer.py +++ b/.github/scripts/build_lambda_layer.py @@ -14,25 +14,19 @@ from pathlib import Path -ARCHITECTURE_PLATFORMS = { - "x86_64": "manylinux2014_x86_64", - "arm64": "manylinux2014_aarch64", -} -SUPPORTED_PYTHON_VERSIONS = ("3.11", "3.12", "3.13", "3.14") +UNIVERSAL_WHEEL_SUFFIX = "-py3-none-any.whl" @dataclass(frozen=True) class BuildConfig: output: Path - target_python: str - architecture: str sdk_distribution: Path otel_distribution: Path build_dir: Path | None = None def build_layer(config: BuildConfig) -> Path: - """Build a Lambda layer containing the SDK and OpenTelemetry plugin.""" + """Build a universal Lambda layer containing the SDK and OTel plugin.""" _validate_config(config) @@ -44,30 +38,20 @@ def build_layer(config: BuildConfig) -> Path: layer_python_dir = work_dir / "python" layer_python_dir.mkdir(parents=True) - _install_layer_dependencies(config, layer_python_dir) + _install_layer_distributions(config, layer_python_dir) _write_zip(config.output, work_dir) return config.output def _validate_config(config: BuildConfig) -> None: - if config.architecture not in ARCHITECTURE_PLATFORMS: - supported = ", ".join(sorted(ARCHITECTURE_PLATFORMS)) - raise ValueError( - f"Unsupported architecture: {config.architecture}. " - f"Supported architectures: {supported}" - ) - - if config.target_python not in SUPPORTED_PYTHON_VERSIONS: - supported = ", ".join(SUPPORTED_PYTHON_VERSIONS) - raise ValueError( - f"Unsupported Python version: {config.target_python}. " - f"Supported versions: {supported}" - ) - for distribution in (config.sdk_distribution, config.otel_distribution): if not distribution.is_file(): raise FileNotFoundError(distribution) + if not distribution.name.endswith(UNIVERSAL_WHEEL_SUFFIX): + raise ValueError( + f"Layer distributions must be universal wheels: {distribution}" + ) def _validate_output_location(output: Path, build_dir: Path) -> None: @@ -75,11 +59,7 @@ def _validate_output_location(output: Path, build_dir: Path) -> None: raise ValueError("Layer output must be outside the build directory") -def _install_layer_dependencies(config: BuildConfig, target_dir: Path) -> None: - python_version = config.target_python - abi = f"cp{python_version.replace('.', '')}" - platform = ARCHITECTURE_PLATFORMS[config.architecture] - +def _install_layer_distributions(config: BuildConfig, target_dir: Path) -> None: command = [ sys.executable, "-m", @@ -88,17 +68,10 @@ def _install_layer_dependencies(config: BuildConfig, target_dir: Path) -> None: "--upgrade", "--target", str(target_dir), - "--platform", - platform, - "--implementation", - "cp", - "--python-version", - python_version, - "--abi", - abi, "--only-binary", ":all:", "--no-compile", + "--no-deps", str(config.sdk_distribution), str(config.otel_distribution), ] @@ -119,21 +92,9 @@ def _write_zip(output: Path, layer_root: Path) -> None: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( prog="build_lambda_layer.py", - description="Build the AWS Durable Execution SDK OTel plugin Lambda layer.", + description="Build the universal AWS Durable Execution SDK OTel plugin layer.", ) parser.add_argument("--output", type=Path, required=True) - parser.add_argument( - "--target-python", - choices=SUPPORTED_PYTHON_VERSIONS, - required=True, - help="Lambda Python minor version.", - ) - parser.add_argument( - "--architecture", - choices=sorted(ARCHITECTURE_PLATFORMS), - required=True, - help="Lambda instruction set architecture.", - ) parser.add_argument("--sdk-distribution", type=Path, required=True) parser.add_argument("--otel-distribution", type=Path, required=True) parser.add_argument( @@ -146,8 +107,6 @@ def main(argv: list[str] | None = None) -> int: output = build_layer( BuildConfig( output=args.output, - target_python=args.target_python, - architecture=args.architecture, sdk_distribution=args.sdk_distribution, otel_distribution=args.otel_distribution, build_dir=args.build_dir, diff --git a/.github/scripts/tests/test_build_lambda_layer.py b/.github/scripts/tests/test_build_lambda_layer.py index 7671b48a..b60e720d 100644 --- a/.github/scripts/tests/test_build_lambda_layer.py +++ b/.github/scripts/tests/test_build_lambda_layer.py @@ -14,7 +14,45 @@ from build_lambda_layer import BuildConfig, build_layer -def test_build_layer_installs_dependencies_and_zips_lambda_layout( +def _write_test_wheel( + directory: Path, + distribution: str, + package_files: tuple[str, ...], + dependencies: tuple[str, ...] = (), +) -> Path: + normalized_distribution = distribution.replace("-", "_") + wheel = directory / f"{normalized_distribution}-1.0.0-py3-none-any.whl" + dist_info = f"{normalized_distribution}-1.0.0.dist-info" + metadata = [ + "Metadata-Version: 2.1", + f"Name: {distribution}", + "Version: 1.0.0", + *(f"Requires-Dist: {dependency}==1.0.0" for dependency in dependencies), + "", + ] + + with zipfile.ZipFile(wheel, "w") as archive: + for package_file in package_files: + archive.writestr(package_file, "") + archive.writestr(f"{dist_info}/METADATA", "\n".join(metadata)) + archive.writestr( + f"{dist_info}/WHEEL", + "\n".join( + ( + "Wheel-Version: 1.0", + "Generator: test", + "Root-Is-Purelib: true", + "Tag: py3-none-any", + "", + ) + ), + ) + archive.writestr(f"{dist_info}/RECORD", "") + + return wheel + + +def test_build_layer_installs_distributions_and_zips_lambda_layout( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: @@ -42,17 +80,17 @@ def fake_run(command: list[str], check: bool) -> subprocess.CompletedProcess[str output = build_layer( BuildConfig( output=tmp_path / "layer.zip", - target_python="3.12", - architecture="arm64", sdk_distribution=sdk_wheel, otel_distribution=otel_wheel, ) ) assert output == tmp_path / "layer.zip" - assert commands[0][commands[0].index("--platform") + 1] == "manylinux2014_aarch64" - assert commands[0][commands[0].index("--abi") + 1] == "cp312" assert "--no-compile" in commands[0] + assert "--no-deps" in commands[0] + assert "--platform" not in commands[0] + assert "--python-version" not in commands[0] + assert "--abi" not in commands[0] assert str(sdk_wheel) in commands[0] assert str(otel_wheel) in commands[0] @@ -66,34 +104,52 @@ def fake_run(command: list[str], check: bool) -> subprocess.CompletedProcess[str ) -@pytest.mark.parametrize( - ("target_python", "architecture", "error"), - [ - ("3.10", "x86_64", "Unsupported Python version"), - ("3.12", "sparc", "Unsupported architecture"), - ], -) -def test_build_layer_rejects_unsupported_targets( - target_python: str, - architecture: str, - error: str, +def test_build_layer_excludes_adot_and_runtime_dependencies( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - sdk_wheel = tmp_path / "sdk.whl" - otel_wheel = tmp_path / "otel.whl" - sdk_wheel.write_text("sdk") - otel_wheel.write_text("otel") + sdk_wheel = _write_test_wheel( + tmp_path, + "aws-durable-execution-sdk-python", + ("aws_durable_execution_sdk_python/__init__.py",), + ("boto3",), + ) + otel_wheel = _write_test_wheel( + tmp_path, + "aws-durable-execution-sdk-python-otel", + ("aws_durable_execution_sdk_python_otel/__init__.py",), + ("aws-opentelemetry-distro", "opentelemetry-api"), + ) + _write_test_wheel(tmp_path, "boto3", ("boto3/__init__.py",)) + _write_test_wheel( + tmp_path, + "aws-opentelemetry-distro", + ("amazon/opentelemetry/distro/__init__.py",), + ) + _write_test_wheel( + tmp_path, + "opentelemetry-api", + ("opentelemetry/__init__.py",), + ) + monkeypatch.setenv("PIP_FIND_LINKS", str(tmp_path)) + monkeypatch.setenv("PIP_NO_INDEX", "1") - with pytest.raises(ValueError, match=error): - build_layer( - BuildConfig( - output=tmp_path / "layer.zip", - target_python=target_python, - architecture=architecture, - sdk_distribution=sdk_wheel, - otel_distribution=otel_wheel, - ) + output = build_layer( + BuildConfig( + output=tmp_path / "layer.zip", + sdk_distribution=sdk_wheel, + otel_distribution=otel_wheel, ) + ) + + with zipfile.ZipFile(output) as archive: + names = archive.namelist() + + assert "python/aws_durable_execution_sdk_python/__init__.py" in names + assert "python/aws_durable_execution_sdk_python_otel/__init__.py" in names + assert not any(name.startswith("python/amazon/") for name in names) + assert not any(name.startswith("python/boto3/") for name in names) + assert not any(name.startswith("python/opentelemetry/") for name in names) def test_build_layer_requires_built_distributions(tmp_path: Path) -> None: @@ -101,17 +157,31 @@ def test_build_layer_requires_built_distributions(tmp_path: Path) -> None: build_layer( BuildConfig( output=tmp_path / "layer.zip", - target_python="3.13", - architecture="x86_64", sdk_distribution=tmp_path / "missing-sdk.whl", otel_distribution=tmp_path / "missing-otel.whl", ) ) +def test_build_layer_requires_universal_wheels(tmp_path: Path) -> None: + sdk_wheel = tmp_path / "sdk-1.0.0-cp311-cp311-manylinux2014_x86_64.whl" + otel_wheel = tmp_path / "otel-1.0.0-py3-none-any.whl" + sdk_wheel.write_text("sdk") + otel_wheel.write_text("otel") + + with pytest.raises(ValueError, match="must be universal wheels"): + build_layer( + BuildConfig( + output=tmp_path / "layer.zip", + sdk_distribution=sdk_wheel, + otel_distribution=otel_wheel, + ) + ) + + def test_build_layer_rejects_output_inside_build_directory(tmp_path: Path) -> None: - sdk_wheel = tmp_path / "sdk.whl" - otel_wheel = tmp_path / "otel.whl" + sdk_wheel = tmp_path / "sdk-1.0.0-py3-none-any.whl" + otel_wheel = tmp_path / "otel-1.0.0-py3-none-any.whl" sdk_wheel.write_text("sdk") otel_wheel.write_text("otel") build_dir = tmp_path / "layer" @@ -120,8 +190,6 @@ def test_build_layer_rejects_output_inside_build_directory(tmp_path: Path) -> No build_layer( BuildConfig( output=build_dir / "layer.zip", - target_python="3.13", - architecture="x86_64", sdk_distribution=sdk_wheel, otel_distribution=otel_wheel, build_dir=build_dir, diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index 221296a6..b28c9792 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -140,45 +140,9 @@ jobs: if-no-files-found: error retention-days: 7 - build-layers: + build-layer: needs: build-distributions runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - target_python: "3.11" - runtime: python3.11 - runtime_slug: python311 - architecture: x86_64 - - target_python: "3.11" - runtime: python3.11 - runtime_slug: python311 - architecture: arm64 - - target_python: "3.12" - runtime: python3.12 - runtime_slug: python312 - architecture: x86_64 - - target_python: "3.12" - runtime: python3.12 - runtime_slug: python312 - architecture: arm64 - - target_python: "3.13" - runtime: python3.13 - runtime_slug: python313 - architecture: x86_64 - - target_python: "3.13" - runtime: python3.13 - runtime_slug: python313 - architecture: arm64 - - target_python: "3.14" - runtime: python3.14 - runtime_slug: python314 - architecture: x86_64 - - target_python: "3.14" - runtime: python3.14 - runtime_slug: python314 - architecture: arm64 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -188,7 +152,7 @@ jobs: - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: ${{ matrix.target_python }} + python-version: "3.11" - name: Download wheels uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -199,28 +163,26 @@ jobs: - name: Build layer zip id: build-layer env: - LAYER_ZIP: dist/${{ env.LAYER_NAME }}-${{ matrix.runtime_slug }}-${{ matrix.architecture }}.zip + LAYER_ZIP: dist/${{ env.LAYER_NAME }}.zip run: | SDK_WHEEL=$(find release-dists -name 'aws_durable_execution_sdk_python-*.whl' -print -quit) OTEL_WHEEL=$(find release-dists -name 'aws_durable_execution_sdk_python_otel-*.whl' -print -quit) python .github/scripts/build_lambda_layer.py \ --sdk-distribution "$SDK_WHEEL" \ --otel-distribution "$OTEL_WHEEL" \ - --target-python "${{ matrix.target_python }}" \ - --architecture "${{ matrix.architecture }}" \ --output "$LAYER_ZIP" echo "layer_zip=${LAYER_ZIP}" >> "$GITHUB_OUTPUT" - name: Upload layer artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: otel-plugin-layer-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + name: otel-plugin-layer path: ${{ steps.build-layer.outputs.layer_zip }} if-no-files-found: error retention-days: 30 publish-layer: - needs: [build-distributions, build-layers] + needs: [build-distributions, build-layer] runs-on: ubuntu-latest environment: name: lambda-layer-publish @@ -229,60 +191,24 @@ jobs: id-token: write env: LAYER_REGIONS: ${{ inputs.regions || vars.LAYER_PUBLISH_REGIONS }} - strategy: - fail-fast: false - matrix: - include: - - target_python: "3.11" - runtime: python3.11 - runtime_slug: python311 - architecture: x86_64 - - target_python: "3.11" - runtime: python3.11 - runtime_slug: python311 - architecture: arm64 - - target_python: "3.12" - runtime: python3.12 - runtime_slug: python312 - architecture: x86_64 - - target_python: "3.12" - runtime: python3.12 - runtime_slug: python312 - architecture: arm64 - - target_python: "3.13" - runtime: python3.13 - runtime_slug: python313 - architecture: x86_64 - - target_python: "3.13" - runtime: python3.13 - runtime_slug: python313 - architecture: arm64 - - target_python: "3.14" - runtime: python3.14 - runtime_slug: python314 - architecture: x86_64 - - target_python: "3.14" - runtime: python3.14 - runtime_slug: python314 - architecture: arm64 steps: - name: Download layer artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: otel-plugin-layer-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + name: otel-plugin-layer path: dist/ - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 with: role-to-assume: ${{ secrets.LAYER_PUBLISH_ROLE_ARN }} - role-session-name: otelLayerPublish-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + role-session-name: otelLayerPublish aws-region: us-east-1 - name: Publish layer versions env: - LAYER_ZIP: dist/${{ env.LAYER_NAME }}-${{ matrix.runtime_slug }}-${{ matrix.architecture }}.zip + LAYER_ZIP: dist/${{ env.LAYER_NAME }}.zip SDK_VERSION: ${{ needs.build-distributions.outputs.sdk_version }} OTEL_VERSION: ${{ needs.build-distributions.outputs.otel_version }} run: | @@ -297,11 +223,9 @@ jobs: continue fi - LAYER_DESCRIPTION="AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }}) sha256:${LOCAL_CODE_SHA256}" + LAYER_DESCRIPTION="AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} sha256:${LOCAL_CODE_SHA256}" if ! EXISTING_RESULT=$(aws lambda list-layer-versions \ --layer-name "$LAYER_NAME" \ - --compatible-runtime "${{ matrix.runtime }}" \ - --compatible-architecture "${{ matrix.architecture }}" \ --region "$REGION" \ --query "LayerVersions[?Description=='${LAYER_DESCRIPTION}'] | [0].[LayerVersionArn,Version]" \ --output text 2>&1); then @@ -339,8 +263,8 @@ jobs: --layer-name "$LAYER_NAME" \ --description "$LAYER_DESCRIPTION" \ --zip-file "fileb://${LAYER_ZIP}" \ - --compatible-runtimes "${{ matrix.runtime }}" \ - --compatible-architectures "${{ matrix.architecture }}" \ + --compatible-runtimes python3.11 python3.12 python3.13 python3.14 \ + --compatible-architectures x86_64 arm64 \ --license-info Apache-2.0 \ --region "$REGION" \ --query '[LayerVersionArn,Version,Content.CodeSha256]' \