diff --git a/.github/lambda-layer-publish.toml b/.github/lambda-layer-publish.toml new file mode 100644 index 00000000..279a10b8 --- /dev/null +++ b/.github/lambda-layer-publish.toml @@ -0,0 +1,2 @@ +[layer] +sdk-version = "1.8.0" diff --git a/.github/scripts/build_lambda_layer.py b/.github/scripts/build_lambda_layer.py new file mode 100644 index 00000000..ff903405 --- /dev/null +++ b/.github/scripts/build_lambda_layer.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + + +ARCHITECTURE_PLATFORMS = { + "x86_64": "manylinux2014_x86_64", + "arm64": "manylinux2014_aarch64", +} +SUPPORTED_PYTHON_VERSIONS = ("3.11", "3.12", "3.13", "3.14") + + +@dataclass(frozen=True) +class BuildConfig: + output: Path + target_python: str + architecture: str + sdk_distribution: Path + otel_distribution: Path + build_dir: Path | None = None + + +def build_layer(config: BuildConfig) -> Path: + """Build a Lambda layer containing the SDK and OpenTelemetry plugin.""" + + _validate_config(config) + + with tempfile.TemporaryDirectory() as temp_dir: + work_dir = config.build_dir or Path(temp_dir) / "layer" + _validate_output_location(config.output, work_dir) + if work_dir.exists(): + shutil.rmtree(work_dir) + layer_python_dir = work_dir / "python" + layer_python_dir.mkdir(parents=True) + + _install_layer_dependencies(config, layer_python_dir) + _write_zip(config.output, work_dir) + + return config.output + + +def _validate_config(config: BuildConfig) -> None: + if config.architecture not in ARCHITECTURE_PLATFORMS: + supported = ", ".join(sorted(ARCHITECTURE_PLATFORMS)) + raise ValueError( + f"Unsupported architecture: {config.architecture}. " + f"Supported architectures: {supported}" + ) + + if config.target_python not in SUPPORTED_PYTHON_VERSIONS: + supported = ", ".join(SUPPORTED_PYTHON_VERSIONS) + raise ValueError( + f"Unsupported Python version: {config.target_python}. " + f"Supported versions: {supported}" + ) + + for distribution in (config.sdk_distribution, config.otel_distribution): + if not distribution.is_file(): + raise FileNotFoundError(distribution) + + +def _validate_output_location(output: Path, build_dir: Path) -> None: + if output.resolve().is_relative_to(build_dir.resolve()): + raise ValueError("Layer output must be outside the build directory") + + +def _install_layer_dependencies(config: BuildConfig, target_dir: Path) -> None: + python_version = config.target_python + abi = f"cp{python_version.replace('.', '')}" + platform = ARCHITECTURE_PLATFORMS[config.architecture] + + command = [ + sys.executable, + "-m", + "pip", + "install", + "--upgrade", + "--target", + str(target_dir), + "--platform", + platform, + "--implementation", + "cp", + "--python-version", + python_version, + "--abi", + abi, + "--only-binary", + ":all:", + "--no-compile", + str(config.sdk_distribution), + str(config.otel_distribution), + ] + subprocess.run(command, check=True) + + +def _write_zip(output: Path, layer_root: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + output.unlink() + + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path in sorted(layer_root.rglob("*")): + if path.is_file(): + archive.write(path, path.relative_to(layer_root)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="build_lambda_layer.py", + description="Build the AWS Durable Execution SDK OTel plugin Lambda layer.", + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--target-python", + choices=SUPPORTED_PYTHON_VERSIONS, + required=True, + help="Lambda Python minor version.", + ) + parser.add_argument( + "--architecture", + choices=sorted(ARCHITECTURE_PLATFORMS), + required=True, + help="Lambda instruction set architecture.", + ) + parser.add_argument("--sdk-distribution", type=Path, required=True) + parser.add_argument("--otel-distribution", type=Path, required=True) + parser.add_argument( + "--build-dir", + type=Path, + help="Optional scratch directory. Existing contents are replaced.", + ) + + args = parser.parse_args(argv) + output = build_layer( + BuildConfig( + output=args.output, + target_python=args.target_python, + architecture=args.architecture, + sdk_distribution=args.sdk_distribution, + otel_distribution=args.otel_distribution, + build_dir=args.build_dir, + ) + ) + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/resolve_layer_sdk_version.py b/.github/scripts/resolve_layer_sdk_version.py new file mode 100644 index 00000000..d96b53fd --- /dev/null +++ b/.github/scripts/resolve_layer_sdk_version.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import argparse +import tomllib +from pathlib import Path + + +def _read_pinned_sdk_version(metadata_path: Path) -> str: + with metadata_path.open("rb") as file: + metadata = tomllib.load(file) + + try: + pinned_version = metadata["layer"]["sdk-version"] + except KeyError as error: + raise ValueError(f"{metadata_path}: missing layer.sdk-version") from error + + if not isinstance(pinned_version, str) or not pinned_version: + raise ValueError(f"{metadata_path}: layer.sdk-version must be a string") + return pinned_version + + +def resolve_layer_sdk_version( + event_name: str, + release_tag: str, + source_sdk_version: str, + metadata_path: Path, +) -> str: + """Resolve the SDK version bundled in the OTel plugin Lambda layer.""" + + pinned_version = _read_pinned_sdk_version(metadata_path) + sdk_release = any(part.startswith("sdk-v") for part in release_tag.split(",")) + + if event_name == "release" and not sdk_release: + return pinned_version + + if event_name == "release" and pinned_version != source_sdk_version: + raise ValueError( + f"New SDK releases must update layer.sdk-version to {source_sdk_version}" + ) + + return source_sdk_version + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Resolve the SDK version for the OTel plugin Lambda layer." + ) + parser.add_argument("--event-name", required=True) + parser.add_argument("--release-tag", default="") + parser.add_argument("--source-sdk-version", required=True) + parser.add_argument("--metadata", type=Path, required=True) + args = parser.parse_args(argv) + + print( + resolve_layer_sdk_version( + event_name=args.event_name, + release_tag=args.release_tag, + source_sdk_version=args.source_sdk_version, + metadata_path=args.metadata, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/tests/test_build_lambda_layer.py b/.github/scripts/tests/test_build_lambda_layer.py new file mode 100644 index 00000000..7671b48a --- /dev/null +++ b/.github/scripts/tests/test_build_lambda_layer.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import zipfile +from pathlib import Path + +import pytest + + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from build_lambda_layer import BuildConfig, build_layer + + +def test_build_layer_installs_dependencies_and_zips_lambda_layout( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + sdk_wheel = tmp_path / "aws_durable_execution_sdk_python-1.0.0-py3-none-any.whl" + otel_wheel = ( + tmp_path / "aws_durable_execution_sdk_python_otel-1.0.0-py3-none-any.whl" + ) + sdk_wheel.write_text("sdk") + otel_wheel.write_text("otel") + commands: list[list[str]] = [] + + def fake_run(command: list[str], check: bool) -> subprocess.CompletedProcess[str]: + commands.append(command) + target = Path(command[command.index("--target") + 1]) + (target / "aws_durable_execution_sdk_python").mkdir() + (target / "aws_durable_execution_sdk_python" / "__init__.py").write_text("") + (target / "aws_durable_execution_sdk_python_otel").mkdir() + (target / "aws_durable_execution_sdk_python_otel" / "__init__.py").write_text( + "" + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + + output = build_layer( + BuildConfig( + output=tmp_path / "layer.zip", + target_python="3.12", + architecture="arm64", + sdk_distribution=sdk_wheel, + otel_distribution=otel_wheel, + ) + ) + + assert output == tmp_path / "layer.zip" + assert commands[0][commands[0].index("--platform") + 1] == "manylinux2014_aarch64" + assert commands[0][commands[0].index("--abi") + 1] == "cp312" + assert "--no-compile" in commands[0] + assert str(sdk_wheel) in commands[0] + assert str(otel_wheel) in commands[0] + + with zipfile.ZipFile(output) as archive: + assert ( + "python/aws_durable_execution_sdk_python/__init__.py" in archive.namelist() + ) + assert ( + "python/aws_durable_execution_sdk_python_otel/__init__.py" + in archive.namelist() + ) + + +@pytest.mark.parametrize( + ("target_python", "architecture", "error"), + [ + ("3.10", "x86_64", "Unsupported Python version"), + ("3.12", "sparc", "Unsupported architecture"), + ], +) +def test_build_layer_rejects_unsupported_targets( + target_python: str, + architecture: str, + error: str, + tmp_path: Path, +) -> None: + sdk_wheel = tmp_path / "sdk.whl" + otel_wheel = tmp_path / "otel.whl" + sdk_wheel.write_text("sdk") + otel_wheel.write_text("otel") + + with pytest.raises(ValueError, match=error): + build_layer( + BuildConfig( + output=tmp_path / "layer.zip", + target_python=target_python, + architecture=architecture, + sdk_distribution=sdk_wheel, + otel_distribution=otel_wheel, + ) + ) + + +def test_build_layer_requires_built_distributions(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + build_layer( + BuildConfig( + output=tmp_path / "layer.zip", + target_python="3.13", + architecture="x86_64", + sdk_distribution=tmp_path / "missing-sdk.whl", + otel_distribution=tmp_path / "missing-otel.whl", + ) + ) + + +def test_build_layer_rejects_output_inside_build_directory(tmp_path: Path) -> None: + sdk_wheel = tmp_path / "sdk.whl" + otel_wheel = tmp_path / "otel.whl" + sdk_wheel.write_text("sdk") + otel_wheel.write_text("otel") + build_dir = tmp_path / "layer" + + with pytest.raises(ValueError, match="outside the build directory"): + build_layer( + BuildConfig( + output=build_dir / "layer.zip", + target_python="3.13", + architecture="x86_64", + sdk_distribution=sdk_wheel, + otel_distribution=otel_wheel, + build_dir=build_dir, + ) + ) diff --git a/.github/scripts/tests/test_resolve_layer_sdk_version.py b/.github/scripts/tests/test_resolve_layer_sdk_version.py new file mode 100644 index 00000000..5308179d --- /dev/null +++ b/.github/scripts/tests/test_resolve_layer_sdk_version.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest + + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from resolve_layer_sdk_version import resolve_layer_sdk_version + + +def _write_metadata(tmp_path: Path, sdk_version: str) -> Path: + metadata = tmp_path / "lambda-layer-publish.toml" + metadata.write_text(f'[layer]\nsdk-version = "{sdk_version}"\n') + return metadata + + +def test_otel_only_release_uses_pinned_published_sdk(tmp_path: Path) -> None: + metadata = _write_metadata(tmp_path, "1.8.0") + + assert ( + resolve_layer_sdk_version( + event_name="release", + release_tag="otel-v0.4.1", + source_sdk_version="1.9.0", + metadata_path=metadata, + ) + == "1.8.0" + ) + + +def test_combined_release_uses_source_sdk(tmp_path: Path) -> None: + metadata = _write_metadata(tmp_path, "1.9.0") + + assert ( + resolve_layer_sdk_version( + event_name="release", + release_tag="sdk-v1.9.0,otel-v0.5.0", + source_sdk_version="1.9.0", + metadata_path=metadata, + ) + == "1.9.0" + ) + + +def test_combined_release_requires_updated_pin(tmp_path: Path) -> None: + metadata = _write_metadata(tmp_path, "1.8.0") + + with pytest.raises(ValueError, match="New SDK releases must update"): + resolve_layer_sdk_version( + event_name="release", + release_tag="sdk-v1.9.0,otel-v0.5.0", + source_sdk_version="1.9.0", + metadata_path=metadata, + ) + + +def test_manual_run_uses_source_sdk(tmp_path: Path) -> None: + metadata = _write_metadata(tmp_path, "1.8.0") + + assert ( + resolve_layer_sdk_version( + event_name="workflow_dispatch", + release_tag="", + source_sdk_version="1.9.0", + metadata_path=metadata, + ) + == "1.9.0" + ) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml new file mode 100644 index 00000000..221296a6 --- /dev/null +++ b/.github/workflows/lambda-layer-publish.yml @@ -0,0 +1,389 @@ +name: Publish OTel Plugin Lambda Layer + +on: + release: + types: [published] + workflow_dispatch: + inputs: + regions: + description: "Comma-separated AWS Regions; defaults to all commercial Regions" + required: false + type: string + +permissions: + contents: read + +concurrency: + group: lambda-layer-publish-${{ github.event.release.tag_name || github.ref_name }} + cancel-in-progress: false + +env: + LAYER_NAME: aws-durable-execution-sdk-python-otel-plugin + SOURCE_REF: ${{ github.event_name == 'workflow_dispatch' && 'main' || github.event.release.tag_name }} + DEFAULT_LAYER_REGIONS: >- + af-south-1, + ap-east-1, + ap-east-2, + ap-northeast-1, + ap-northeast-2, + ap-northeast-3, + ap-south-1, + ap-south-2, + ap-southeast-1, + ap-southeast-2, + ap-southeast-3, + ap-southeast-4, + ap-southeast-5, + ap-southeast-6, + ap-southeast-7, + ca-central-1, + ca-west-1, + eu-central-1, + eu-central-2, + eu-north-1, + eu-south-1, + eu-south-2, + eu-west-1, + eu-west-2, + eu-west-3, + il-central-1, + me-central-1, + me-south-1, + mx-central-1, + sa-east-1, + us-east-1, + us-east-2, + us-west-1, + us-west-2 + +jobs: + build-distributions: + if: >- + (github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/main') || + contains(github.event.release.tag_name, 'otel-v') + runs-on: ubuntu-latest + outputs: + sdk_version: ${{ steps.versions.outputs.sdk_version }} + otel_version: ${{ steps.versions.outputs.otel_version }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.SOURCE_REF }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Install Hatch + run: python -m pip install --upgrade hatch==1.16.5 + + - name: Read package versions + id: versions + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: | + SOURCE_SDK_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/__about__.py | cut -d'"' -f2) + OTEL_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python-otel/src/aws_durable_execution_sdk_python_otel/__about__.py | cut -d'"' -f2) + SDK_VERSION=$(python .github/scripts/resolve_layer_sdk_version.py \ + --event-name "$GITHUB_EVENT_NAME" \ + --release-tag "$RELEASE_TAG" \ + --source-sdk-version "$SOURCE_SDK_VERSION" \ + --metadata .github/lambda-layer-publish.toml) + echo "sdk_version=${SDK_VERSION}" >> "$GITHUB_OUTPUT" + echo "otel_version=${OTEL_VERSION}" >> "$GITHUB_OUTPUT" + + - name: Build SDK distribution + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.release.tag_name, 'sdk-v') + working-directory: packages/aws-durable-execution-sdk-python + run: hatch build + + - name: Download published SDK distribution + if: >- + github.event_name == 'release' && + !contains(github.event.release.tag_name, 'sdk-v') + env: + SDK_VERSION: ${{ steps.versions.outputs.sdk_version }} + working-directory: packages/aws-durable-execution-sdk-python + run: | + python -m pip download \ + --dest dist \ + --no-deps \ + --only-binary=:all: \ + "aws-durable-execution-sdk-python==${SDK_VERSION}" + + - name: Build OTel plugin distribution + working-directory: packages/aws-durable-execution-sdk-python-otel + run: hatch build + + - name: Verify legal files + run: | + python .github/scripts/check_dist_legal_files.py \ + packages/aws-durable-execution-sdk-python \ + packages/aws-durable-execution-sdk-python-otel + + - name: Stage wheels + run: | + mkdir release-dists + cp packages/aws-durable-execution-sdk-python/dist/*.whl release-dists/ + cp packages/aws-durable-execution-sdk-python-otel/dist/*.whl release-dists/ + + - name: Upload wheels + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lambda-layer-distributions + path: release-dists/ + if-no-files-found: error + retention-days: 7 + + build-layers: + needs: build-distributions + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - target_python: "3.11" + runtime: python3.11 + runtime_slug: python311 + architecture: x86_64 + - target_python: "3.11" + runtime: python3.11 + runtime_slug: python311 + architecture: arm64 + - target_python: "3.12" + runtime: python3.12 + runtime_slug: python312 + architecture: x86_64 + - target_python: "3.12" + runtime: python3.12 + runtime_slug: python312 + architecture: arm64 + - target_python: "3.13" + runtime: python3.13 + runtime_slug: python313 + architecture: x86_64 + - target_python: "3.13" + runtime: python3.13 + runtime_slug: python313 + architecture: arm64 + - target_python: "3.14" + runtime: python3.14 + runtime_slug: python314 + architecture: x86_64 + - target_python: "3.14" + runtime: python3.14 + runtime_slug: python314 + architecture: arm64 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.SOURCE_REF }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.target_python }} + + - name: Download wheels + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: lambda-layer-distributions + path: release-dists/ + + - name: Build layer zip + id: build-layer + env: + LAYER_ZIP: dist/${{ env.LAYER_NAME }}-${{ matrix.runtime_slug }}-${{ matrix.architecture }}.zip + run: | + SDK_WHEEL=$(find release-dists -name 'aws_durable_execution_sdk_python-*.whl' -print -quit) + OTEL_WHEEL=$(find release-dists -name 'aws_durable_execution_sdk_python_otel-*.whl' -print -quit) + python .github/scripts/build_lambda_layer.py \ + --sdk-distribution "$SDK_WHEEL" \ + --otel-distribution "$OTEL_WHEEL" \ + --target-python "${{ matrix.target_python }}" \ + --architecture "${{ matrix.architecture }}" \ + --output "$LAYER_ZIP" + echo "layer_zip=${LAYER_ZIP}" >> "$GITHUB_OUTPUT" + + - name: Upload layer artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: otel-plugin-layer-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + path: ${{ steps.build-layer.outputs.layer_zip }} + if-no-files-found: error + retention-days: 30 + + publish-layer: + needs: [build-distributions, build-layers] + runs-on: ubuntu-latest + environment: + name: lambda-layer-publish + permissions: + contents: read + id-token: write + env: + LAYER_REGIONS: ${{ inputs.regions || vars.LAYER_PUBLISH_REGIONS }} + strategy: + fail-fast: false + matrix: + include: + - target_python: "3.11" + runtime: python3.11 + runtime_slug: python311 + architecture: x86_64 + - target_python: "3.11" + runtime: python3.11 + runtime_slug: python311 + architecture: arm64 + - target_python: "3.12" + runtime: python3.12 + runtime_slug: python312 + architecture: x86_64 + - target_python: "3.12" + runtime: python3.12 + runtime_slug: python312 + architecture: arm64 + - target_python: "3.13" + runtime: python3.13 + runtime_slug: python313 + architecture: x86_64 + - target_python: "3.13" + runtime: python3.13 + runtime_slug: python313 + architecture: arm64 + - target_python: "3.14" + runtime: python3.14 + runtime_slug: python314 + architecture: x86_64 + - target_python: "3.14" + runtime: python3.14 + runtime_slug: python314 + architecture: arm64 + + steps: + - name: Download layer artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: otel-plugin-layer-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + path: dist/ + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: ${{ secrets.LAYER_PUBLISH_ROLE_ARN }} + role-session-name: otelLayerPublish-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + aws-region: us-east-1 + + - name: Publish layer versions + env: + LAYER_ZIP: dist/${{ env.LAYER_NAME }}-${{ matrix.runtime_slug }}-${{ matrix.architecture }}.zip + SDK_VERSION: ${{ needs.build-distributions.outputs.sdk_version }} + OTEL_VERSION: ${{ needs.build-distributions.outputs.otel_version }} + run: | + PUBLISHED=false + FAILED_REGIONS=() + LOCAL_CODE_SHA256=$(openssl dgst -sha256 -binary "$LAYER_ZIP" | openssl base64 -A) + REGION_LIST=${LAYER_REGIONS:-$DEFAULT_LAYER_REGIONS} + IFS=',' read -ra REGIONS <<< "$REGION_LIST" + for REGION in "${REGIONS[@]}"; do + REGION=$(echo "$REGION" | xargs) + if [ -z "$REGION" ]; then + continue + fi + + LAYER_DESCRIPTION="AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }}) sha256:${LOCAL_CODE_SHA256}" + if ! EXISTING_RESULT=$(aws lambda list-layer-versions \ + --layer-name "$LAYER_NAME" \ + --compatible-runtime "${{ matrix.runtime }}" \ + --compatible-architecture "${{ matrix.architecture }}" \ + --region "$REGION" \ + --query "LayerVersions[?Description=='${LAYER_DESCRIPTION}'] | [0].[LayerVersionArn,Version]" \ + --output text 2>&1); then + if [[ "$EXISTING_RESULT" == *"ResourceNotFoundException"* ]]; then + EXISTING_RESULT="" + else + echo "$EXISTING_RESULT" >&2 + echo "::warning::Failed to list existing layer versions in ${REGION}" + FAILED_REGIONS+=("$REGION") + continue + fi + fi + + if [ -n "$EXISTING_RESULT" ] && [ "$EXISTING_RESULT" != "None" ]; then + read -r LAYER_VERSION_ARN VERSION_NUMBER <<< "$EXISTING_RESULT" + if ! EXISTING_CODE_SHA256=$(aws lambda get-layer-version \ + --layer-name "$LAYER_NAME" \ + --version-number "$VERSION_NUMBER" \ + --region "$REGION" \ + --query 'Content.CodeSha256' \ + --output text 2>&1); then + echo "$EXISTING_CODE_SHA256" >&2 + echo "::warning::Failed to read ${LAYER_VERSION_ARN}" + FAILED_REGIONS+=("$REGION") + continue + fi + if [ "$EXISTING_CODE_SHA256" != "$LOCAL_CODE_SHA256" ]; then + echo "::error::Artifact hash mismatch for ${LAYER_VERSION_ARN}" + FAILED_REGIONS+=("$REGION") + continue + fi + echo "Reusing ${LAYER_VERSION_ARN}" + else + if ! PUBLISH_RESULT=$(aws lambda publish-layer-version \ + --layer-name "$LAYER_NAME" \ + --description "$LAYER_DESCRIPTION" \ + --zip-file "fileb://${LAYER_ZIP}" \ + --compatible-runtimes "${{ matrix.runtime }}" \ + --compatible-architectures "${{ matrix.architecture }}" \ + --license-info Apache-2.0 \ + --region "$REGION" \ + --query '[LayerVersionArn,Version,Content.CodeSha256]' \ + --output text); then + echo "::warning::Failed to publish the layer in ${REGION}" + FAILED_REGIONS+=("$REGION") + continue + fi + read -r LAYER_VERSION_ARN VERSION_NUMBER PUBLISHED_CODE_SHA256 <<< "$PUBLISH_RESULT" + if [ "$PUBLISHED_CODE_SHA256" != "$LOCAL_CODE_SHA256" ]; then + echo "::error::Published artifact hash mismatch for ${LAYER_VERSION_ARN}" + FAILED_REGIONS+=("$REGION") + continue + fi + fi + + if ! PERMISSION_RESULT=$(aws lambda add-layer-version-permission \ + --layer-name "$LAYER_NAME" \ + --version-number "$VERSION_NUMBER" \ + --statement-id public-layer-access \ + --action lambda:GetLayerVersion \ + --principal "*" \ + --region "$REGION" \ + 2>&1); then + if [[ "$PERMISSION_RESULT" != *"ResourceConflictException"* ]]; then + echo "$PERMISSION_RESULT" >&2 + echo "::warning::Failed to grant public access to ${LAYER_VERSION_ARN}" + FAILED_REGIONS+=("$REGION") + continue + fi + fi + + echo "Available ${LAYER_VERSION_ARN}" + echo "- \`${LAYER_VERSION_ARN}\`" >> "$GITHUB_STEP_SUMMARY" + PUBLISHED=true + done + + if [ "${#FAILED_REGIONS[@]}" -gt 0 ]; then + echo "::error::Layer publishing failed in: ${FAILED_REGIONS[*]}" + exit 1 + fi + + if [ "$PUBLISHED" != true ]; then + echo "No AWS regions were configured for layer publishing." + exit 1 + fi diff --git a/.github/workflows/test-parser.yml b/.github/workflows/test-parser.yml index 52d8836a..fde37133 100644 --- a/.github/workflows/test-parser.yml +++ b/.github/workflows/test-parser.yml @@ -1,13 +1,15 @@ -name: Test Parser +name: Test GitHub Scripts on: pull_request: paths: + - '.github/scripts/build_lambda_layer.py' - '.github/scripts/parse_sdk_branch.py' - '.github/scripts/tests/**' push: branches: [ main ] paths: + - '.github/scripts/build_lambda_layer.py' - '.github/scripts/parse_sdk_branch.py' - '.github/scripts/tests/**' @@ -15,10 +17,16 @@ permissions: contents: read jobs: - test-parser: + test-scripts: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Run parser tests - run: python .github/scripts/tests/test_parse_sdk_branch.py + + - name: Install test dependencies + run: python -m pip install pytest + + - name: Run script tests + run: | + python -m pytest \ + .github/scripts/tests/test_build_lambda_layer.py \ + .github/scripts/tests/test_parse_sdk_branch.py diff --git a/RELEASING.md b/RELEASING.md index cc9e1faa..6bae46b5 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -69,6 +69,29 @@ The workflow runs on the `release: [published]` event, so it fires whenever a re > **Note:** The workflow builds and publishes all packages in the matrix. Ensure the version in each package's `__about__.py` is correct before publishing. If only one package has a version bump, PyPI will reject the re-upload of the unchanged package (which is expected and harmless since `fail-fast: false` is set). +Releases containing an `otel-v` tag also trigger the +[`lambda-layer-publish.yml`](.github/workflows/lambda-layer-publish.yml) +workflow. It builds the SDK and OTel plugin into Lambda layers for each +supported Python runtime and architecture, then publishes public versions of the +`aws-durable-execution-sdk-python-otel-plugin` layer. +For OTel-only releases, the workflow downloads the exact SDK version pinned by +`layer.sdk-version` in `.github/lambda-layer-publish.toml`; that version must +already be published to PyPI. Combined SDK and OTel releases require the pin to +match the new SDK version and build both distributions from the tagged source. + +The publishing job uses the `lambda-layer-publish` GitHub environment and its +`LAYER_PUBLISH_ROLE_ARN` secret. Set the optional `LAYER_PUBLISH_REGIONS` +environment variable to a comma-separated list of AWS Regions. When unset, the +workflow publishes to every commercial AWS Region supported by Lambda. +The workflow can also be run manually from the Actions tab on `main`; its +optional `regions` input overrides `LAYER_PUBLISH_REGIONS` for that run. +Each runtime and architecture layer archive is built once and retained as a +workflow artifact so retries publish the exact same resolved dependencies. Its +SHA-256 is included in the layer description and verified before reuse. +The publishing role must allow `lambda:PublishLayerVersion` and +`lambda:AddLayerVersionPermission`, as well as `lambda:ListLayerVersions` and +`lambda:GetLayerVersion` for identity-checked, idempotent release retries. + ## Release Notes Format Release notes should maintain separate timelines for each package. Use the following structure: @@ -111,6 +134,8 @@ If only one package is being released, include only that package's section. Each Before publishing a release: - [ ] Version bumped in the relevant `__about__.py` file(s) +- [ ] OTel layer SDK pin identifies a compatible published SDK, or matches the + SDK version included in a combined release - [ ] Changes merged to `main` - [ ] CI checks pass on `main` - [ ] Release notes written with separate sections per package