From aad9124730ed5d1e1f0f901ad183a824ef16edb8 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:28:34 -0700 Subject: [PATCH 01/11] ci: publish otel plugin lambda layer --- .github/scripts/build_lambda_layer.py | 155 ++++++++++++ .../scripts/tests/test_build_lambda_layer.py | 109 ++++++++ .github/workflows/lambda-layer-publish.yml | 237 ++++++++++++++++++ .github/workflows/test-parser.yml | 18 +- RELEASING.md | 11 + 5 files changed, 525 insertions(+), 5 deletions(-) create mode 100644 .github/scripts/build_lambda_layer.py create mode 100644 .github/scripts/tests/test_build_lambda_layer.py create mode 100644 .github/workflows/lambda-layer-publish.yml diff --git a/.github/scripts/build_lambda_layer.py b/.github/scripts/build_lambda_layer.py new file mode 100644 index 00000000..d1ff8be7 --- /dev/null +++ b/.github/scripts/build_lambda_layer.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + + +ARCHITECTURE_PLATFORMS = { + "x86_64": "manylinux2014_x86_64", + "arm64": "manylinux2014_aarch64", +} +SUPPORTED_PYTHON_VERSIONS = ("3.11", "3.12", "3.13", "3.14") + + +@dataclass(frozen=True) +class BuildConfig: + output: Path + target_python: str + architecture: str + sdk_distribution: Path + otel_distribution: Path + build_dir: Path | None = None + + +def build_layer(config: BuildConfig) -> Path: + """Build a Lambda layer containing the SDK and OpenTelemetry plugin.""" + + _validate_config(config) + + with tempfile.TemporaryDirectory() as temp_dir: + work_dir = config.build_dir or Path(temp_dir) / "layer" + if work_dir.exists(): + shutil.rmtree(work_dir) + layer_python_dir = work_dir / "python" + layer_python_dir.mkdir(parents=True) + + _install_layer_dependencies(config, layer_python_dir) + _write_zip(config.output, work_dir) + + return config.output + + +def _validate_config(config: BuildConfig) -> None: + if config.architecture not in ARCHITECTURE_PLATFORMS: + supported = ", ".join(sorted(ARCHITECTURE_PLATFORMS)) + raise ValueError( + f"Unsupported architecture: {config.architecture}. " + f"Supported architectures: {supported}" + ) + + if config.target_python not in SUPPORTED_PYTHON_VERSIONS: + supported = ", ".join(SUPPORTED_PYTHON_VERSIONS) + raise ValueError( + f"Unsupported Python version: {config.target_python}. " + f"Supported versions: {supported}" + ) + + for distribution in (config.sdk_distribution, config.otel_distribution): + if not distribution.is_file(): + raise FileNotFoundError(distribution) + + +def _install_layer_dependencies(config: BuildConfig, target_dir: Path) -> None: + python_version = config.target_python + abi = f"cp{python_version.replace('.', '')}" + platform = ARCHITECTURE_PLATFORMS[config.architecture] + + command = [ + sys.executable, + "-m", + "pip", + "install", + "--upgrade", + "--target", + str(target_dir), + "--platform", + platform, + "--implementation", + "cp", + "--python-version", + python_version, + "--abi", + abi, + "--only-binary", + ":all:", + "--no-compile", + str(config.sdk_distribution), + str(config.otel_distribution), + ] + subprocess.run(command, check=True) + + +def _write_zip(output: Path, layer_root: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + output.unlink() + + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path in sorted(layer_root.rglob("*")): + if path.is_file(): + archive.write(path, path.relative_to(layer_root)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="build_lambda_layer.py", + description="Build the AWS Durable Execution SDK OTel plugin Lambda layer.", + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--target-python", + choices=SUPPORTED_PYTHON_VERSIONS, + required=True, + help="Lambda Python minor version.", + ) + parser.add_argument( + "--architecture", + choices=sorted(ARCHITECTURE_PLATFORMS), + required=True, + help="Lambda instruction set architecture.", + ) + parser.add_argument("--sdk-distribution", type=Path, required=True) + parser.add_argument("--otel-distribution", type=Path, required=True) + parser.add_argument( + "--build-dir", + type=Path, + help="Optional scratch directory. Existing contents are replaced.", + ) + + args = parser.parse_args(argv) + output = build_layer( + BuildConfig( + output=args.output, + target_python=args.target_python, + architecture=args.architecture, + sdk_distribution=args.sdk_distribution, + otel_distribution=args.otel_distribution, + build_dir=args.build_dir, + ) + ) + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/tests/test_build_lambda_layer.py b/.github/scripts/tests/test_build_lambda_layer.py new file mode 100644 index 00000000..987c14fb --- /dev/null +++ b/.github/scripts/tests/test_build_lambda_layer.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import zipfile +from pathlib import Path + +import pytest + + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from build_lambda_layer import BuildConfig, build_layer + + +def test_build_layer_installs_dependencies_and_zips_lambda_layout( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + sdk_wheel = tmp_path / "aws_durable_execution_sdk_python-1.0.0-py3-none-any.whl" + otel_wheel = ( + tmp_path / "aws_durable_execution_sdk_python_otel-1.0.0-py3-none-any.whl" + ) + sdk_wheel.write_text("sdk") + otel_wheel.write_text("otel") + commands: list[list[str]] = [] + + def fake_run(command: list[str], check: bool) -> subprocess.CompletedProcess[str]: + commands.append(command) + target = Path(command[command.index("--target") + 1]) + (target / "aws_durable_execution_sdk_python").mkdir() + (target / "aws_durable_execution_sdk_python" / "__init__.py").write_text("") + (target / "aws_durable_execution_sdk_python_otel").mkdir() + (target / "aws_durable_execution_sdk_python_otel" / "__init__.py").write_text( + "" + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(subprocess, "run", fake_run) + + output = build_layer( + BuildConfig( + output=tmp_path / "layer.zip", + target_python="3.12", + architecture="arm64", + sdk_distribution=sdk_wheel, + otel_distribution=otel_wheel, + ) + ) + + assert output == tmp_path / "layer.zip" + assert commands[0][commands[0].index("--platform") + 1] == "manylinux2014_aarch64" + assert commands[0][commands[0].index("--abi") + 1] == "cp312" + assert "--no-compile" in commands[0] + assert str(sdk_wheel) in commands[0] + assert str(otel_wheel) in commands[0] + + with zipfile.ZipFile(output) as archive: + assert ( + "python/aws_durable_execution_sdk_python/__init__.py" in archive.namelist() + ) + assert ( + "python/aws_durable_execution_sdk_python_otel/__init__.py" + in archive.namelist() + ) + + +@pytest.mark.parametrize( + ("target_python", "architecture", "error"), + [ + ("3.10", "x86_64", "Unsupported Python version"), + ("3.12", "sparc", "Unsupported architecture"), + ], +) +def test_build_layer_rejects_unsupported_targets( + target_python: str, + architecture: str, + error: str, + tmp_path: Path, +) -> None: + sdk_wheel = tmp_path / "sdk.whl" + otel_wheel = tmp_path / "otel.whl" + sdk_wheel.write_text("sdk") + otel_wheel.write_text("otel") + + with pytest.raises(ValueError, match=error): + build_layer( + BuildConfig( + output=tmp_path / "layer.zip", + target_python=target_python, + architecture=architecture, + sdk_distribution=sdk_wheel, + otel_distribution=otel_wheel, + ) + ) + + +def test_build_layer_requires_built_distributions(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + build_layer( + BuildConfig( + output=tmp_path / "layer.zip", + target_python="3.13", + architecture="x86_64", + sdk_distribution=tmp_path / "missing-sdk.whl", + otel_distribution=tmp_path / "missing-otel.whl", + ) + ) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml new file mode 100644 index 00000000..8d99d5fc --- /dev/null +++ b/.github/workflows/lambda-layer-publish.yml @@ -0,0 +1,237 @@ +name: Publish OTel Plugin Lambda Layer + +on: + release: + types: [published] + +permissions: + contents: read + +concurrency: + group: lambda-layer-publish-${{ github.event.release.tag_name }} + cancel-in-progress: false + +env: + LAYER_NAME: aws-durable-execution-sdk-python-otel-plugin + DEFAULT_LAYER_REGIONS: >- + af-south-1, + ap-east-1, + ap-east-2, + ap-northeast-1, + ap-northeast-2, + ap-northeast-3, + ap-south-1, + ap-south-2, + ap-southeast-1, + ap-southeast-2, + ap-southeast-3, + ap-southeast-4, + ap-southeast-5, + ap-southeast-6, + ap-southeast-7, + ca-central-1, + ca-west-1, + eu-central-1, + eu-central-2, + eu-north-1, + eu-south-1, + eu-south-2, + eu-west-1, + eu-west-2, + eu-west-3, + il-central-1, + me-central-1, + me-south-1, + mx-central-1, + sa-east-1, + us-east-1, + us-east-2, + us-west-1, + us-west-2 + +jobs: + build-distributions: + if: contains(github.event.release.tag_name, 'otel-v') + runs-on: ubuntu-latest + outputs: + sdk_version: ${{ steps.versions.outputs.sdk_version }} + otel_version: ${{ steps.versions.outputs.otel_version }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.release.tag_name }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Install Hatch + run: python -m pip install --upgrade hatch==1.16.5 + + - name: Build SDK distribution + working-directory: packages/aws-durable-execution-sdk-python + run: hatch build + + - name: Build OTel plugin distribution + working-directory: packages/aws-durable-execution-sdk-python-otel + run: hatch build + + - name: Verify legal files + run: | + python .github/scripts/check_dist_legal_files.py \ + packages/aws-durable-execution-sdk-python \ + packages/aws-durable-execution-sdk-python-otel + + - name: Read package versions + id: versions + run: | + SDK_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/__about__.py | cut -d'"' -f2) + OTEL_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python-otel/src/aws_durable_execution_sdk_python_otel/__about__.py | cut -d'"' -f2) + echo "sdk_version=${SDK_VERSION}" >> "$GITHUB_OUTPUT" + echo "otel_version=${OTEL_VERSION}" >> "$GITHUB_OUTPUT" + + - name: Stage wheels + run: | + mkdir release-dists + cp packages/aws-durable-execution-sdk-python/dist/*.whl release-dists/ + cp packages/aws-durable-execution-sdk-python-otel/dist/*.whl release-dists/ + + - name: Upload wheels + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lambda-layer-distributions + path: release-dists/ + if-no-files-found: error + retention-days: 7 + + publish-layer: + needs: build-distributions + runs-on: ubuntu-latest + environment: + name: lambda-layer-publish + permissions: + contents: read + id-token: write + env: + LAYER_REGIONS: ${{ vars.LAYER_PUBLISH_REGIONS }} + strategy: + fail-fast: false + matrix: + include: + - target_python: "3.11" + runtime: python3.11 + runtime_slug: python311 + architecture: x86_64 + - target_python: "3.11" + runtime: python3.11 + runtime_slug: python311 + architecture: arm64 + - target_python: "3.12" + runtime: python3.12 + runtime_slug: python312 + architecture: x86_64 + - target_python: "3.12" + runtime: python3.12 + runtime_slug: python312 + architecture: arm64 + - target_python: "3.13" + runtime: python3.13 + runtime_slug: python313 + architecture: x86_64 + - target_python: "3.13" + runtime: python3.13 + runtime_slug: python313 + architecture: arm64 + - target_python: "3.14" + runtime: python3.14 + runtime_slug: python314 + architecture: x86_64 + - target_python: "3.14" + runtime: python3.14 + runtime_slug: python314 + architecture: arm64 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.release.tag_name }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Download wheels + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: lambda-layer-distributions + path: release-dists/ + + - name: Build layer zip + id: build-layer + env: + LAYER_ZIP: dist/${{ env.LAYER_NAME }}-${{ matrix.runtime_slug }}-${{ matrix.architecture }}.zip + run: | + SDK_WHEEL=$(find release-dists -name 'aws_durable_execution_sdk_python-*.whl' -print -quit) + OTEL_WHEEL=$(find release-dists -name 'aws_durable_execution_sdk_python_otel-*.whl' -print -quit) + python .github/scripts/build_lambda_layer.py \ + --sdk-distribution "$SDK_WHEEL" \ + --otel-distribution "$OTEL_WHEEL" \ + --target-python "${{ matrix.target_python }}" \ + --architecture "${{ matrix.architecture }}" \ + --output "$LAYER_ZIP" + echo "layer_zip=${LAYER_ZIP}" >> "$GITHUB_OUTPUT" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: ${{ secrets.LAYER_PUBLISH_ROLE_ARN }} + role-session-name: otelLayerPublish-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + aws-region: us-east-1 + + - name: Publish layer versions + env: + LAYER_ZIP: ${{ steps.build-layer.outputs.layer_zip }} + SDK_VERSION: ${{ needs.build-distributions.outputs.sdk_version }} + OTEL_VERSION: ${{ needs.build-distributions.outputs.otel_version }} + run: | + PUBLISHED=false + REGION_LIST=${LAYER_REGIONS:-$DEFAULT_LAYER_REGIONS} + IFS=',' read -ra REGIONS <<< "$REGION_LIST" + for REGION in "${REGIONS[@]}"; do + REGION=$(echo "$REGION" | xargs) + if [ -z "$REGION" ]; then + continue + fi + + LAYER_VERSION_ARN=$(aws lambda publish-layer-version \ + --layer-name "$LAYER_NAME" \ + --description "AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }})" \ + --zip-file "fileb://${LAYER_ZIP}" \ + --compatible-runtimes "${{ matrix.runtime }}" \ + --compatible-architectures "${{ matrix.architecture }}" \ + --license-info Apache-2.0 \ + --region "$REGION" \ + --query LayerVersionArn \ + --output text) + + echo "Published ${LAYER_VERSION_ARN}" + echo "- \`${LAYER_VERSION_ARN}\`" >> "$GITHUB_STEP_SUMMARY" + PUBLISHED=true + done + + if [ "$PUBLISHED" != true ]; then + echo "No AWS regions were configured for layer publishing." + exit 1 + fi + + - name: Upload layer artifact + if: always() && steps.build-layer.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: otel-plugin-layer-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + path: ${{ steps.build-layer.outputs.layer_zip }} + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/test-parser.yml b/.github/workflows/test-parser.yml index 52d8836a..fde37133 100644 --- a/.github/workflows/test-parser.yml +++ b/.github/workflows/test-parser.yml @@ -1,13 +1,15 @@ -name: Test Parser +name: Test GitHub Scripts on: pull_request: paths: + - '.github/scripts/build_lambda_layer.py' - '.github/scripts/parse_sdk_branch.py' - '.github/scripts/tests/**' push: branches: [ main ] paths: + - '.github/scripts/build_lambda_layer.py' - '.github/scripts/parse_sdk_branch.py' - '.github/scripts/tests/**' @@ -15,10 +17,16 @@ permissions: contents: read jobs: - test-parser: + test-scripts: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Run parser tests - run: python .github/scripts/tests/test_parse_sdk_branch.py + + - name: Install test dependencies + run: python -m pip install pytest + + - name: Run script tests + run: | + python -m pytest \ + .github/scripts/tests/test_build_lambda_layer.py \ + .github/scripts/tests/test_parse_sdk_branch.py diff --git a/RELEASING.md b/RELEASING.md index cc9e1faa..5128435e 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -69,6 +69,17 @@ The workflow runs on the `release: [published]` event, so it fires whenever a re > **Note:** The workflow builds and publishes all packages in the matrix. Ensure the version in each package's `__about__.py` is correct before publishing. If only one package has a version bump, PyPI will reject the re-upload of the unchanged package (which is expected and harmless since `fail-fast: false` is set). +Releases containing an `otel-v` tag also trigger the +[`lambda-layer-publish.yml`](.github/workflows/lambda-layer-publish.yml) +workflow. It builds the SDK and OTel plugin into Lambda layers for each +supported Python runtime and architecture, then publishes versions of the +`aws-durable-execution-sdk-python-otel-plugin` layer. + +The publishing job uses the `lambda-layer-publish` GitHub environment and its +`LAYER_PUBLISH_ROLE_ARN` secret. Set the optional `LAYER_PUBLISH_REGIONS` +environment variable to a comma-separated list of AWS Regions. When unset, the +workflow publishes to every commercial AWS Region supported by Lambda. + ## Release Notes Format Release notes should maintain separate timelines for each package. Use the following structure: From 3c5ecdf75499950b0decb1c40a4395728f6caebe Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:35:15 -0700 Subject: [PATCH 02/11] ci: make otel plugin layers public --- .github/workflows/lambda-layer-publish.yml | 14 ++++++++++++-- RELEASING.md | 4 +++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index 8d99d5fc..f6356282 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -206,7 +206,7 @@ jobs: continue fi - LAYER_VERSION_ARN=$(aws lambda publish-layer-version \ + PUBLISH_RESULT=$(aws lambda publish-layer-version \ --layer-name "$LAYER_NAME" \ --description "AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }})" \ --zip-file "fileb://${LAYER_ZIP}" \ @@ -214,8 +214,18 @@ jobs: --compatible-architectures "${{ matrix.architecture }}" \ --license-info Apache-2.0 \ --region "$REGION" \ - --query LayerVersionArn \ + --query '[LayerVersionArn,Version]' \ --output text) + read -r LAYER_VERSION_ARN VERSION_NUMBER <<< "$PUBLISH_RESULT" + + aws lambda add-layer-version-permission \ + --layer-name "$LAYER_NAME" \ + --version-number "$VERSION_NUMBER" \ + --statement-id public-layer-access \ + --action lambda:GetLayerVersion \ + --principal "*" \ + --region "$REGION" \ + > /dev/null echo "Published ${LAYER_VERSION_ARN}" echo "- \`${LAYER_VERSION_ARN}\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/RELEASING.md b/RELEASING.md index 5128435e..5511c48a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -72,13 +72,15 @@ The workflow runs on the `release: [published]` event, so it fires whenever a re Releases containing an `otel-v` tag also trigger the [`lambda-layer-publish.yml`](.github/workflows/lambda-layer-publish.yml) workflow. It builds the SDK and OTel plugin into Lambda layers for each -supported Python runtime and architecture, then publishes versions of the +supported Python runtime and architecture, then publishes public versions of the `aws-durable-execution-sdk-python-otel-plugin` layer. The publishing job uses the `lambda-layer-publish` GitHub environment and its `LAYER_PUBLISH_ROLE_ARN` secret. Set the optional `LAYER_PUBLISH_REGIONS` environment variable to a comma-separated list of AWS Regions. When unset, the workflow publishes to every commercial AWS Region supported by Lambda. +The publishing role must allow `lambda:PublishLayerVersion` and +`lambda:AddLayerVersionPermission`. ## Release Notes Format From f5469383b1ef7f43a01a3fe000e5e894daec02f0 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:40:56 -0700 Subject: [PATCH 03/11] ci: continue layer publishing across regions --- .github/workflows/lambda-layer-publish.yml | 50 ++++++++++++++-------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index f6356282..1f26dc5f 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -198,6 +198,7 @@ jobs: OTEL_VERSION: ${{ needs.build-distributions.outputs.otel_version }} run: | PUBLISHED=false + FAILED_REGIONS=() REGION_LIST=${LAYER_REGIONS:-$DEFAULT_LAYER_REGIONS} IFS=',' read -ra REGIONS <<< "$REGION_LIST" for REGION in "${REGIONS[@]}"; do @@ -206,32 +207,45 @@ jobs: continue fi - PUBLISH_RESULT=$(aws lambda publish-layer-version \ - --layer-name "$LAYER_NAME" \ - --description "AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }})" \ - --zip-file "fileb://${LAYER_ZIP}" \ - --compatible-runtimes "${{ matrix.runtime }}" \ - --compatible-architectures "${{ matrix.architecture }}" \ - --license-info Apache-2.0 \ - --region "$REGION" \ - --query '[LayerVersionArn,Version]' \ - --output text) + if ! PUBLISH_RESULT=$(aws lambda publish-layer-version \ + --layer-name "$LAYER_NAME" \ + --description "AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }})" \ + --zip-file "fileb://${LAYER_ZIP}" \ + --compatible-runtimes "${{ matrix.runtime }}" \ + --compatible-architectures "${{ matrix.architecture }}" \ + --license-info Apache-2.0 \ + --region "$REGION" \ + --query '[LayerVersionArn,Version]' \ + --output text); then + echo "::warning::Failed to publish the layer in ${REGION}" + FAILED_REGIONS+=("$REGION") + continue + fi read -r LAYER_VERSION_ARN VERSION_NUMBER <<< "$PUBLISH_RESULT" - aws lambda add-layer-version-permission \ - --layer-name "$LAYER_NAME" \ - --version-number "$VERSION_NUMBER" \ - --statement-id public-layer-access \ - --action lambda:GetLayerVersion \ - --principal "*" \ - --region "$REGION" \ - > /dev/null + if ! aws lambda add-layer-version-permission \ + --layer-name "$LAYER_NAME" \ + --version-number "$VERSION_NUMBER" \ + --statement-id public-layer-access \ + --action lambda:GetLayerVersion \ + --principal "*" \ + --region "$REGION" \ + > /dev/null; then + echo "::warning::Published ${LAYER_VERSION_ARN}, but failed to grant public access in ${REGION}" + FAILED_REGIONS+=("$REGION") + continue + fi echo "Published ${LAYER_VERSION_ARN}" echo "- \`${LAYER_VERSION_ARN}\`" >> "$GITHUB_STEP_SUMMARY" PUBLISHED=true done + if [ "${#FAILED_REGIONS[@]}" -gt 0 ]; then + echo "::error::Layer publishing failed in: ${FAILED_REGIONS[*]}" + exit 1 + fi + if [ "$PUBLISHED" != true ]; then echo "No AWS regions were configured for layer publishing." exit 1 From 98368102c7c4d6e0b95e86d8f8f3c25bf3316e46 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:56:17 -0700 Subject: [PATCH 04/11] ci: make layer publishing idempotent --- .github/workflows/lambda-layer-publish.yml | 53 +++++++++++++++------- RELEASING.md | 3 +- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index 1f26dc5f..de007c9d 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -161,7 +161,7 @@ jobs: - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: "3.11" + python-version: ${{ matrix.target_python }} - name: Download wheels uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -207,36 +207,57 @@ jobs: continue fi - if ! PUBLISH_RESULT=$(aws lambda publish-layer-version \ + LAYER_DESCRIPTION="AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }})" + if ! EXISTING_RESULT=$(aws lambda list-layer-versions \ --layer-name "$LAYER_NAME" \ - --description "AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }})" \ - --zip-file "fileb://${LAYER_ZIP}" \ - --compatible-runtimes "${{ matrix.runtime }}" \ - --compatible-architectures "${{ matrix.architecture }}" \ - --license-info Apache-2.0 \ + --compatible-runtime "${{ matrix.runtime }}" \ + --compatible-architecture "${{ matrix.architecture }}" \ --region "$REGION" \ - --query '[LayerVersionArn,Version]' \ + --query "LayerVersions[?Description=='${LAYER_DESCRIPTION}'] | [0].[LayerVersionArn,Version]" \ --output text); then - echo "::warning::Failed to publish the layer in ${REGION}" + echo "::warning::Failed to list existing layer versions in ${REGION}" FAILED_REGIONS+=("$REGION") continue fi - read -r LAYER_VERSION_ARN VERSION_NUMBER <<< "$PUBLISH_RESULT" - if ! aws lambda add-layer-version-permission \ + if [ -n "$EXISTING_RESULT" ] && [ "$EXISTING_RESULT" != "None" ]; then + read -r LAYER_VERSION_ARN VERSION_NUMBER <<< "$EXISTING_RESULT" + echo "Reusing ${LAYER_VERSION_ARN}" + else + if ! PUBLISH_RESULT=$(aws lambda publish-layer-version \ + --layer-name "$LAYER_NAME" \ + --description "$LAYER_DESCRIPTION" \ + --zip-file "fileb://${LAYER_ZIP}" \ + --compatible-runtimes "${{ matrix.runtime }}" \ + --compatible-architectures "${{ matrix.architecture }}" \ + --license-info Apache-2.0 \ + --region "$REGION" \ + --query '[LayerVersionArn,Version]' \ + --output text); then + echo "::warning::Failed to publish the layer in ${REGION}" + FAILED_REGIONS+=("$REGION") + continue + fi + read -r LAYER_VERSION_ARN VERSION_NUMBER <<< "$PUBLISH_RESULT" + fi + + if ! PERMISSION_RESULT=$(aws lambda add-layer-version-permission \ --layer-name "$LAYER_NAME" \ --version-number "$VERSION_NUMBER" \ --statement-id public-layer-access \ --action lambda:GetLayerVersion \ --principal "*" \ --region "$REGION" \ - > /dev/null; then - echo "::warning::Published ${LAYER_VERSION_ARN}, but failed to grant public access in ${REGION}" - FAILED_REGIONS+=("$REGION") - continue + 2>&1); then + if [[ "$PERMISSION_RESULT" != *"ResourceConflictException"* ]]; then + echo "$PERMISSION_RESULT" >&2 + echo "::warning::Failed to grant public access to ${LAYER_VERSION_ARN}" + FAILED_REGIONS+=("$REGION") + continue + fi fi - echo "Published ${LAYER_VERSION_ARN}" + echo "Available ${LAYER_VERSION_ARN}" echo "- \`${LAYER_VERSION_ARN}\`" >> "$GITHUB_STEP_SUMMARY" PUBLISHED=true done diff --git a/RELEASING.md b/RELEASING.md index 5511c48a..dcee9bdf 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -80,7 +80,8 @@ The publishing job uses the `lambda-layer-publish` GitHub environment and its environment variable to a comma-separated list of AWS Regions. When unset, the workflow publishes to every commercial AWS Region supported by Lambda. The publishing role must allow `lambda:PublishLayerVersion` and -`lambda:AddLayerVersionPermission`. +`lambda:AddLayerVersionPermission`, as well as `lambda:ListLayerVersions` for +idempotent release retries. ## Release Notes Format From c9e0ab0ad8ba75cf3d71e9e1fdbef698ecdf7981 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:20:08 -0700 Subject: [PATCH 05/11] ci: allow manual layer publishing --- .github/workflows/lambda-layer-publish.yml | 18 +++++++++++++----- RELEASING.md | 2 ++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index de007c9d..9619ea55 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -3,12 +3,18 @@ name: Publish OTel Plugin Lambda Layer on: release: types: [published] + workflow_dispatch: + inputs: + regions: + description: "Comma-separated AWS Regions; defaults to all commercial Regions" + required: false + type: string permissions: contents: read concurrency: - group: lambda-layer-publish-${{ github.event.release.tag_name }} + group: lambda-layer-publish-${{ github.event.release.tag_name || github.ref_name }} cancel-in-progress: false env: @@ -51,7 +57,9 @@ env: jobs: build-distributions: - if: contains(github.event.release.tag_name, 'otel-v') + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.release.tag_name, 'otel-v') runs-on: ubuntu-latest outputs: sdk_version: ${{ steps.versions.outputs.sdk_version }} @@ -60,7 +68,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.release.tag_name }} + ref: ${{ github.event.release.tag_name || github.ref }} - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -115,7 +123,7 @@ jobs: contents: read id-token: write env: - LAYER_REGIONS: ${{ vars.LAYER_PUBLISH_REGIONS }} + LAYER_REGIONS: ${{ inputs.regions || vars.LAYER_PUBLISH_REGIONS }} strategy: fail-fast: false matrix: @@ -156,7 +164,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.release.tag_name }} + ref: ${{ github.event.release.tag_name || github.ref }} - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 diff --git a/RELEASING.md b/RELEASING.md index dcee9bdf..7055910d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -79,6 +79,8 @@ The publishing job uses the `lambda-layer-publish` GitHub environment and its `LAYER_PUBLISH_ROLE_ARN` secret. Set the optional `LAYER_PUBLISH_REGIONS` environment variable to a comma-separated list of AWS Regions. When unset, the workflow publishes to every commercial AWS Region supported by Lambda. +The workflow can also be run manually from the Actions tab on `main`; its +optional `regions` input overrides `LAYER_PUBLISH_REGIONS` for that run. The publishing role must allow `lambda:PublishLayerVersion` and `lambda:AddLayerVersionPermission`, as well as `lambda:ListLayerVersions` for idempotent release retries. From a558296d51b03ccc28f69f1933c6b42615719fe0 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:29:39 -0700 Subject: [PATCH 06/11] ci: reuse built layer artifacts --- .github/workflows/lambda-layer-publish.yml | 82 +++++++++++++++++----- RELEASING.md | 2 + 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index 9619ea55..4c478756 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -114,16 +114,9 @@ jobs: if-no-files-found: error retention-days: 7 - publish-layer: + build-layers: needs: build-distributions runs-on: ubuntu-latest - environment: - name: lambda-layer-publish - permissions: - contents: read - id-token: write - env: - LAYER_REGIONS: ${{ inputs.regions || vars.LAYER_PUBLISH_REGIONS }} strategy: fail-fast: false matrix: @@ -192,6 +185,68 @@ jobs: --output "$LAYER_ZIP" echo "layer_zip=${LAYER_ZIP}" >> "$GITHUB_OUTPUT" + - name: Upload layer artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: otel-plugin-layer-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + path: ${{ steps.build-layer.outputs.layer_zip }} + if-no-files-found: error + retention-days: 30 + + publish-layer: + needs: [build-distributions, build-layers] + runs-on: ubuntu-latest + environment: + name: lambda-layer-publish + permissions: + contents: read + id-token: write + env: + LAYER_REGIONS: ${{ inputs.regions || vars.LAYER_PUBLISH_REGIONS }} + strategy: + fail-fast: false + matrix: + include: + - target_python: "3.11" + runtime: python3.11 + runtime_slug: python311 + architecture: x86_64 + - target_python: "3.11" + runtime: python3.11 + runtime_slug: python311 + architecture: arm64 + - target_python: "3.12" + runtime: python3.12 + runtime_slug: python312 + architecture: x86_64 + - target_python: "3.12" + runtime: python3.12 + runtime_slug: python312 + architecture: arm64 + - target_python: "3.13" + runtime: python3.13 + runtime_slug: python313 + architecture: x86_64 + - target_python: "3.13" + runtime: python3.13 + runtime_slug: python313 + architecture: arm64 + - target_python: "3.14" + runtime: python3.14 + runtime_slug: python314 + architecture: x86_64 + - target_python: "3.14" + runtime: python3.14 + runtime_slug: python314 + architecture: arm64 + + steps: + - name: Download layer artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: otel-plugin-layer-${{ matrix.runtime_slug }}-${{ matrix.architecture }} + path: dist/ + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 with: @@ -201,7 +256,7 @@ jobs: - name: Publish layer versions env: - LAYER_ZIP: ${{ steps.build-layer.outputs.layer_zip }} + LAYER_ZIP: dist/${{ env.LAYER_NAME }}-${{ matrix.runtime_slug }}-${{ matrix.architecture }}.zip SDK_VERSION: ${{ needs.build-distributions.outputs.sdk_version }} OTEL_VERSION: ${{ needs.build-distributions.outputs.otel_version }} run: | @@ -279,12 +334,3 @@ jobs: echo "No AWS regions were configured for layer publishing." exit 1 fi - - - name: Upload layer artifact - if: always() && steps.build-layer.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: otel-plugin-layer-${{ matrix.runtime_slug }}-${{ matrix.architecture }} - path: ${{ steps.build-layer.outputs.layer_zip }} - if-no-files-found: error - retention-days: 30 diff --git a/RELEASING.md b/RELEASING.md index 7055910d..3fadf217 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -81,6 +81,8 @@ environment variable to a comma-separated list of AWS Regions. When unset, the workflow publishes to every commercial AWS Region supported by Lambda. The workflow can also be run manually from the Actions tab on `main`; its optional `regions` input overrides `LAYER_PUBLISH_REGIONS` for that run. +Each runtime and architecture layer archive is built once and retained as a +workflow artifact so retries publish the exact same resolved dependencies. The publishing role must allow `lambda:PublishLayerVersion` and `lambda:AddLayerVersionPermission`, as well as `lambda:ListLayerVersions` for idempotent release retries. From 8837616f4a29bfb487ba5e97eb713699a99b5562 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:37:25 -0700 Subject: [PATCH 07/11] fix: reject layer output inside build directory --- .github/scripts/build_lambda_layer.py | 6 ++++++ .../scripts/tests/test_build_lambda_layer.py | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/.github/scripts/build_lambda_layer.py b/.github/scripts/build_lambda_layer.py index d1ff8be7..ff903405 100644 --- a/.github/scripts/build_lambda_layer.py +++ b/.github/scripts/build_lambda_layer.py @@ -38,6 +38,7 @@ def build_layer(config: BuildConfig) -> Path: with tempfile.TemporaryDirectory() as temp_dir: work_dir = config.build_dir or Path(temp_dir) / "layer" + _validate_output_location(config.output, work_dir) if work_dir.exists(): shutil.rmtree(work_dir) layer_python_dir = work_dir / "python" @@ -69,6 +70,11 @@ def _validate_config(config: BuildConfig) -> None: raise FileNotFoundError(distribution) +def _validate_output_location(output: Path, build_dir: Path) -> None: + if output.resolve().is_relative_to(build_dir.resolve()): + raise ValueError("Layer output must be outside the build directory") + + def _install_layer_dependencies(config: BuildConfig, target_dir: Path) -> None: python_version = config.target_python abi = f"cp{python_version.replace('.', '')}" diff --git a/.github/scripts/tests/test_build_lambda_layer.py b/.github/scripts/tests/test_build_lambda_layer.py index 987c14fb..7671b48a 100644 --- a/.github/scripts/tests/test_build_lambda_layer.py +++ b/.github/scripts/tests/test_build_lambda_layer.py @@ -107,3 +107,23 @@ def test_build_layer_requires_built_distributions(tmp_path: Path) -> None: otel_distribution=tmp_path / "missing-otel.whl", ) ) + + +def test_build_layer_rejects_output_inside_build_directory(tmp_path: Path) -> None: + sdk_wheel = tmp_path / "sdk.whl" + otel_wheel = tmp_path / "otel.whl" + sdk_wheel.write_text("sdk") + otel_wheel.write_text("otel") + build_dir = tmp_path / "layer" + + with pytest.raises(ValueError, match="outside the build directory"): + build_layer( + BuildConfig( + output=build_dir / "layer.zip", + target_python="3.13", + architecture="x86_64", + sdk_distribution=sdk_wheel, + otel_distribution=otel_wheel, + build_dir=build_dir, + ) + ) From 8a1e2e5b36ec287bbc26d3fa07abfbed16b1fffa Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:00:21 -0700 Subject: [PATCH 08/11] ci: harden layer release workflow --- .github/workflows/lambda-layer-publish.yml | 54 ++++++++++++++++------ RELEASING.md | 3 ++ 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index 4c478756..fd978bed 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -19,6 +19,7 @@ concurrency: env: LAYER_NAME: aws-durable-execution-sdk-python-otel-plugin + SOURCE_REF: ${{ github.event_name == 'workflow_dispatch' && 'main' || github.event.release.tag_name }} DEFAULT_LAYER_REGIONS: >- af-south-1, ap-east-1, @@ -58,7 +59,8 @@ env: jobs: build-distributions: if: >- - github.event_name == 'workflow_dispatch' || + (github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/main') || contains(github.event.release.tag_name, 'otel-v') runs-on: ubuntu-latest outputs: @@ -68,7 +70,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.release.tag_name || github.ref }} + ref: ${{ env.SOURCE_REF }} - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -78,10 +80,35 @@ jobs: - name: Install Hatch run: python -m pip install --upgrade hatch==1.16.5 + - name: Read package versions + id: versions + run: | + SDK_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/__about__.py | cut -d'"' -f2) + OTEL_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python-otel/src/aws_durable_execution_sdk_python_otel/__about__.py | cut -d'"' -f2) + echo "sdk_version=${SDK_VERSION}" >> "$GITHUB_OUTPUT" + echo "otel_version=${OTEL_VERSION}" >> "$GITHUB_OUTPUT" + - name: Build SDK distribution + if: >- + github.event_name == 'workflow_dispatch' || + contains(github.event.release.tag_name, 'sdk-v') working-directory: packages/aws-durable-execution-sdk-python run: hatch build + - name: Download published SDK distribution + if: >- + github.event_name == 'release' && + !contains(github.event.release.tag_name, 'sdk-v') + env: + SDK_VERSION: ${{ steps.versions.outputs.sdk_version }} + working-directory: packages/aws-durable-execution-sdk-python + run: | + python -m pip download \ + --dest dist \ + --no-deps \ + --only-binary=:all: \ + "aws-durable-execution-sdk-python==${SDK_VERSION}" + - name: Build OTel plugin distribution working-directory: packages/aws-durable-execution-sdk-python-otel run: hatch build @@ -92,14 +119,6 @@ jobs: packages/aws-durable-execution-sdk-python \ packages/aws-durable-execution-sdk-python-otel - - name: Read package versions - id: versions - run: | - SDK_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/__about__.py | cut -d'"' -f2) - OTEL_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python-otel/src/aws_durable_execution_sdk_python_otel/__about__.py | cut -d'"' -f2) - echo "sdk_version=${SDK_VERSION}" >> "$GITHUB_OUTPUT" - echo "otel_version=${OTEL_VERSION}" >> "$GITHUB_OUTPUT" - - name: Stage wheels run: | mkdir release-dists @@ -157,7 +176,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.release.tag_name || github.ref }} + ref: ${{ env.SOURCE_REF }} - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -277,10 +296,15 @@ jobs: --compatible-architecture "${{ matrix.architecture }}" \ --region "$REGION" \ --query "LayerVersions[?Description=='${LAYER_DESCRIPTION}'] | [0].[LayerVersionArn,Version]" \ - --output text); then - echo "::warning::Failed to list existing layer versions in ${REGION}" - FAILED_REGIONS+=("$REGION") - continue + --output text 2>&1); then + if [[ "$EXISTING_RESULT" == *"ResourceNotFoundException"* ]]; then + EXISTING_RESULT="" + else + echo "$EXISTING_RESULT" >&2 + echo "::warning::Failed to list existing layer versions in ${REGION}" + FAILED_REGIONS+=("$REGION") + continue + fi fi if [ -n "$EXISTING_RESULT" ] && [ "$EXISTING_RESULT" != "None" ]; then diff --git a/RELEASING.md b/RELEASING.md index 3fadf217..f9bca15e 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -74,6 +74,9 @@ Releases containing an `otel-v` tag also trigger the workflow. It builds the SDK and OTel plugin into Lambda layers for each supported Python runtime and architecture, then publishes public versions of the `aws-durable-execution-sdk-python-otel-plugin` layer. +For OTel-only releases, the workflow downloads the exact SDK version already +published to PyPI. Combined SDK and OTel releases build both distributions from +the tagged source. The publishing job uses the `lambda-layer-publish` GitHub environment and its `LAYER_PUBLISH_ROLE_ARN` secret. Set the optional `LAYER_PUBLISH_REGIONS` From 15b88cc44a19e6808372144a6df4e4efe0504b57 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:38:53 -0700 Subject: [PATCH 09/11] ci: verify lambda layer artifact hashes --- .github/workflows/lambda-layer-publish.yml | 28 +++++++++++++++++++--- RELEASING.md | 7 +++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index fd978bed..dc07cdb1 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -281,6 +281,7 @@ jobs: run: | PUBLISHED=false FAILED_REGIONS=() + LOCAL_CODE_SHA256=$(openssl dgst -sha256 -binary "$LAYER_ZIP" | openssl base64 -A) REGION_LIST=${LAYER_REGIONS:-$DEFAULT_LAYER_REGIONS} IFS=',' read -ra REGIONS <<< "$REGION_LIST" for REGION in "${REGIONS[@]}"; do @@ -289,7 +290,7 @@ jobs: continue fi - LAYER_DESCRIPTION="AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }})" + LAYER_DESCRIPTION="AWS Durable Execution SDK ${SDK_VERSION} OTel plugin ${OTEL_VERSION} (${{ matrix.runtime }}/${{ matrix.architecture }}) sha256:${LOCAL_CODE_SHA256}" if ! EXISTING_RESULT=$(aws lambda list-layer-versions \ --layer-name "$LAYER_NAME" \ --compatible-runtime "${{ matrix.runtime }}" \ @@ -309,6 +310,22 @@ jobs: if [ -n "$EXISTING_RESULT" ] && [ "$EXISTING_RESULT" != "None" ]; then read -r LAYER_VERSION_ARN VERSION_NUMBER <<< "$EXISTING_RESULT" + if ! EXISTING_CODE_SHA256=$(aws lambda get-layer-version \ + --layer-name "$LAYER_NAME" \ + --version-number "$VERSION_NUMBER" \ + --region "$REGION" \ + --query 'Content.CodeSha256' \ + --output text 2>&1); then + echo "$EXISTING_CODE_SHA256" >&2 + echo "::warning::Failed to read ${LAYER_VERSION_ARN}" + FAILED_REGIONS+=("$REGION") + continue + fi + if [ "$EXISTING_CODE_SHA256" != "$LOCAL_CODE_SHA256" ]; then + echo "::error::Artifact hash mismatch for ${LAYER_VERSION_ARN}" + FAILED_REGIONS+=("$REGION") + continue + fi echo "Reusing ${LAYER_VERSION_ARN}" else if ! PUBLISH_RESULT=$(aws lambda publish-layer-version \ @@ -319,13 +336,18 @@ jobs: --compatible-architectures "${{ matrix.architecture }}" \ --license-info Apache-2.0 \ --region "$REGION" \ - --query '[LayerVersionArn,Version]' \ + --query '[LayerVersionArn,Version,Content.CodeSha256]' \ --output text); then echo "::warning::Failed to publish the layer in ${REGION}" FAILED_REGIONS+=("$REGION") continue fi - read -r LAYER_VERSION_ARN VERSION_NUMBER <<< "$PUBLISH_RESULT" + read -r LAYER_VERSION_ARN VERSION_NUMBER PUBLISHED_CODE_SHA256 <<< "$PUBLISH_RESULT" + if [ "$PUBLISHED_CODE_SHA256" != "$LOCAL_CODE_SHA256" ]; then + echo "::error::Published artifact hash mismatch for ${LAYER_VERSION_ARN}" + FAILED_REGIONS+=("$REGION") + continue + fi fi if ! PERMISSION_RESULT=$(aws lambda add-layer-version-permission \ diff --git a/RELEASING.md b/RELEASING.md index f9bca15e..756fb59b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -85,10 +85,11 @@ workflow publishes to every commercial AWS Region supported by Lambda. The workflow can also be run manually from the Actions tab on `main`; its optional `regions` input overrides `LAYER_PUBLISH_REGIONS` for that run. Each runtime and architecture layer archive is built once and retained as a -workflow artifact so retries publish the exact same resolved dependencies. +workflow artifact so retries publish the exact same resolved dependencies. Its +SHA-256 is included in the layer description and verified before reuse. The publishing role must allow `lambda:PublishLayerVersion` and -`lambda:AddLayerVersionPermission`, as well as `lambda:ListLayerVersions` for -idempotent release retries. +`lambda:AddLayerVersionPermission`, as well as `lambda:ListLayerVersions` and +`lambda:GetLayerVersion` for identity-checked, idempotent release retries. ## Release Notes Format From fd00b8917b669f623ab3b3204d526ea9c41affb6 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:42:23 -0700 Subject: [PATCH 10/11] ci: pin sdk version for otel-only layers --- .github/scripts/resolve_layer_sdk_version.py | 75 +++++++++++++++++++ .../tests/test_resolve_layer_sdk_version.py | 72 ++++++++++++++++++ .github/workflows/lambda-layer-publish.yml | 9 ++- RELEASING.md | 8 +- .../pyproject.toml | 3 + 5 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/resolve_layer_sdk_version.py create mode 100644 .github/scripts/tests/test_resolve_layer_sdk_version.py diff --git a/.github/scripts/resolve_layer_sdk_version.py b/.github/scripts/resolve_layer_sdk_version.py new file mode 100644 index 00000000..5ad2809d --- /dev/null +++ b/.github/scripts/resolve_layer_sdk_version.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import argparse +import tomllib +from pathlib import Path + + +def _read_pinned_sdk_version(otel_pyproject: Path) -> str: + with otel_pyproject.open("rb") as file: + metadata = tomllib.load(file) + + try: + pinned_version = metadata["tool"]["lambda-layer"]["sdk-version"] + except KeyError as error: + raise ValueError( + f"{otel_pyproject}: missing tool.lambda-layer.sdk-version" + ) from error + + if not isinstance(pinned_version, str) or not pinned_version: + raise ValueError( + f"{otel_pyproject}: tool.lambda-layer.sdk-version must be a string" + ) + return pinned_version + + +def resolve_layer_sdk_version( + event_name: str, + release_tag: str, + source_sdk_version: str, + otel_pyproject: Path, +) -> str: + """Resolve the SDK version bundled in the OTel plugin Lambda layer.""" + + pinned_version = _read_pinned_sdk_version(otel_pyproject) + sdk_release = any(part.startswith("sdk-v") for part in release_tag.split(",")) + + if event_name == "release" and not sdk_release: + return pinned_version + + if event_name == "release" and pinned_version != source_sdk_version: + raise ValueError( + "New SDK releases must update tool.lambda-layer.sdk-version " + f"to {source_sdk_version}" + ) + + return source_sdk_version + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Resolve the SDK version for the OTel plugin Lambda layer." + ) + parser.add_argument("--event-name", required=True) + parser.add_argument("--release-tag", default="") + parser.add_argument("--source-sdk-version", required=True) + parser.add_argument("--otel-pyproject", type=Path, required=True) + args = parser.parse_args(argv) + + print( + resolve_layer_sdk_version( + event_name=args.event_name, + release_tag=args.release_tag, + source_sdk_version=args.source_sdk_version, + otel_pyproject=args.otel_pyproject, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/tests/test_resolve_layer_sdk_version.py b/.github/scripts/tests/test_resolve_layer_sdk_version.py new file mode 100644 index 00000000..05b44da1 --- /dev/null +++ b/.github/scripts/tests/test_resolve_layer_sdk_version.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest + + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from resolve_layer_sdk_version import resolve_layer_sdk_version + + +def _write_otel_pyproject(tmp_path: Path, sdk_version: str) -> Path: + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text(f'[tool.lambda-layer]\nsdk-version = "{sdk_version}"\n') + return pyproject + + +def test_otel_only_release_uses_pinned_published_sdk(tmp_path: Path) -> None: + pyproject = _write_otel_pyproject(tmp_path, "1.8.0") + + assert ( + resolve_layer_sdk_version( + event_name="release", + release_tag="otel-v0.4.1", + source_sdk_version="1.9.0", + otel_pyproject=pyproject, + ) + == "1.8.0" + ) + + +def test_combined_release_uses_source_sdk(tmp_path: Path) -> None: + pyproject = _write_otel_pyproject(tmp_path, "1.9.0") + + assert ( + resolve_layer_sdk_version( + event_name="release", + release_tag="sdk-v1.9.0,otel-v0.5.0", + source_sdk_version="1.9.0", + otel_pyproject=pyproject, + ) + == "1.9.0" + ) + + +def test_combined_release_requires_updated_pin(tmp_path: Path) -> None: + pyproject = _write_otel_pyproject(tmp_path, "1.8.0") + + with pytest.raises(ValueError, match="New SDK releases must update"): + resolve_layer_sdk_version( + event_name="release", + release_tag="sdk-v1.9.0,otel-v0.5.0", + source_sdk_version="1.9.0", + otel_pyproject=pyproject, + ) + + +def test_manual_run_uses_source_sdk(tmp_path: Path) -> None: + pyproject = _write_otel_pyproject(tmp_path, "1.8.0") + + assert ( + resolve_layer_sdk_version( + event_name="workflow_dispatch", + release_tag="", + source_sdk_version="1.9.0", + otel_pyproject=pyproject, + ) + == "1.9.0" + ) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index dc07cdb1..c273e59f 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -82,9 +82,16 @@ jobs: - name: Read package versions id: versions + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} run: | - SDK_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/__about__.py | cut -d'"' -f2) + SOURCE_SDK_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/__about__.py | cut -d'"' -f2) OTEL_VERSION=$(grep "^__version__" packages/aws-durable-execution-sdk-python-otel/src/aws_durable_execution_sdk_python_otel/__about__.py | cut -d'"' -f2) + SDK_VERSION=$(python .github/scripts/resolve_layer_sdk_version.py \ + --event-name "$GITHUB_EVENT_NAME" \ + --release-tag "$RELEASE_TAG" \ + --source-sdk-version "$SOURCE_SDK_VERSION" \ + --otel-pyproject packages/aws-durable-execution-sdk-python-otel/pyproject.toml) echo "sdk_version=${SDK_VERSION}" >> "$GITHUB_OUTPUT" echo "otel_version=${OTEL_VERSION}" >> "$GITHUB_OUTPUT" diff --git a/RELEASING.md b/RELEASING.md index 756fb59b..4de60d04 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -74,8 +74,10 @@ Releases containing an `otel-v` tag also trigger the workflow. It builds the SDK and OTel plugin into Lambda layers for each supported Python runtime and architecture, then publishes public versions of the `aws-durable-execution-sdk-python-otel-plugin` layer. -For OTel-only releases, the workflow downloads the exact SDK version already -published to PyPI. Combined SDK and OTel releases build both distributions from +For OTel-only releases, the workflow downloads the exact SDK version pinned by +`tool.lambda-layer.sdk-version` in the OTel package's `pyproject.toml`; that +version must already be published to PyPI. Combined SDK and OTel releases +require the pin to match the new SDK version and build both distributions from the tagged source. The publishing job uses the `lambda-layer-publish` GitHub environment and its @@ -133,6 +135,8 @@ If only one package is being released, include only that package's section. Each Before publishing a release: - [ ] Version bumped in the relevant `__about__.py` file(s) +- [ ] OTel layer SDK pin identifies a compatible published SDK, or matches the + SDK version included in a combined release - [ ] Changes merged to `main` - [ ] CI checks pass on `main` - [ ] Release notes written with separate sections per package diff --git a/packages/aws-durable-execution-sdk-python-otel/pyproject.toml b/packages/aws-durable-execution-sdk-python-otel/pyproject.toml index 9f9d4ff4..904851af 100644 --- a/packages/aws-durable-execution-sdk-python-otel/pyproject.toml +++ b/packages/aws-durable-execution-sdk-python-otel/pyproject.toml @@ -33,6 +33,9 @@ dependencies = [ otel-invocation = "aws_durable_execution_sdk_python_otel.plugin_provider:INVOCATION_OTEL_PLUGIN_PROVIDER" otel-execution = "aws_durable_execution_sdk_python_otel.plugin_provider:EXECUTION_OTEL_PLUGIN_PROVIDER" +[tool.lambda-layer] +sdk-version = "1.8.0" + [project.optional-dependencies] # Instrumentation used by ExecutionOtelPlugin's auto-configured provider path. # Kept optional so the InvocationOtelPlugin (ADOT / global provider) install From 51e2a0f58e64a1af49a59b584b54fecadb5bc382 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:59:34 -0700 Subject: [PATCH 11/11] ci: move layer sdk pin to release metadata --- .github/lambda-layer-publish.toml | 2 ++ .github/scripts/resolve_layer_sdk_version.py | 25 ++++++++----------- .../tests/test_resolve_layer_sdk_version.py | 24 +++++++++--------- .github/workflows/lambda-layer-publish.yml | 2 +- RELEASING.md | 7 +++--- .../pyproject.toml | 3 --- 6 files changed, 28 insertions(+), 35 deletions(-) create mode 100644 .github/lambda-layer-publish.toml diff --git a/.github/lambda-layer-publish.toml b/.github/lambda-layer-publish.toml new file mode 100644 index 00000000..279a10b8 --- /dev/null +++ b/.github/lambda-layer-publish.toml @@ -0,0 +1,2 @@ +[layer] +sdk-version = "1.8.0" diff --git a/.github/scripts/resolve_layer_sdk_version.py b/.github/scripts/resolve_layer_sdk_version.py index 5ad2809d..d96b53fd 100644 --- a/.github/scripts/resolve_layer_sdk_version.py +++ b/.github/scripts/resolve_layer_sdk_version.py @@ -9,21 +9,17 @@ from pathlib import Path -def _read_pinned_sdk_version(otel_pyproject: Path) -> str: - with otel_pyproject.open("rb") as file: +def _read_pinned_sdk_version(metadata_path: Path) -> str: + with metadata_path.open("rb") as file: metadata = tomllib.load(file) try: - pinned_version = metadata["tool"]["lambda-layer"]["sdk-version"] + pinned_version = metadata["layer"]["sdk-version"] except KeyError as error: - raise ValueError( - f"{otel_pyproject}: missing tool.lambda-layer.sdk-version" - ) from error + raise ValueError(f"{metadata_path}: missing layer.sdk-version") from error if not isinstance(pinned_version, str) or not pinned_version: - raise ValueError( - f"{otel_pyproject}: tool.lambda-layer.sdk-version must be a string" - ) + raise ValueError(f"{metadata_path}: layer.sdk-version must be a string") return pinned_version @@ -31,11 +27,11 @@ def resolve_layer_sdk_version( event_name: str, release_tag: str, source_sdk_version: str, - otel_pyproject: Path, + metadata_path: Path, ) -> str: """Resolve the SDK version bundled in the OTel plugin Lambda layer.""" - pinned_version = _read_pinned_sdk_version(otel_pyproject) + pinned_version = _read_pinned_sdk_version(metadata_path) sdk_release = any(part.startswith("sdk-v") for part in release_tag.split(",")) if event_name == "release" and not sdk_release: @@ -43,8 +39,7 @@ def resolve_layer_sdk_version( if event_name == "release" and pinned_version != source_sdk_version: raise ValueError( - "New SDK releases must update tool.lambda-layer.sdk-version " - f"to {source_sdk_version}" + f"New SDK releases must update layer.sdk-version to {source_sdk_version}" ) return source_sdk_version @@ -57,7 +52,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--event-name", required=True) parser.add_argument("--release-tag", default="") parser.add_argument("--source-sdk-version", required=True) - parser.add_argument("--otel-pyproject", type=Path, required=True) + parser.add_argument("--metadata", type=Path, required=True) args = parser.parse_args(argv) print( @@ -65,7 +60,7 @@ def main(argv: list[str] | None = None) -> int: event_name=args.event_name, release_tag=args.release_tag, source_sdk_version=args.source_sdk_version, - otel_pyproject=args.otel_pyproject, + metadata_path=args.metadata, ) ) return 0 diff --git a/.github/scripts/tests/test_resolve_layer_sdk_version.py b/.github/scripts/tests/test_resolve_layer_sdk_version.py index 05b44da1..5308179d 100644 --- a/.github/scripts/tests/test_resolve_layer_sdk_version.py +++ b/.github/scripts/tests/test_resolve_layer_sdk_version.py @@ -12,61 +12,61 @@ from resolve_layer_sdk_version import resolve_layer_sdk_version -def _write_otel_pyproject(tmp_path: Path, sdk_version: str) -> Path: - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text(f'[tool.lambda-layer]\nsdk-version = "{sdk_version}"\n') - return pyproject +def _write_metadata(tmp_path: Path, sdk_version: str) -> Path: + metadata = tmp_path / "lambda-layer-publish.toml" + metadata.write_text(f'[layer]\nsdk-version = "{sdk_version}"\n') + return metadata def test_otel_only_release_uses_pinned_published_sdk(tmp_path: Path) -> None: - pyproject = _write_otel_pyproject(tmp_path, "1.8.0") + metadata = _write_metadata(tmp_path, "1.8.0") assert ( resolve_layer_sdk_version( event_name="release", release_tag="otel-v0.4.1", source_sdk_version="1.9.0", - otel_pyproject=pyproject, + metadata_path=metadata, ) == "1.8.0" ) def test_combined_release_uses_source_sdk(tmp_path: Path) -> None: - pyproject = _write_otel_pyproject(tmp_path, "1.9.0") + metadata = _write_metadata(tmp_path, "1.9.0") assert ( resolve_layer_sdk_version( event_name="release", release_tag="sdk-v1.9.0,otel-v0.5.0", source_sdk_version="1.9.0", - otel_pyproject=pyproject, + metadata_path=metadata, ) == "1.9.0" ) def test_combined_release_requires_updated_pin(tmp_path: Path) -> None: - pyproject = _write_otel_pyproject(tmp_path, "1.8.0") + metadata = _write_metadata(tmp_path, "1.8.0") with pytest.raises(ValueError, match="New SDK releases must update"): resolve_layer_sdk_version( event_name="release", release_tag="sdk-v1.9.0,otel-v0.5.0", source_sdk_version="1.9.0", - otel_pyproject=pyproject, + metadata_path=metadata, ) def test_manual_run_uses_source_sdk(tmp_path: Path) -> None: - pyproject = _write_otel_pyproject(tmp_path, "1.8.0") + metadata = _write_metadata(tmp_path, "1.8.0") assert ( resolve_layer_sdk_version( event_name="workflow_dispatch", release_tag="", source_sdk_version="1.9.0", - otel_pyproject=pyproject, + metadata_path=metadata, ) == "1.9.0" ) diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index c273e59f..221296a6 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -91,7 +91,7 @@ jobs: --event-name "$GITHUB_EVENT_NAME" \ --release-tag "$RELEASE_TAG" \ --source-sdk-version "$SOURCE_SDK_VERSION" \ - --otel-pyproject packages/aws-durable-execution-sdk-python-otel/pyproject.toml) + --metadata .github/lambda-layer-publish.toml) echo "sdk_version=${SDK_VERSION}" >> "$GITHUB_OUTPUT" echo "otel_version=${OTEL_VERSION}" >> "$GITHUB_OUTPUT" diff --git a/RELEASING.md b/RELEASING.md index 4de60d04..6bae46b5 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -75,10 +75,9 @@ workflow. It builds the SDK and OTel plugin into Lambda layers for each supported Python runtime and architecture, then publishes public versions of the `aws-durable-execution-sdk-python-otel-plugin` layer. For OTel-only releases, the workflow downloads the exact SDK version pinned by -`tool.lambda-layer.sdk-version` in the OTel package's `pyproject.toml`; that -version must already be published to PyPI. Combined SDK and OTel releases -require the pin to match the new SDK version and build both distributions from -the tagged source. +`layer.sdk-version` in `.github/lambda-layer-publish.toml`; that version must +already be published to PyPI. Combined SDK and OTel releases require the pin to +match the new SDK version and build both distributions from the tagged source. The publishing job uses the `lambda-layer-publish` GitHub environment and its `LAYER_PUBLISH_ROLE_ARN` secret. Set the optional `LAYER_PUBLISH_REGIONS` diff --git a/packages/aws-durable-execution-sdk-python-otel/pyproject.toml b/packages/aws-durable-execution-sdk-python-otel/pyproject.toml index 904851af..9f9d4ff4 100644 --- a/packages/aws-durable-execution-sdk-python-otel/pyproject.toml +++ b/packages/aws-durable-execution-sdk-python-otel/pyproject.toml @@ -33,9 +33,6 @@ dependencies = [ otel-invocation = "aws_durable_execution_sdk_python_otel.plugin_provider:INVOCATION_OTEL_PLUGIN_PROVIDER" otel-execution = "aws_durable_execution_sdk_python_otel.plugin_provider:EXECUTION_OTEL_PLUGIN_PROVIDER" -[tool.lambda-layer] -sdk-version = "1.8.0" - [project.optional-dependencies] # Instrumentation used by ExecutionOtelPlugin's auto-configured provider path. # Kept optional so the InvocationOtelPlugin (ADOT / global provider) install