diff --git a/.github/scripts/resolve_layer_publish_targets.py b/.github/scripts/resolve_layer_publish_targets.py new file mode 100644 index 00000000..77b18f71 --- /dev/null +++ b/.github/scripts/resolve_layer_publish_targets.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass + + +DEFAULT_AUDIENCE = "sts.amazonaws.com" +CHINA_AUDIENCE = "sts.amazonaws.com.cn" + + +@dataclass(frozen=True) +class CredentialGroup: + name: str + regions: tuple[str, ...] + role_secret: str + aws_region: str + audience: str = DEFAULT_AUDIENCE + + +@dataclass(frozen=True) +class PublishTarget: + name: str + regions: tuple[str, ...] + role_secret: str + aws_region: str + audience: str + + def to_workflow_matrix_entry(self) -> dict[str, str]: + return { + "name": self.name, + "regions": ",".join(self.regions), + "role_secret": self.role_secret, + "aws_region": self.aws_region, + "audience": self.audience, + } + + +DEFAULT_COMMERCIAL_REGIONS = ( + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", +) + +OPT_IN_REGIONS = ( + "af-south-1", + "ap-east-1", + "ap-east-2", + "ap-south-2", + "ap-southeast-3", + "ap-southeast-4", + "ap-southeast-5", + "ap-southeast-6", + "ap-southeast-7", + "ca-west-1", + "eu-central-2", + "eu-south-1", + "eu-south-2", + "il-central-1", + "me-central-1", + "me-south-1", + "mx-central-1", +) + +CHINA_REGIONS = ("cn-north-1", "cn-northwest-1") +US_GOV_REGIONS = ("us-gov-east-1", "us-gov-west-1") + + +def _region_role_secret(region: str) -> str: + return f"LAYER_PUBLISH_ROLE_ARN_{region.upper().replace('-', '_')}" + + +CREDENTIAL_GROUPS = ( + CredentialGroup( + name="commercial-default", + regions=DEFAULT_COMMERCIAL_REGIONS, + role_secret="LAYER_PUBLISH_ROLE_ARN", + aws_region="us-east-1", + ), + *( + CredentialGroup( + name=region, + regions=(region,), + role_secret=_region_role_secret(region), + aws_region=region, + ) + for region in OPT_IN_REGIONS + ), + CredentialGroup( + name="china", + regions=CHINA_REGIONS, + role_secret="LAYER_PUBLISH_ROLE_ARN_CHINA", + aws_region="cn-north-1", + audience=CHINA_AUDIENCE, + ), + CredentialGroup( + name="us-gov", + regions=US_GOV_REGIONS, + role_secret="LAYER_PUBLISH_ROLE_ARN_US_GOV", + aws_region="us-gov-east-1", + ), +) + +SUPPORTED_REGIONS = frozenset( + region for group in CREDENTIAL_GROUPS for region in group.regions +) + + +def resolve_publish_targets(regions: str | None) -> tuple[PublishTarget, ...]: + requested_regions = { + region.strip() for region in (regions or "").split(",") if region.strip() + } + unknown_regions = requested_regions - SUPPORTED_REGIONS + if unknown_regions: + unknown_list = ", ".join(sorted(unknown_regions)) + raise ValueError(f"Unsupported layer publish regions: {unknown_list}") + + targets: list[PublishTarget] = [] + for group in CREDENTIAL_GROUPS: + selected_regions = ( + tuple(region for region in group.regions if region in requested_regions) + if requested_regions + else group.regions + ) + if selected_regions: + targets.append( + PublishTarget( + name=group.name, + regions=selected_regions, + role_secret=group.role_secret, + aws_region=group.aws_region, + audience=group.audience, + ) + ) + return tuple(targets) + + +def serialize_publish_targets(targets: tuple[PublishTarget, ...]) -> str: + entries = [target.to_workflow_matrix_entry() for target in targets] + return json.dumps(entries, separators=(",", ":")) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Resolve credential-scoped Lambda layer publish targets." + ) + parser.add_argument( + "--regions", + default="", + help="Optional comma-separated AWS Regions; defaults to every supported Region.", + ) + args = parser.parse_args() + + try: + targets = resolve_publish_targets(args.regions) + except ValueError as error: + parser.error(str(error)) + print(serialize_publish_targets(targets)) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/tests/test_lambda_layer_publish_workflow.py b/.github/scripts/tests/test_lambda_layer_publish_workflow.py new file mode 100644 index 00000000..75b441c5 --- /dev/null +++ b/.github/scripts/tests/test_lambda_layer_publish_workflow.py @@ -0,0 +1,51 @@ +from pathlib import Path + +import yaml + + +WORKFLOW_PATH = Path(__file__).parents[2] / "workflows" / "lambda-layer-publish.yml" + + +def _step_by_name(steps: list[dict[str, object]], name: str) -> dict[str, object]: + return next(step for step in steps if step.get("name") == name) + + +def test_publish_workflow_uses_resolved_credential_matrix() -> None: + workflow = yaml.safe_load(WORKFLOW_PATH.read_text()) + build_job = workflow["jobs"]["build-distributions"] + publish_job = workflow["jobs"]["publish-layer"] + + assert build_job["outputs"]["publish_targets"] == ( + "${{ steps.publish-targets.outputs.publish_targets }}" + ) + resolve_step = _step_by_name(build_job["steps"], "Resolve publish targets") + assert resolve_step["env"]["LAYER_REGIONS"] == ( + "${{ inputs.regions || vars.LAYER_PUBLISH_REGIONS }}" + ) + assert ".github/scripts/resolve_layer_publish_targets.py" in resolve_step["run"] + + matrix = publish_job["strategy"]["matrix"] + assert matrix["publish_target"] == ( + "${{ fromJSON(needs.build-distributions.outputs.publish_targets) }}" + ) + assert set(matrix) == {"publish_target"} + + +def test_publish_workflow_resolves_role_and_partition_settings_from_matrix() -> None: + workflow = yaml.safe_load(WORKFLOW_PATH.read_text()) + publish_steps = workflow["jobs"]["publish-layer"]["steps"] + credentials_step = _step_by_name(publish_steps, "Configure AWS credentials") + publish_step = _step_by_name(publish_steps, "Publish layer versions") + + assert credentials_step["with"]["role-to-assume"] == ( + "${{ secrets[matrix.publish_target.role_secret] }}" + ) + assert credentials_step["with"]["aws-region"] == ( + "${{ matrix.publish_target.aws_region }}" + ) + assert credentials_step["with"]["audience"] == ( + "${{ matrix.publish_target.audience }}" + ) + assert publish_step["env"]["LAYER_REGIONS"] == ( + "${{ matrix.publish_target.regions }}" + ) diff --git a/.github/scripts/tests/test_resolve_layer_publish_targets.py b/.github/scripts/tests/test_resolve_layer_publish_targets.py new file mode 100644 index 00000000..d63b99f4 --- /dev/null +++ b/.github/scripts/tests/test_resolve_layer_publish_targets.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import json +import os +import sys + +import pytest + + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from resolve_layer_publish_targets import ( + CHINA_AUDIENCE, + CHINA_REGIONS, + CREDENTIAL_GROUPS, + DEFAULT_AUDIENCE, + DEFAULT_COMMERCIAL_REGIONS, + OPT_IN_REGIONS, + SUPPORTED_REGIONS, + US_GOV_REGIONS, + resolve_publish_targets, + serialize_publish_targets, +) + + +EXPECTED_DEFAULT_COMMERCIAL_REGIONS = { + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", +} +EXPECTED_OPT_IN_REGIONS = { + "af-south-1", + "ap-east-1", + "ap-east-2", + "ap-south-2", + "ap-southeast-3", + "ap-southeast-4", + "ap-southeast-5", + "ap-southeast-6", + "ap-southeast-7", + "ca-west-1", + "eu-central-2", + "eu-south-1", + "eu-south-2", + "il-central-1", + "me-central-1", + "me-south-1", + "mx-central-1", +} + + +def test_region_sets_match_the_configured_aws_partitions() -> None: + assert set(DEFAULT_COMMERCIAL_REGIONS) == EXPECTED_DEFAULT_COMMERCIAL_REGIONS + assert set(OPT_IN_REGIONS) == EXPECTED_OPT_IN_REGIONS + assert CHINA_REGIONS == ("cn-north-1", "cn-northwest-1") + assert US_GOV_REGIONS == ("us-gov-east-1", "us-gov-west-1") + + +def test_default_targets_use_separate_credentials_for_each_region_scope() -> None: + targets = resolve_publish_targets(None) + + assert len(targets) == len(OPT_IN_REGIONS) + 3 + assert { + region for target in targets for region in target.regions + } == SUPPORTED_REGIONS + + default_target = next( + target for target in targets if target.name == "commercial-default" + ) + assert default_target.regions == DEFAULT_COMMERCIAL_REGIONS + assert default_target.role_secret == "LAYER_PUBLISH_ROLE_ARN" + assert default_target.audience == DEFAULT_AUDIENCE + + for region in OPT_IN_REGIONS: + target = next(target for target in targets if target.name == region) + assert target.regions == (region,) + assert target.role_secret == ( + f"LAYER_PUBLISH_ROLE_ARN_{region.upper().replace('-', '_')}" + ) + assert target.aws_region == region + assert target.audience == DEFAULT_AUDIENCE + + china_target = next(target for target in targets if target.name == "china") + assert china_target.regions == CHINA_REGIONS + assert china_target.role_secret == "LAYER_PUBLISH_ROLE_ARN_CHINA" + assert china_target.audience == CHINA_AUDIENCE + + us_gov_target = next(target for target in targets if target.name == "us-gov") + assert us_gov_target.regions == US_GOV_REGIONS + assert us_gov_target.role_secret == "LAYER_PUBLISH_ROLE_ARN_US_GOV" + assert us_gov_target.audience == DEFAULT_AUDIENCE + + +def test_requested_regions_only_include_required_credential_groups() -> None: + targets = resolve_publish_targets(" cn-northwest-1,us-east-1,ap-east-1,cn-north-1 ") + + assert [(target.name, target.regions) for target in targets] == [ + ("commercial-default", ("us-east-1",)), + ("ap-east-1", ("ap-east-1",)), + ("china", CHINA_REGIONS), + ] + + +def test_requested_regions_reject_unsupported_regions() -> None: + with pytest.raises( + ValueError, + match="Unsupported layer publish regions: moon-west-1, test-east-1", + ): + resolve_publish_targets("test-east-1,moon-west-1") + + +def test_serialized_targets_are_valid_workflow_matrix_entries() -> None: + targets = resolve_publish_targets("us-gov-west-1") + + assert json.loads(serialize_publish_targets(targets)) == [ + { + "name": "us-gov", + "regions": "us-gov-west-1", + "role_secret": "LAYER_PUBLISH_ROLE_ARN_US_GOV", + "aws_region": "us-gov-east-1", + "audience": DEFAULT_AUDIENCE, + } + ] + + +def test_credential_groups_do_not_assign_a_region_more_than_once() -> None: + configured_regions = [ + region for group in CREDENTIAL_GROUPS for region in group.regions + ] + + assert len(configured_regions) == len(set(configured_regions)) + assert len(CREDENTIAL_GROUPS) <= 256 diff --git a/.github/workflows/lambda-layer-publish.yml b/.github/workflows/lambda-layer-publish.yml index b28c9792..de1ddeca 100644 --- a/.github/workflows/lambda-layer-publish.yml +++ b/.github/workflows/lambda-layer-publish.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: inputs: regions: - description: "Comma-separated AWS Regions; defaults to all commercial Regions" + description: "Comma-separated AWS Regions; defaults to all configured Regions" required: false type: string @@ -20,41 +20,6 @@ concurrency: env: LAYER_NAME: aws-durable-execution-sdk-python-otel-plugin SOURCE_REF: ${{ github.event_name == 'workflow_dispatch' && 'main' || github.event.release.tag_name }} - DEFAULT_LAYER_REGIONS: >- - af-south-1, - ap-east-1, - ap-east-2, - ap-northeast-1, - ap-northeast-2, - ap-northeast-3, - ap-south-1, - ap-south-2, - ap-southeast-1, - ap-southeast-2, - ap-southeast-3, - ap-southeast-4, - ap-southeast-5, - ap-southeast-6, - ap-southeast-7, - ca-central-1, - ca-west-1, - eu-central-1, - eu-central-2, - eu-north-1, - eu-south-1, - eu-south-2, - eu-west-1, - eu-west-2, - eu-west-3, - il-central-1, - me-central-1, - me-south-1, - mx-central-1, - sa-east-1, - us-east-1, - us-east-2, - us-west-1, - us-west-2 jobs: build-distributions: @@ -66,6 +31,7 @@ jobs: outputs: sdk_version: ${{ steps.versions.outputs.sdk_version }} otel_version: ${{ steps.versions.outputs.otel_version }} + publish_targets: ${{ steps.publish-targets.outputs.publish_targets }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -95,6 +61,15 @@ jobs: echo "sdk_version=${SDK_VERSION}" >> "$GITHUB_OUTPUT" echo "otel_version=${OTEL_VERSION}" >> "$GITHUB_OUTPUT" + - name: Resolve publish targets + id: publish-targets + env: + LAYER_REGIONS: ${{ inputs.regions || vars.LAYER_PUBLISH_REGIONS }} + run: | + PUBLISH_TARGETS=$(python .github/scripts/resolve_layer_publish_targets.py \ + --regions "$LAYER_REGIONS") + echo "publish_targets=${PUBLISH_TARGETS}" >> "$GITHUB_OUTPUT" + - name: Build SDK distribution if: >- github.event_name == 'workflow_dispatch' || @@ -189,8 +164,10 @@ jobs: permissions: contents: read id-token: write - env: - LAYER_REGIONS: ${{ inputs.regions || vars.LAYER_PUBLISH_REGIONS }} + strategy: + fail-fast: false + matrix: + publish_target: ${{ fromJSON(needs.build-distributions.outputs.publish_targets) }} steps: - name: Download layer artifact @@ -202,12 +179,14 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 with: - role-to-assume: ${{ secrets.LAYER_PUBLISH_ROLE_ARN }} - role-session-name: otelLayerPublish - aws-region: us-east-1 + role-to-assume: ${{ secrets[matrix.publish_target.role_secret] }} + role-session-name: otelLayerPublish-${{ matrix.publish_target.name }} + aws-region: ${{ matrix.publish_target.aws_region }} + audience: ${{ matrix.publish_target.audience }} - name: Publish layer versions env: + LAYER_REGIONS: ${{ matrix.publish_target.regions }} LAYER_ZIP: dist/${{ env.LAYER_NAME }}.zip SDK_VERSION: ${{ needs.build-distributions.outputs.sdk_version }} OTEL_VERSION: ${{ needs.build-distributions.outputs.otel_version }} @@ -215,8 +194,7 @@ jobs: PUBLISHED=false FAILED_REGIONS=() LOCAL_CODE_SHA256=$(openssl dgst -sha256 -binary "$LAYER_ZIP" | openssl base64 -A) - REGION_LIST=${LAYER_REGIONS:-$DEFAULT_LAYER_REGIONS} - IFS=',' read -ra REGIONS <<< "$REGION_LIST" + IFS=',' read -ra REGIONS <<< "$LAYER_REGIONS" for REGION in "${REGIONS[@]}"; do REGION=$(echo "$REGION" | xargs) if [ -z "$REGION" ]; then diff --git a/.github/workflows/test-parser.yml b/.github/workflows/test-parser.yml index fde37133..5ddada9b 100644 --- a/.github/workflows/test-parser.yml +++ b/.github/workflows/test-parser.yml @@ -5,13 +5,17 @@ on: paths: - '.github/scripts/build_lambda_layer.py' - '.github/scripts/parse_sdk_branch.py' + - '.github/scripts/resolve_layer_publish_targets.py' - '.github/scripts/tests/**' + - '.github/workflows/lambda-layer-publish.yml' push: branches: [ main ] paths: - '.github/scripts/build_lambda_layer.py' - '.github/scripts/parse_sdk_branch.py' + - '.github/scripts/resolve_layer_publish_targets.py' - '.github/scripts/tests/**' + - '.github/workflows/lambda-layer-publish.yml' permissions: contents: read @@ -23,10 +27,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install test dependencies - run: python -m pip install pytest + run: python -m pip install pytest PyYAML==6.0.2 - name: Run script tests run: | python -m pytest \ .github/scripts/tests/test_build_lambda_layer.py \ - .github/scripts/tests/test_parse_sdk_branch.py + .github/scripts/tests/test_lambda_layer_publish_workflow.py \ + .github/scripts/tests/test_parse_sdk_branch.py \ + .github/scripts/tests/test_resolve_layer_publish_targets.py diff --git a/RELEASING.md b/RELEASING.md index 6bae46b5..f9284a45 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -79,16 +79,30 @@ For OTel-only releases, the workflow downloads the exact SDK version pinned by already be published to PyPI. Combined SDK and OTel releases require the pin to match the new SDK version and build both distributions from the tagged source. -The publishing job uses the `lambda-layer-publish` GitHub environment and its -`LAYER_PUBLISH_ROLE_ARN` secret. Set the optional `LAYER_PUBLISH_REGIONS` -environment variable to a comma-separated list of AWS Regions. When unset, the -workflow publishes to every commercial AWS Region supported by Lambda. +The publishing job uses the `lambda-layer-publish` GitHub environment and +credential-scoped role ARN secrets: + +- `LAYER_PUBLISH_ROLE_ARN` covers the default-enabled commercial Regions. +- Each opt-in commercial Region uses + `LAYER_PUBLISH_ROLE_ARN_`, with the Region uppercased and hyphens + replaced by underscores. Configure one secret for each of `af-south-1`, + `ap-east-1`, `ap-east-2`, `ap-south-2`, `ap-southeast-3`, + `ap-southeast-4`, `ap-southeast-5`, `ap-southeast-6`, `ap-southeast-7`, + `ca-west-1`, `eu-central-2`, `eu-south-1`, `eu-south-2`, `il-central-1`, + `me-central-1`, `me-south-1`, and `mx-central-1`. +- `LAYER_PUBLISH_ROLE_ARN_CHINA` covers `cn-north-1` and `cn-northwest-1`. +- `LAYER_PUBLISH_ROLE_ARN_US_GOV` covers `us-gov-east-1` and + `us-gov-west-1`. + +Set the optional `LAYER_PUBLISH_REGIONS` environment variable to a +comma-separated list of these AWS Regions. When unset, the workflow publishes +to every configured Region. The workflow can also be run manually from the Actions tab on `main`; its optional `regions` input overrides `LAYER_PUBLISH_REGIONS` for that run. -Each runtime and architecture layer archive is built once and retained as a -workflow artifact so retries publish the exact same resolved dependencies. Its -SHA-256 is included in the layer description and verified before reuse. -The publishing role must allow `lambda:PublishLayerVersion` and +The universal layer archive is built once and retained as a workflow artifact +so retries publish the exact same resolved dependencies. Its SHA-256 is included +in the layer description and verified before reuse. +Each publishing role must allow `lambda:PublishLayerVersion` and `lambda:AddLayerVersionPermission`, as well as `lambda:ListLayerVersions` and `lambda:GetLayerVersion` for identity-checked, idempotent release retries.