From fb1a96b69d186c6c4433655343a9ef5dfd7dd7f1 Mon Sep 17 00:00:00 2001 From: Adrian Gavrila Date: Mon, 14 Sep 2026 15:12:09 -0400 Subject: [PATCH 1/3] Decouple CoPyRIT code and infrastructure deployments Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- gui-deploy.yml | 94 ++++----- infra/README.md | 39 +++- infra/pipelines/deploy-infra.yml | 60 ++++++ infra/pipelines/deploy_code.py | 192 +++++++++++++++++ infra/pipelines/deploy_public_nat.sh | 18 +- tests/unit/infra/test_code_deployment.py | 204 +++++++++++++++++++ tests/unit/infra/test_pipeline_guardrails.py | 46 ++++- 7 files changed, 581 insertions(+), 72 deletions(-) create mode 100644 infra/pipelines/deploy-infra.yml create mode 100644 infra/pipelines/deploy_code.py create mode 100644 tests/unit/infra/test_code_deployment.py diff --git a/gui-deploy.yml b/gui-deploy.yml index 92b618bb73..c9cbd899ae 100644 --- a/gui-deploy.yml +++ b/gui-deploy.yml @@ -1,9 +1,7 @@ # CI/CD pipeline for the CoPyRIT GUI. # -# Every deployment uses the single topology defined by infra/main.bicep: -# public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. -# Environment variable groups also provide adminGroupObjectId for configuration -# authorization and optional pyritConfigFileUri for blob-backed configuration. +# Code-only test deployment is the default. Infrastructure reconciliation is +# explicit and runs with the current image before deploying new code. trigger: branches: @@ -19,6 +17,10 @@ trigger: pr: none parameters: + - name: deployInfra + displayName: 'Deploy infrastructure before code' + type: boolean + default: false - name: deployToProd displayName: 'Deploy to production' type: boolean @@ -123,9 +125,19 @@ stages: immutable_image="$PYRIT_ACR_LOGIN_SERVER/$PYRIT_IMAGE_NAME@$digest" echo "##vso[task.setvariable variable=immutableImage;isOutput=true]$immutable_image" + - ${{ if eq(parameters.deployInfra, true) }}: + - template: infra/pipelines/deploy-infra.yml + parameters: + stageName: DeployTestInfra + dependsOn: Build + slot: test + - stage: DeployTest - displayName: 'Deploy to Test' - dependsOn: Build + displayName: 'Deploy code to Test' + dependsOn: + - Build + - ${{ if eq(parameters.deployInfra, true) }}: + - DeployTestInfra variables: - group: copyrit-gui-common - group: copyrit-gui-test @@ -135,7 +147,7 @@ stages: vmImage: 'ubuntu-latest' jobs: - deployment: DeployToTest - displayName: 'Deploy test environment' + displayName: 'Deploy test application' timeoutInMinutes: 120 environment: 'copyrit-test' strategy: @@ -146,35 +158,18 @@ stages: fetchDepth: 1 - task: AzureCLI@2 - displayName: 'Preview, deploy, and verify test' + displayName: 'Deploy and verify test code without changing networking' env: PYRIT_SLOT: test - PYRIT_BUILD_ID: $(Build.BuildId) - PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) - PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) PYRIT_APP_NAME: $(deploymentAppName) PYRIT_CONTAINER_IMAGE: $(immutableImage) - PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) - PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) - PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) - PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) - PYRIT_ENTRA_TENANT_ID: $(entraTenantId) - PYRIT_ENTRA_CLIENT_ID: $(entraClientId) - PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) - PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId) - PYRIT_CONFIG_FILE_URI: $(pyritConfigFileUri) - PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) - PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) - PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) PYRIT_ACR_RESOURCE_ID: $(acrResourceId) - PYRIT_ENABLE_OTEL: $(enableOtel) - PYRIT_ENV_SECRET_NAME: $(envSecretName) inputs: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' - scriptLocation: 'scriptPath' - scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' + scriptLocation: 'inlineScript' + inlineScript: python3 "$(Build.SourcesDirectory)/infra/pipelines/deploy_code.py" - stage: ApproveProd displayName: 'Approve Production Deployment' @@ -213,16 +208,28 @@ stages: allowApproversToApproveTheirOwnRuns: false instructions: | Confirm test is healthy, Entra and backend group authorization work, - the static test egress IP is allow-listed, and production variables - describe the same public ACA plus fixed NAT topology. + and the static test egress IP is allow-listed. + Infrastructure deployment requested: ${{ parameters.deployInfra }}. + Code-only deployment preserves the existing public/private access mode. + If infrastructure is requested, approve the production Private Link + cutover and configuration changes before the new code is deployed. onTimeout: reject + - ${{ if eq(parameters.deployInfra, true) }}: + - template: infra/pipelines/deploy-infra.yml + parameters: + stageName: DeployProdInfra + dependsOn: ApproveProd + slot: prod + - stage: DeployProd - displayName: 'Deploy to Production' + displayName: 'Deploy code to Production' dependsOn: - ApproveProd - Build - condition: succeeded('ApproveProd') + - ${{ if eq(parameters.deployInfra, true) }}: + - DeployProdInfra + condition: and(succeeded(), succeeded('ApproveProd')) variables: - group: copyrit-gui-common - group: copyrit-gui-prod @@ -232,7 +239,7 @@ stages: vmImage: 'ubuntu-latest' jobs: - deployment: DeployToProd - displayName: 'Deploy production environment' + displayName: 'Deploy production application' timeoutInMinutes: 120 environment: 'copyrit-prod' strategy: @@ -243,32 +250,15 @@ stages: fetchDepth: 1 - task: AzureCLI@2 - displayName: 'Preview, deploy, and verify production' + displayName: 'Deploy and verify production code without changing networking' env: PYRIT_SLOT: prod - PYRIT_BUILD_ID: $(Build.BuildId) - PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) - PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) PYRIT_APP_NAME: $(deploymentAppName) PYRIT_CONTAINER_IMAGE: $(immutableImage) - PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) - PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) - PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) - PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) - PYRIT_ENTRA_TENANT_ID: $(entraTenantId) - PYRIT_ENTRA_CLIENT_ID: $(entraClientId) - PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) - PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId) - PYRIT_CONFIG_FILE_URI: $(pyritConfigFileUri) - PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) - PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) - PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) PYRIT_ACR_RESOURCE_ID: $(acrResourceId) - PYRIT_ENABLE_OTEL: $(enableOtel) - PYRIT_ENV_SECRET_NAME: $(envSecretName) inputs: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' - scriptLocation: 'scriptPath' - scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' + scriptLocation: 'inlineScript' + inlineScript: python3 "$(Build.SourcesDirectory)/infra/pipelines/deploy_code.py" diff --git a/infra/README.md b/infra/README.md index 85725e9578..abccff4ec0 100644 --- a/infra/README.md +++ b/infra/README.md @@ -72,7 +72,7 @@ flowchart TB app -.->|"Traces after agent setup"| appInsights ``` -The base topology is public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. `enableFrontDoor=true` adds Front Door Premium as the preferred managed HTTPS URL. By default, the ACA origin remains concurrently public and can bypass Front Door. `enableFrontDoorPrivateLink=true` instead connects Premium Front Door to the ACA environment through Private Link; setting `disableContainerAppsPublicAccess=true` then removes the direct public ACA path. Bicep rejects public-access shutdown unless both Front Door and its Private Link origin are enabled. The team ADO workflow uses this isolated-origin mode; community examples leave all three Front Door settings disabled. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door rather than the original client. Front Door changes inbound routing only: outbound connections from ACA continue to use the NAT Gateway's static IPv4. +The base topology is public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. `enableFrontDoor=true` adds Front Door Premium as the preferred managed HTTPS URL. By default, the ACA origin remains concurrently public and can bypass Front Door. `enableFrontDoorPrivateLink=true` instead connects Premium Front Door to the ACA environment through Private Link; setting `disableContainerAppsPublicAccess=true` then removes the direct public ACA path. Bicep rejects public-access shutdown unless both Front Door and its Private Link origin are enabled. The team ADO infrastructure stage uses this isolated-origin mode; code-only runs preserve the existing access mode. Community examples leave all three Front Door settings disabled. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door rather than the original client. Front Door changes inbound routing only: outbound connections from ACA continue to use the NAT Gateway's static IPv4. ## Development Workflow @@ -403,19 +403,40 @@ az deployment group show -g -n \ > This section documents the repository maintainers' internal pipeline. It depends on Microsoft team-owned ADO service connections, environments, and variable groups. It is not required or expected for community deployments; use direct Bicep or `deploy_instance.py` instead. -`gui-deploy.yml` is an **update-only** workflow for the pre-created test-v2 and prod-v2 stacks: +`gui-deploy.yml` is an **update-only** workflow for the pre-created test-v2 and prod-v2 stacks. Both parameters default to `false`: + +| `deployInfra` | `deployToProd` | Workflow | +| --- | --- | --- | +| `false` | `false` | Build, deploy code to test | +| `true` | `false` | Build, deploy test infrastructure with the current image, deploy code to test | +| `false` | `true` | Build, deploy code to test, production approval, deploy the same image to production | +| `true` | `true` | Build, test infrastructure, test code, production approval, production infrastructure, production code | + +Qualifying merges to `main` automatically deploy **code only to test**. Production remains opt-in: manually queue a commit merged to `main` with `deployToProd=true`. Approval rejects on timeout and the requester cannot self-approve. An infrastructure failure blocks the corresponding code stage; a code failure never invokes infrastructure rollback. + +#### Code-only deployment + +`infra/pipelines/deploy_code.py` updates only the existing `pyrit-gui` container image. It validates the subscription, registry digest, environment, and single-container/single-revision topology before writing. Application configuration, identity, networking, NAT, and Front Door are not redeployed. + +The script verifies the exact new revision and its existing access mode: direct ACA `/api/health` when public access is enabled, or Front Door `/api/health` when public access is disabled. It reports the mode explicitly and never falls back from a failed private path to public access. A public-mode success is **not** certification of Private Link readiness. The data-plane check has a five-minute budget after revision readiness. + +Code deployment does not downgrade or create a database; application startup still follows the image's normal migration behavior. Failures are reported without automatic image or database rollback because migrations may make the previous image incompatible. The previous image digest is logged for an explicit recovery decision. + +#### Optional infrastructure deployment + +Set `deployInfra=true` when changing Bicep-managed configuration or networking. Each infrastructure stage runs **before** the new code, using the image already deployed to that environment, not the Build output. Infrastructure rollback therefore cannot install a newly built application that has not passed code verification. + +`infra/pipelines/deploy_public_nat.sh` retains the existing infrastructure safeguards: 1. Build the source image and push a commit-SHA tag to ACR. -2. Capture the exact pushed digest and pass it across stages. +2. Capture the exact pushed digest for the code stages; infrastructure retains the currently deployed digest. 3. Require the existing app, environment, VNet, subnet, NAT, and reserved PIP; validate their IDs, prefixes, tags, SKU, allocation, and attachments. 4. Run a full ARM `what-if` through a fail-closed validator; reject malformed results, deletions, cross-resource-group writes, protected-network deltas other than the documented read-only NAT/PIP normalization, and core network, app, or Log Analytics workspace creates. The expected PIP protection lock may be created. 5. Preserve policy-managed PIP tags and deploy with Front Door Private Link, ACA public access disabled, and PIP protection enabled. 6. Validate the AFD origin targets the expected ACA environment, approve only active requests with the deterministic message, and require the ACA-side connection to report `Approved`. AFD can continue to display `Pending` after approval, so successful AFD health is the data-plane readiness signal. -7. Allow up to 30 minutes for Front Door propagation, then verify ACA public access is disabled, the digest-pinned revision and Front Door `/api/health` are healthy, direct ACA access is unavailable, and the PIP resource ID/address is unchanged. +7. Allow up to 30 minutes for Front Door propagation, then verify ACA public access is disabled, the retained-image revision and Front Door `/api/health` are healthy, direct ACA access is unavailable, and the PIP resource ID/address is unchanged. 8. If cutover validation fails, redeploy the prior public AFD origin and re-enable ACA public access; otherwise print the Front Door URL and static egress IPv4. -Qualifying merges to `main` automatically deploy test. Production deployment is independent of PyRIT package releases: manually queue a commit merged to `main` with `deployToProd=true`. The workflow deploys test first, then requires a timeout-rejecting manual approval whose requester cannot self-approve. - `copyrit-gui-common` supplies the shared image settings: | Variable | Purpose | @@ -441,7 +462,7 @@ Both `copyrit-gui-test` and `copyrit-gui-prod` supply: | `keyVaultResourceId`, `envSecretName` | Existing runtime configuration secret | | `acrResourceId`, `enableOtel` | Registry resource ID and observability setting | -The container image is not a library variable. The Build stage publishes the exact pushed digest as `immutableImage`, and both deployment stages consume that output. Do not add the legacy `image`, `resourceGroup`, `appName`, or `enablePrivateEndpoint` variables; the current workflow does not consume them. +The container image is not a library variable. The Build stage publishes the exact pushed digest as `immutableImage`, and both code stages consume that output. Infrastructure stages discover the existing image directly from ACA. Code-only runs consume only `deploymentResourceGroup`, `deploymentAppName`, and `acrResourceId` from each environment group; changes to the other configuration values require `deployInfra=true`. Do not add the legacy `image`, `resourceGroup`, `appName`, or `enablePrivateEndpoint` variables; the current workflow does not consume them. Pipeline definition 139 reads `gui-deploy.yml` from the GitHub commit being queued. Treat YAML and variable-group contract changes as one release: do not remove old keys before the commit that consumes the replacement keys reaches the target branch. Otherwise ADO leaves unresolved `$(name)` text in Bash, where it is interpreted as command substitution. @@ -451,7 +472,7 @@ The resource group, registry, image-pull authorization, managed identity, Key Va The internal workflow is update-only for networking: its app name and prefixes must resolve to the existing app/environment/VNet/subnet/NAT/PIP. It records the current PIP resource ID and address before preview, requires protected resources to remain unchanged except Azure read-only normalization, and verifies the same PIP/address after deployment. -The workflow also creates a `CanNotDelete` lock scoped to the reserved PIP. Its validated Front Door origin uses Private Link to the ACA environment, and the ACA public endpoint is disabled after deployment. +The optional infrastructure stage also creates a `CanNotDelete` lock scoped to the reserved PIP. Its validated Front Door origin uses Private Link to the ACA environment, and the ACA public endpoint is disabled after a successful infrastructure deployment. Code-only runs preserve the existing configuration, including an existing public-access fallback. ## Post-Deployment @@ -604,7 +625,7 @@ Supported Azure integrations, including OpenAI, Content Safety, and Speech, can ## Notes -- **Network topology**: Public ACA-managed HTTPS ingress with optional `allowedCidr` plus VNet-integrated fixed NAT egress is the base topology. Front Door Premium is an optional inbound layer. Private Link plus disabled ACA public access makes Front Door the only public application path. The team ADO workflow enables this isolated-origin mode. `allowedCidr` must be empty when Front Door is enabled; Bicep rejects the combination. +- **Network topology**: Public ACA-managed HTTPS ingress with optional `allowedCidr` plus VNet-integrated fixed NAT egress is the base topology. Front Door Premium is an optional inbound layer. Private Link plus disabled ACA public access makes Front Door the only public application path. The team ADO infrastructure stage enables this isolated-origin mode; default code-only runs preserve existing networking. `allowedCidr` must be empty when Front Door is enabled; Bicep rejects the combination. - **Ingress vs. egress**: Front Door affects inbound requests only. The reserved NAT public IP remains the source for ACA-originated outbound connections. - **NAT routing**: NAT Gateway supplies the outbound source IP only while the subnet's effective default route remains `Internet`. A UDR or propagated BGP `0.0.0.0/0` route to a firewall or gateway takes precedence; in that topology, allow-list the egress device's public IP instead. - **Network outputs**: `egressPublicIpAddress`, `natGatewayId`, `acaInfrastructureSubnetId`, and `vnetName` describe the created network. diff --git a/infra/pipelines/deploy-infra.yml b/infra/pipelines/deploy-infra.yml new file mode 100644 index 0000000000..143b2e9ac0 --- /dev/null +++ b/infra/pipelines/deploy-infra.yml @@ -0,0 +1,60 @@ +parameters: + - name: stageName + type: string + - name: dependsOn + type: string + - name: slot + type: string + values: + - test + - prod + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Deploy ${{ parameters.slot }} infrastructure' + dependsOn: ${{ parameters.dependsOn }} + variables: + - group: copyrit-gui-common + - group: copyrit-gui-${{ parameters.slot }} + pool: + vmImage: 'ubuntu-latest' + jobs: + - deployment: DeployInfrastructure + displayName: 'Preview, deploy, and verify infrastructure with the current image' + timeoutInMinutes: 120 + environment: 'copyrit-${{ parameters.slot }}' + strategy: + runOnce: + deploy: + steps: + - checkout: self + fetchDepth: 1 + - task: AzureCLI@2 + displayName: 'Preview, deploy, and verify ${{ parameters.slot }} infrastructure' + env: + PYRIT_SLOT: ${{ parameters.slot }} + PYRIT_BUILD_ID: $(Build.BuildId) + PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) + PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) + PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) + PYRIT_APP_NAME: $(deploymentAppName) + PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) + PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) + PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) + PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) + PYRIT_ENTRA_TENANT_ID: $(entraTenantId) + PYRIT_ENTRA_CLIENT_ID: $(entraClientId) + PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) + PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId) + PYRIT_CONFIG_FILE_URI: $(pyritConfigFileUri) + PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) + PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) + PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) + PYRIT_ACR_RESOURCE_ID: $(acrResourceId) + PYRIT_ENABLE_OTEL: $(enableOtel) + PYRIT_ENV_SECRET_NAME: $(envSecretName) + inputs: + azureSubscription: '$(azureServiceConnection)' + scriptType: 'bash' + scriptLocation: 'scriptPath' + scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' diff --git a/infra/pipelines/deploy_code.py b/infra/pipelines/deploy_code.py new file mode 100644 index 0000000000..d2803f623a --- /dev/null +++ b/infra/pipelines/deploy_code.py @@ -0,0 +1,192 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +"""Update only the existing CoPyRIT container image and verify its current access mode.""" + +import os +import re +import subprocess +import sys +import time + + +def _az(*arguments: str) -> str: + return subprocess.run( + ["az", *arguments, "--only-show-errors"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + +def _value(*arguments: str, query: str) -> str: + value = _az(*arguments, "--query", query, "--output", "tsv") + if not value or value in {"None", "null"} or "\n" in value or "\t" in value: + raise ValueError(f"Azure returned no single value for {query}") + return value + + +def _health_url(*, app_arguments: tuple[str, ...], environment_id: str, app_name: str) -> tuple[str, str]: + access = _value("containerapp", "env", "show", "--ids", environment_id, query="properties.publicNetworkAccess") + if access == "Enabled": + hostname = _value("containerapp", "show", *app_arguments, query="properties.configuration.ingress.fqdn") + suffix = ".azurecontainerapps.io" + elif access == "Disabled": + resource_group_id = environment_id.split("/providers/")[0] + hostname = _value( + "rest", + "--method", + "get", + "--url", + f"https://management.azure.com{resource_group_id}/providers/Microsoft.Cdn/profiles/" + f"{app_name}-afd/afdEndpoints?api-version=2024-09-01", + query="value[?properties.enabledState=='Enabled'].properties.hostName", + ) + suffix = ".azurefd.net" + else: + raise ValueError(f"Unsupported ACA public network access state: {access}") + if not re.fullmatch(r"[a-z0-9][a-z0-9.-]*", hostname) or not hostname.endswith(suffix): + raise ValueError("Azure returned an unexpected health-check hostname") + return f"https://{hostname}/api/health", access + + +def _wait_for_revision(*, app_arguments: tuple[str, ...], revision: str, image: str) -> None: + arguments = ("containerapp", "revision", "show", *app_arguments, "--revision", revision) + if _value(*arguments, query="properties.template.containers[0].image") != image: + raise ValueError("The deployed revision does not contain the requested image") + for attempt in range(1, 6): + health = _az(*arguments, "--query", "properties.healthState", "--output", "tsv") + print(f"Revision {revision} health attempt {attempt}/5: {health or ''}", flush=True) + if health == "Healthy": + return + if attempt < 5: + time.sleep(120) + raise RuntimeError("Deployed revision did not become healthy; networking was not changed") + + +def _wait_for_http_health(url: str) -> None: + deadline = time.monotonic() + 300 + while (remaining := deadline - time.monotonic()) > 0: + response = subprocess.run( + [ + "curl", + "--silent", + "--show-error", + "--output", + os.devnull, + "--write-out", + "%{http_code}", + "--max-time", + str(min(30, remaining)), + url, + ], + capture_output=True, + text=True, + check=False, + ) + status = response.stdout.strip() + print(f"Application health at {url}: {status or ''}", flush=True) + if response.returncode: + print(f"Health request failed: {response.stderr.strip()}", file=sys.stderr) + elif status == "200": + return + time.sleep(max(0, min(30, deadline - time.monotonic()))) + raise RuntimeError("Application health check failed; networking was not changed") + + +def deploy_code(*, slot: str, resource_group: str, app_name: str, acr_resource_id: str, image: str) -> None: + """Deploy an immutable image without applying templates or changing access settings.""" + if slot not in {"test", "prod"}: + raise ValueError("Invalid deployment slot") + if not re.fullmatch(r"[a-zA-Z0-9_.()-]{1,90}", resource_group) or resource_group.endswith("."): + raise ValueError("Invalid deployment resource group") + if not re.fullmatch(r"[a-z][a-z0-9-]{0,30}[a-z0-9]", app_name): + raise ValueError("Invalid container app name") + acr = re.fullmatch( + r"/subscriptions/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})" + r"/resourcegroups/[^/]+/providers/microsoft\.containerregistry/registries/([a-z0-9]{5,50})", + acr_resource_id.casefold(), + ) + if acr is None: + raise ValueError("ACR resource ID is not canonical") + subscription, registry = acr.groups() + if not re.fullmatch( + rf"{registry}\.azurecr\.io/[a-z0-9]+(?:[._-][a-z0-9]+)*" + r"(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*@sha256:[0-9a-fA-F]{64}", + image, + ): + raise ValueError("Image must be an immutable digest from the configured ACR") + if _value("account", "show", query="id").casefold() != subscription: + raise ValueError("Azure subscription does not match ACR") + + app_arguments = ("--resource-group", resource_group, "--name", app_name) + show = ("containerapp", "show", *app_arguments) + resource_group_id = f"/subscriptions/{subscription}/resourceGroups/{resource_group}" + environment_id = f"{resource_group_id}/providers/Microsoft.App/managedEnvironments/{app_name}-env" + if _value(*show, query="properties.managedEnvironmentId").casefold() != environment_id.casefold(): + raise ValueError("Existing app does not belong to the expected environment") + if _value(*show, query="properties.configuration.activeRevisionsMode") != "Single": + raise ValueError("Code-only deployment requires the existing single-revision topology") + if ( + _value(*show, query="length(properties.template.containers)") != "1" + or _value(*show, query="properties.template.containers[0].name") != "pyrit-gui" + ): + raise ValueError("Code-only deployment requires the existing pyrit-gui container") + url, access = _health_url(app_arguments=app_arguments, environment_id=environment_id, app_name=app_name) + previous_image = _value(*show, query="properties.template.containers[0].image") + print(f"Code-only {slot} deployment; ACA public access: {access}; verification URL: {url}", flush=True) + print(f"Previous image (not automatically restored): {previous_image}", flush=True) + if previous_image != image: + _az( + "containerapp", + "update", + *app_arguments, + "--container-name", + "pyrit-gui", + "--image", + image, + "--output", + "none", + ) + revision = _value(*show, query="properties.latestRevisionName") + _wait_for_revision(app_arguments=app_arguments, revision=revision, image=image) + _wait_for_http_health(url) + if ( + _value(*show, query="properties.latestRevisionName") != revision + or _value(*show, query="properties.latestReadyRevisionName") != revision + or _value(*show, query="properties.template.containers[0].image") != image + ): + raise RuntimeError("The verified image revision is not the current ready revision") + final_url, final_access = _health_url(app_arguments=app_arguments, environment_id=environment_id, app_name=app_name) + if (final_url, final_access) != (url, access): + raise RuntimeError("The application's access mode changed during code-only deployment") + print(f"Code deployment healthy: {revision}; networking unchanged; verified {url}", flush=True) + + +def main() -> int: + """Read pipeline inputs and fail without rolling back infrastructure.""" + inputs = { + "slot": "PYRIT_SLOT", + "resource_group": "PYRIT_DEPLOYMENT_RESOURCE_GROUP", + "app_name": "PYRIT_APP_NAME", + "acr_resource_id": "PYRIT_ACR_RESOURCE_ID", + "image": "PYRIT_CONTAINER_IMAGE", + } + try: + values = {} + for name, variable in inputs.items(): + value = os.environ.get(variable, "") + if not value or value.startswith("$("): + raise ValueError(f"Required deployment value is missing or unresolved: {variable}") + values[name] = value + deploy_code(**values) + except (ValueError, RuntimeError, OSError, subprocess.CalledProcessError) as error: + print(f"##vso[task.logissue type=error]Code deployment failed: {error}", file=sys.stderr) + if isinstance(error, subprocess.CalledProcessError) and error.stderr: + print(error.stderr, file=sys.stderr) + print("No automatic image, database, or infrastructure rollback was attempted.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/pipelines/deploy_public_nat.sh b/infra/pipelines/deploy_public_nat.sh index db577aaa3e..a286b6240d 100644 --- a/infra/pipelines/deploy_public_nat.sh +++ b/infra/pipelines/deploy_public_nat.sh @@ -15,7 +15,6 @@ required_variables=( PYRIT_AGENT_TEMP_DIRECTORY PYRIT_DEPLOYMENT_RESOURCE_GROUP PYRIT_APP_NAME - PYRIT_CONTAINER_IMAGE PYRIT_VNET_ADDRESS_PREFIX PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX PYRIT_MANAGED_IDENTITY_RESOURCE_ID @@ -195,7 +194,7 @@ normalized_expected_pip_id=$(lowercase "$expected_pip_id") existing_app=$(az containerapp show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME" \ - --query '{id:id,environmentId:properties.managedEnvironmentId,tags:tags}' -o json 2>/dev/null || true) + --query '{id:id,environmentId:properties.managedEnvironmentId,tags:tags,containers:properties.template.containers[].{name:name,image:image}}' -o json 2>/dev/null || true) existing_environment=$(az containerapp env show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-env" \ @@ -259,23 +258,30 @@ if [[ "$deployment_tags" == *'<'* || "$deployment_tags" == "null" \ exit 1 fi -if [[ ! "$PYRIT_CONTAINER_IMAGE" =~ ^([^/]+)/(.+)@(sha256:[0-9a-fA-F]{64})$ ]]; then - echo "##vso[task.logissue type=error]Built image must be an immutable registry digest" +if [[ "$(jq '.containers | length' <<< "$existing_app")" != "1" \ + || "$(jq -r '.containers[0].name' <<< "$existing_app")" != "pyrit-gui" ]]; then + echo "##vso[task.logissue type=error]Infrastructure deployment requires the existing pyrit-gui container" + exit 1 +fi +current_image=$(jq -r '.containers[0].image // empty' <<< "$existing_app") +if [[ ! "$current_image" =~ ^([^/]+)/(.+)@(sha256:[0-9a-fA-F]{64})$ ]]; then + echo "##vso[task.logissue type=error]Current image must be an immutable registry digest" exit 1 fi registry_server=${BASH_REMATCH[1]} repository=${BASH_REMATCH[2]} digest=${BASH_REMATCH[3]} if [[ "$registry_server" != "$acr_name.azurecr.io" ]]; then - echo "##vso[task.logissue type=error]Built image registry does not match ACR resource ID" + echo "##vso[task.logissue type=error]Current image registry does not match ACR resource ID" exit 1 fi repository_pattern='^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*$' if [[ ! "$repository" =~ $repository_pattern ]]; then - echo "##vso[task.logissue type=error]Built image repository is invalid" + echo "##vso[task.logissue type=error]Current image repository is invalid" exit 1 fi immutable_image="$registry_server/$repository@$digest" +echo "Infrastructure-only deployment; retaining current image: $immutable_image" private_link_request_message="Azure Front Door private access to $PYRIT_APP_NAME" parameters=( diff --git a/tests/unit/infra/test_code_deployment.py b/tests/unit/infra/test_code_deployment.py new file mode 100644 index 0000000000..283b6892b0 --- /dev/null +++ b/tests/unit/infra/test_code_deployment.py @@ -0,0 +1,204 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +"""Exercise code-only deployment with no Azure or network calls.""" + +import os +import subprocess +import unittest +from unittest.mock import patch + +from infra.pipelines import deploy_code + +SUBSCRIPTION = "11111111-1111-1111-1111-111111111111" +ENVIRONMENT = ( + f"/subscriptions/{SUBSCRIPTION}/resourceGroups/copyrit-test" + "/providers/Microsoft.App/managedEnvironments/copyrit-test-env" +) +IMAGE = f"copyritacr.azurecr.io/pyrit@sha256:{'a' * 64}" +PREVIOUS_IMAGE = f"copyritacr.azurecr.io/pyrit@sha256:{'b' * 64}" +ACA_HOST = "copyrit-test.example.westus2.azurecontainerapps.io" +AFD_HOST = "copyrit-test.example.azurefd.net" +REVISION = "copyrit-test--0000002" + + +class TestCodeDeployment(unittest.TestCase): + def setUp(self) -> None: + self.arguments = { + "slot": "test", + "resource_group": "copyrit-test", + "app_name": "copyrit-test", + "acr_resource_id": ( + f"/subscriptions/{SUBSCRIPTION}/resourceGroups/shared" + "/providers/Microsoft.ContainerRegistry/registries/copyritacr" + ), + "image": IMAGE, + } + self.values = { + "id": SUBSCRIPTION, + "properties.managedEnvironmentId": ENVIRONMENT, + "properties.configuration.activeRevisionsMode": "Single", + "length(properties.template.containers)": "1", + "properties.template.containers[0].name": "pyrit-gui", + "properties.configuration.ingress.fqdn": ACA_HOST, + "properties.publicNetworkAccess": "Enabled", + "properties.latestRevisionName": REVISION, + "properties.latestReadyRevisionName": REVISION, + "properties.healthState": "Healthy", + "value[?properties.enabledState=='Enabled'].properties.hostName": AFD_HOST, + } + self.current_image = PREVIOUS_IMAGE + self.calls: list[tuple[str, ...]] = [] + self.az_patch = patch.object(deploy_code, "_az", side_effect=self._az) + self.az_patch.start() + self.addCleanup(self.az_patch.stop) + self.http = patch.object(deploy_code, "_wait_for_http_health").start() + self.addCleanup(patch.stopall) + + def _az(self, *arguments: str) -> str: + self.calls.append(arguments) + if arguments[:2] == ("containerapp", "update"): + self.current_image = arguments[arguments.index("--image") + 1] + return "" + query = arguments[arguments.index("--query") + 1] + if query == "properties.template.containers[0].image": + return self.current_image + return self.values[query] + + def test_public_mode_updates_only_image_and_checks_aca(self) -> None: + deploy_code.deploy_code(**self.arguments) + + updates = [call for call in self.calls if call[:2] == ("containerapp", "update")] + assert updates == [ + ( + "containerapp", + "update", + "--resource-group", + "copyrit-test", + "--name", + "copyrit-test", + "--container-name", + "pyrit-gui", + "--image", + IMAGE, + "--output", + "none", + ) + ] + self.http.assert_called_once_with(f"https://{ACA_HOST}/api/health") + assert not any(call[0] in {"deployment", "network", "rest"} for call in self.calls) + + def test_private_mode_checks_front_door_without_changing_network(self) -> None: + self.values["properties.publicNetworkAccess"] = "Disabled" + self.arguments["slot"] = "prod" + + deploy_code.deploy_code(**self.arguments) + + self.http.assert_called_once_with(f"https://{AFD_HOST}/api/health") + rest_calls = [call for call in self.calls if call[0] == "rest"] + assert len(rest_calls) == 2 + assert all(call[1:3] == ("--method", "get") for call in rest_calls) + assert not any(call[0] in {"deployment", "network"} for call in self.calls) + + def test_same_image_still_verified_without_update(self) -> None: + self.current_image = IMAGE + + deploy_code.deploy_code(**self.arguments) + + assert not any(call[:2] == ("containerapp", "update") for call in self.calls) + self.http.assert_called_once() + + def test_invalid_inputs_fail_before_azure_calls(self) -> None: + invalid = { + "slot": "other", + "resource_group": "../wrong", + "app_name": "wrong/name", + "acr_resource_id": "/subscriptions/wrong", + "image": "another.azurecr.io/pyrit:latest", + } + for key, value in invalid.items(): + with self.subTest(key=key), self.assertRaises(ValueError): + deploy_code.deploy_code(**(self.arguments | {key: value})) + assert not self.calls + + def test_invalid_existing_topology_never_updates(self) -> None: + invalid = { + "id": "22222222-2222-2222-2222-222222222222", + "properties.managedEnvironmentId": ENVIRONMENT + "-other", + "properties.configuration.activeRevisionsMode": "Multiple", + "length(properties.template.containers)": "2", + "properties.template.containers[0].name": "other", + "properties.publicNetworkAccess": "unexpected", + "properties.configuration.ingress.fqdn": "attacker.example", + } + for query, value in invalid.items(): + with self.subTest(query=query), patch.dict(self.values, {query: value}), self.assertRaises(ValueError): + deploy_code.deploy_code(**self.arguments) + assert not any(call[:2] == ("containerapp", "update") for call in self.calls) + + def test_private_mode_requires_one_valid_front_door_endpoint(self) -> None: + self.values["properties.publicNetworkAccess"] = "Disabled" + query = "value[?properties.enabledState=='Enabled'].properties.hostName" + for value in ("", "None", f"{AFD_HOST}\nother.azurefd.net", "attacker.example"): + with self.subTest(value=value), patch.dict(self.values, {query: value}), self.assertRaises(ValueError): + deploy_code.deploy_code(**self.arguments) + assert not any(call[:2] == ("containerapp", "update") for call in self.calls) + + def test_unhealthy_revision_fails_without_rollback(self) -> None: + self.values["properties.healthState"] = "Unhealthy" + with patch.object(deploy_code.time, "sleep"), self.assertRaisesRegex(RuntimeError, "revision"): + deploy_code.deploy_code(**self.arguments) + assert self.current_image == IMAGE + assert len([call for call in self.calls if call[:2] == ("containerapp", "update")]) == 1 + self.http.assert_not_called() + assert not any(call[0] in {"deployment", "network", "rest"} for call in self.calls) + + def test_http_failure_does_not_fallback_or_roll_back(self) -> None: + self.values["properties.publicNetworkAccess"] = "Disabled" + self.http.side_effect = RuntimeError("Health check failed") + with self.assertRaisesRegex(RuntimeError, "Health check"): + deploy_code.deploy_code(**self.arguments) + self.http.assert_called_once_with(f"https://{AFD_HOST}/api/health") + assert self.current_image == IMAGE + assert len([call for call in self.calls if call[:2] == ("containerapp", "update")]) == 1 + + def test_current_ready_revision_must_be_the_verified_revision(self) -> None: + self.values["properties.latestReadyRevisionName"] = "copyrit-test--old" + with self.assertRaisesRegex(RuntimeError, "current ready revision"): + deploy_code.deploy_code(**self.arguments) + + def test_access_mode_change_is_not_reported_as_success(self) -> None: + self.http.side_effect = lambda _: self.values.update({"properties.publicNetworkAccess": "Disabled"}) + with self.assertRaisesRegex(RuntimeError, "access mode changed"): + deploy_code.deploy_code(**self.arguments) + + def test_main_reports_unresolved_input_without_azure_calls(self) -> None: + with patch.dict(os.environ, {"PYRIT_SLOT": "$(slot)"}): + assert deploy_code.main() == 1 + assert not self.calls + + +class TestHttpVerification(unittest.TestCase): + def test_http_success_does_not_follow_redirects(self) -> None: + response = subprocess.CompletedProcess(args=[], returncode=0, stdout="200", stderr="") + with patch.object(deploy_code.subprocess, "run", return_value=response) as run: + deploy_code._wait_for_http_health("https://example.azurefd.net/api/health") + arguments = run.call_args.args[0] + assert "--location" not in arguments + assert "--insecure" not in arguments + assert "--max-time" in arguments + + def test_http_errors_and_redirects_exhaust_the_bounded_budget(self) -> None: + for status, exit_code in (("302", 0), ("504", 0), ("200", 28)): + response = subprocess.CompletedProcess(args=[], returncode=exit_code, stdout=status, stderr="failure") + with ( + self.subTest(status=status, exit_code=exit_code), + patch.object(deploy_code.subprocess, "run", return_value=response), + patch.object(deploy_code.time, "monotonic", side_effect=[0, 290, 299, 301]), + patch.object(deploy_code.time, "sleep"), + self.assertRaisesRegex(RuntimeError, "health check failed"), + ): + deploy_code._wait_for_http_health("https://example.azurefd.net/api/health") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/infra/test_pipeline_guardrails.py b/tests/unit/infra/test_pipeline_guardrails.py index c124cae9c8..0ddb9c740c 100644 --- a/tests/unit/infra/test_pipeline_guardrails.py +++ b/tests/unit/infra/test_pipeline_guardrails.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Guard the single-topology Azure DevOps deployment contract.""" +"""Guard the independent application and infrastructure deployment contract.""" import json import os @@ -12,9 +12,12 @@ import unittest from pathlib import Path +import yaml + REPO_ROOT = Path(__file__).resolve().parents[3] PIPELINE = REPO_ROOT / "gui-deploy.yml" DEPLOY_SCRIPT = REPO_ROOT / "infra" / "pipelines" / "deploy_public_nat.sh" +INFRA_TEMPLATE = REPO_ROOT / "infra" / "pipelines" / "deploy-infra.yml" WHAT_IF_VALIDATOR = REPO_ROOT / "infra" / "pipelines" / "validate_what_if.py" EXAMPLE_PARAMETERS = REPO_ROOT / "infra" / "parameters.example.json" DEMO_PARAMETERS = REPO_ROOT / "infra" / "parameters.demo.json" @@ -50,6 +53,7 @@ class TestPipelineGuardrails(unittest.TestCase): def setUpClass(cls): cls.pipeline = PIPELINE.read_text(encoding="utf-8") cls.deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + cls.infra_template = INFRA_TEMPLATE.read_text(encoding="utf-8") def test_pipeline_has_one_test_and_prod_workflow(self): assert "deploymentTarget" not in self.pipeline @@ -60,7 +64,39 @@ def test_pipeline_has_one_test_and_prod_workflow(self): assert "stage: DeployProd" in self.pipeline assert "DeployReplacement" not in self.pipeline assert self.pipeline.count("timeoutInMinutes: 120") == 2 - assert self.pipeline.count("scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh'") == 2 + assert ( + self.pipeline.count('inlineScript: python3 "$(Build.SourcesDirectory)/infra/pipelines/deploy_code.py"') == 2 + ) + assert "deploy_public_nat.sh" not in self.pipeline + assert "scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh'" in self.infra_template + + def test_infrastructure_is_explicit_and_runs_before_code(self) -> None: + pipeline = yaml.safe_load(self.pipeline) + parameters = {parameter["name"]: parameter for parameter in pipeline["parameters"]} + assert parameters["deployInfra"]["default"] is False + assert parameters["deployToProd"]["default"] is False + stages = pipeline["stages"] + conditional = "${{ if eq(parameters.deployInfra, true) }}" + infrastructure = [stage[conditional][0] for stage in stages if conditional in stage] + assert [stage["parameters"] for stage in infrastructure] == [ + {"stageName": "DeployTestInfra", "dependsOn": "Build", "slot": "test"}, + {"stageName": "DeployProdInfra", "dependsOn": "ApproveProd", "slot": "prod"}, + ] + assert all(stage["template"] == "infra/pipelines/deploy-infra.yml" for stage in infrastructure) + code_stages = {stage["stage"]: stage for stage in stages if stage.get("stage") in {"DeployTest", "DeployProd"}} + assert code_stages["DeployTest"]["dependsOn"] == ["Build", {conditional: ["DeployTestInfra"]}] + assert code_stages["DeployProd"]["dependsOn"] == [ + "ApproveProd", + "Build", + {conditional: ["DeployProdInfra"]}, + ] + template = yaml.safe_load(self.infra_template)["stages"][0] + assert template["dependsOn"] == "${{ parameters.dependsOn }}" + assert "condition" not in template # Infrastructure must not run after a skipped approval. + assert "PYRIT_CONTAINER_IMAGE" not in self.infra_template + assert "current_image=$(jq" in self.deploy_script + assert "retaining current image" in self.deploy_script + assert "PYRIT_CONTAINER_IMAGE" not in self.deploy_script def test_production_remains_opt_in_and_independently_approved(self): assert "job: ValidateProdConfiguration" in self.pipeline @@ -79,7 +115,7 @@ def test_production_remains_opt_in_and_independently_approved(self): assert '"$BUILD_SOURCEBRANCH" != refs/heads/main' in self.pipeline assert "eq(variables['Build.SourceBranch'], 'refs/heads/main')" in self.pipeline assert "refs/heads/releases/" not in self.pipeline - assert "condition: succeeded('ApproveProd')" in self.pipeline + assert "condition: and(succeeded(), succeeded('ApproveProd'))" in self.pipeline def test_deploy_resolves_digest_and_previews_before_apply(self): assert "name: BuildImage" in self.pipeline @@ -103,9 +139,9 @@ def test_deploy_resolves_digest_and_previews_before_apply(self): assert '"disableContainerAppsPublicAccess=true"' in self.deploy_script def test_pipeline_passes_values_via_environment(self): - deploy_yaml = self.pipeline[self.pipeline.index("stage: DeployTest") :] + deploy_yaml = self.infra_template assert "PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup)" in deploy_yaml - assert "PYRIT_CONTAINER_IMAGE: $(immutableImage)" in deploy_yaml + assert "PYRIT_CONTAINER_IMAGE: $(immutableImage)" in self.pipeline assert "PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr)" in deploy_yaml assert "PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId)" in deploy_yaml assert "PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId)" in deploy_yaml From b0ed4902ffa6e6ab0de05df4a6201bf764b33c08 Mon Sep 17 00:00:00 2001 From: Adrian Gavrila Date: Tue, 15 Sep 2026 09:12:17 -0400 Subject: [PATCH 2/3] Simplify the infrastructure deployment label Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- gui-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui-deploy.yml b/gui-deploy.yml index c9cbd899ae..3325beea3b 100644 --- a/gui-deploy.yml +++ b/gui-deploy.yml @@ -18,7 +18,7 @@ pr: none parameters: - name: deployInfra - displayName: 'Deploy infrastructure before code' + displayName: 'Deploy Infrastructure' type: boolean default: false - name: deployToProd From 2959ceb616abef6703212c9838c449566aa17ec3 Mon Sep 17 00:00:00 2001 From: Adrian Gavrila Date: Tue, 15 Sep 2026 20:57:23 -0400 Subject: [PATCH 3/3] Keep infrastructure stages separate from app deployment Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- gui-deploy.yml | 103 ++- infra/README.md | 48 +- infra/main.bicep | 101 ++- infra/pipelines/deploy-infra.yml | 9 +- infra/pipelines/deploy_code.py | 192 ------ .../{deploy_public_nat.sh => deploy_gui.sh} | 424 +++++++----- tests/unit/infra/test_bicep_topology.py | 44 +- tests/unit/infra/test_code_deployment.py | 604 +++++++++++++----- tests/unit/infra/test_pipeline_guardrails.py | 216 +++++-- 9 files changed, 1089 insertions(+), 652 deletions(-) delete mode 100644 infra/pipelines/deploy_code.py rename infra/pipelines/{deploy_public_nat.sh => deploy_gui.sh} (56%) diff --git a/gui-deploy.yml b/gui-deploy.yml index 3325beea3b..d61f3bc913 100644 --- a/gui-deploy.yml +++ b/gui-deploy.yml @@ -1,7 +1,7 @@ # CI/CD pipeline for the CoPyRIT GUI. # -# Code-only test deployment is the default. Infrastructure reconciliation is -# explicit and runs with the current image before deploying new code. +# App-only test deployment is the default. Infrastructure reconciliation is +# explicit and leaves the running app unchanged before deploying new code. trigger: branches: @@ -125,19 +125,24 @@ stages: immutable_image="$PYRIT_ACR_LOGIN_SERVER/$PYRIT_IMAGE_NAME@$digest" echo "##vso[task.setvariable variable=immutableImage;isOutput=true]$immutable_image" - - ${{ if eq(parameters.deployInfra, true) }}: - - template: infra/pipelines/deploy-infra.yml - parameters: - stageName: DeployTestInfra - dependsOn: Build - slot: test + - template: infra/pipelines/deploy-infra.yml + parameters: + stageName: DeployTestInfra + dependsOn: Build + slot: test + deployInfra: ${{ parameters.deployInfra }} - stage: DeployTest - displayName: 'Deploy code to Test' + displayName: 'Deploy app to Test' dependsOn: - Build - - ${{ if eq(parameters.deployInfra, true) }}: - - DeployTestInfra + - DeployTestInfra + condition: >- + and(not(canceled()), + in(dependencies.Build.result, 'Succeeded', 'SucceededWithIssues'), + or(in(dependencies.DeployTestInfra.result, 'Succeeded', 'SucceededWithIssues'), + and(eq('${{ parameters.deployInfra }}', 'false'), + eq(dependencies.DeployTestInfra.result, 'Skipped')))) variables: - group: copyrit-gui-common - group: copyrit-gui-test @@ -158,18 +163,36 @@ stages: fetchDepth: 1 - task: AzureCLI@2 - displayName: 'Deploy and verify test code without changing networking' + displayName: 'Preview, deploy, and verify test application' env: PYRIT_SLOT: test + PYRIT_DEPLOY_INFRA: 'false' + PYRIT_BUILD_ID: $(Build.BuildId) + PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) + PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) PYRIT_APP_NAME: $(deploymentAppName) PYRIT_CONTAINER_IMAGE: $(immutableImage) + PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) + PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) + PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) + PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) + PYRIT_ENTRA_TENANT_ID: $(entraTenantId) + PYRIT_ENTRA_CLIENT_ID: $(entraClientId) + PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) + PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId) + PYRIT_CONFIG_FILE_URI: $(pyritConfigFileUri) + PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) + PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) + PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) PYRIT_ACR_RESOURCE_ID: $(acrResourceId) + PYRIT_ENABLE_OTEL: $(enableOtel) + PYRIT_ENV_SECRET_NAME: $(envSecretName) inputs: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: python3 "$(Build.SourcesDirectory)/infra/pipelines/deploy_code.py" + scriptLocation: 'scriptPath' + scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_gui.sh' - stage: ApproveProd displayName: 'Approve Production Deployment' @@ -210,26 +233,32 @@ stages: Confirm test is healthy, Entra and backend group authorization work, and the static test egress IP is allow-listed. Infrastructure deployment requested: ${{ parameters.deployInfra }}. - Code-only deployment preserves the existing public/private access mode. + App deployment reconciles application configuration and preserves + the existing public/private access mode. If infrastructure is requested, approve the production Private Link cutover and configuration changes before the new code is deployed. onTimeout: reject - - ${{ if eq(parameters.deployInfra, true) }}: - - template: infra/pipelines/deploy-infra.yml - parameters: - stageName: DeployProdInfra - dependsOn: ApproveProd - slot: prod + - template: infra/pipelines/deploy-infra.yml + parameters: + stageName: DeployProdInfra + dependsOn: ApproveProd + slot: prod + deployInfra: ${{ parameters.deployInfra }} - stage: DeployProd - displayName: 'Deploy code to Production' + displayName: 'Deploy app to Production' dependsOn: - ApproveProd - Build - - ${{ if eq(parameters.deployInfra, true) }}: - - DeployProdInfra - condition: and(succeeded(), succeeded('ApproveProd')) + - DeployProdInfra + condition: >- + and(not(canceled()), + in(dependencies.Build.result, 'Succeeded', 'SucceededWithIssues'), + in(dependencies.ApproveProd.result, 'Succeeded', 'SucceededWithIssues'), + or(in(dependencies.DeployProdInfra.result, 'Succeeded', 'SucceededWithIssues'), + and(eq('${{ parameters.deployInfra }}', 'false'), + eq(dependencies.DeployProdInfra.result, 'Skipped')))) variables: - group: copyrit-gui-common - group: copyrit-gui-prod @@ -250,15 +279,33 @@ stages: fetchDepth: 1 - task: AzureCLI@2 - displayName: 'Deploy and verify production code without changing networking' + displayName: 'Preview, deploy, and verify production application' env: PYRIT_SLOT: prod + PYRIT_DEPLOY_INFRA: 'false' + PYRIT_BUILD_ID: $(Build.BuildId) + PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) + PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) PYRIT_APP_NAME: $(deploymentAppName) PYRIT_CONTAINER_IMAGE: $(immutableImage) + PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) + PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) + PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) + PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) + PYRIT_ENTRA_TENANT_ID: $(entraTenantId) + PYRIT_ENTRA_CLIENT_ID: $(entraClientId) + PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) + PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId) + PYRIT_CONFIG_FILE_URI: $(pyritConfigFileUri) + PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) + PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) + PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) PYRIT_ACR_RESOURCE_ID: $(acrResourceId) + PYRIT_ENABLE_OTEL: $(enableOtel) + PYRIT_ENV_SECRET_NAME: $(envSecretName) inputs: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: python3 "$(Build.SourcesDirectory)/infra/pipelines/deploy_code.py" + scriptLocation: 'scriptPath' + scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_gui.sh' diff --git a/infra/README.md b/infra/README.md index abccff4ec0..76d7ee5ee4 100644 --- a/infra/README.md +++ b/infra/README.md @@ -72,7 +72,7 @@ flowchart TB app -.->|"Traces after agent setup"| appInsights ``` -The base topology is public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. `enableFrontDoor=true` adds Front Door Premium as the preferred managed HTTPS URL. By default, the ACA origin remains concurrently public and can bypass Front Door. `enableFrontDoorPrivateLink=true` instead connects Premium Front Door to the ACA environment through Private Link; setting `disableContainerAppsPublicAccess=true` then removes the direct public ACA path. Bicep rejects public-access shutdown unless both Front Door and its Private Link origin are enabled. The team ADO infrastructure stage uses this isolated-origin mode; code-only runs preserve the existing access mode. Community examples leave all three Front Door settings disabled. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door rather than the original client. Front Door changes inbound routing only: outbound connections from ACA continue to use the NAT Gateway's static IPv4. +The base topology is public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. `enableFrontDoor=true` adds Front Door Premium as the preferred managed HTTPS URL. By default, the ACA origin remains concurrently public and can bypass Front Door. `enableFrontDoorPrivateLink=true` instead connects Premium Front Door to the ACA environment through Private Link; setting `disableContainerAppsPublicAccess=true` then removes the direct public ACA path. Bicep rejects public-access shutdown unless both Front Door and its Private Link origin are enabled. The team ADO infrastructure stage uses this isolated-origin mode; app-only runs preserve the existing access mode. Community examples leave all three Front Door settings disabled. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door rather than the original client. Front Door changes inbound routing only: outbound connections from ACA continue to use the NAT Gateway's static IPv4. ## Development Workflow @@ -105,7 +105,7 @@ Community users can deploy `main.bicep` directly using the instructions below. F - **Authentication**: [MSAL](https://learn.microsoft.com/en-us/entra/msal/) [PKCE](https://oauth.net/2/pkce/) on the frontend (`@azure/msal-browser`) and public-client device-code authentication for the PyRIT CLI, backed by Microsoft Graph middleware on the backend. Both clients send delegated Graph tokens, and the backend authenticates them through Graph `/me`. These public-client flows require no client secrets or certificates. - **Authorization**: Entra group checks use `allowedGroupObjectIds` for application access and `adminGroupObjectId` for backend configuration routes. Requires delegated Graph `User.Read`; the backend calls `/me/checkMemberGroups` and compares the returned transitive memberships with the configured group IDs. Each security group must also be assigned to the enterprise app (see Prerequisites ยง3). Authenticated deployments require at least one allowed group and fail to start without one. `/api/health`, `/api/auth/config`, and `/api/media` are intentional public exceptions; other `/api` routes require authentication when auth is enabled. Successful identity and membership results are cached in-process for 60 seconds, keyed by a SHA-256 token digest, to reduce Graph latency and throttling. Bearer tokens themselves are not stored in the cache. - **Identity**: `deploy_instance.py` creates its user-assigned managed identity (UAMI) and grants AcrPull and Storage Blob Data Contributor before deploying Bicep. A direct Bicep deployment can create `-identity`, but the template creates no role assignments, so its first revision can remain unhealthy until required roles are granted and the revision is restarted. A healthy one-pass direct deployment uses an existing, pre-authorized UAMI. `AZURE_CLIENT_ID` is set to the UAMI's client ID so `DefaultAzureCredential` selects the correct identity. -- **Network**: The template always creates a VNet-integrated external Container Apps environment, one delegated ACA infrastructure subnet, a Standard NAT Gateway, and a static outbound IPv4. ACA supplies the generated HTTPS hostname and trusted certificate. In direct-ACA mode, `allowedCidr` optionally restricts public ingress to one IPv4 CIDR; an empty value permits public ingress. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door backend addresses, not the original client; Bicep and the team pipeline reject the invalid combination. Entra sign-in, enterprise-app assignment, and backend group checks remain mandatory application access controls. +- **Network**: With the default `deployInfra=true`, the template creates a VNet-integrated external Container Apps environment, one delegated ACA infrastructure subnet, a Standard NAT Gateway, and a static outbound IPv4. ACA supplies the generated HTTPS hostname and trusted certificate. In direct-ACA mode, `allowedCidr` optionally restricts public ingress to one IPv4 CIDR; an empty value permits public ingress. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door backend addresses, not the original client; Bicep and the team pipeline reject the invalid combination. Entra sign-in, enterprise-app assignment, and backend group checks remain mandatory application access controls. - **Front Door**: `enableFrontDoor=true` creates a Premium profile, managed `azurefd.net` endpoint, HTTPS ACA origin, `/api/health` probe, uncached catch-all route, and 240-second origin response timeout matching the ACA HTTP ingress limit. `enableFrontDoorPrivateLink=true` targets the ACA managed environment with group ID `managedEnvironments`. The resulting private endpoint connection must be approved before AFD can route privately. `disableContainerAppsPublicAccess=true` disables the ACA environment public endpoint and CORS then permits only the AFD origin. The module does not create a WAF policy; application authentication and authorization remain mandatory. - **Routing**: Inbound requests through Front Door do not traverse the NAT Gateway. When ACA public access remains enabled, users can also reach ACA directly. When Private Link is enabled and public access is disabled, all public application traffic enters through Front Door. Outbound connections from the ACA environment that leave the virtual network use the NAT Gateway's static public IPv4. - **Response headers**: `SecurityHeadersMiddleware` adds [CSP](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP), HTTP Strict Transport Security (HSTS, production only), X-Frame-Options, X-Content-Type-Options, Referrer-Policy, Permissions-Policy, and Cache-Control (`no-store` on API routes). Swagger/OpenAPI disabled in production. @@ -407,35 +407,41 @@ az deployment group show -g -n \ | `deployInfra` | `deployToProd` | Workflow | | --- | --- | --- | -| `false` | `false` | Build, deploy code to test | -| `true` | `false` | Build, deploy test infrastructure with the current image, deploy code to test | -| `false` | `true` | Build, deploy code to test, production approval, deploy the same image to production | -| `true` | `true` | Build, test infrastructure, test code, production approval, production infrastructure, production code | +| `false` | `false` | Build, deploy app to test | +| `true` | `false` | Build, reconcile test infrastructure without changing the app, deploy app to test | +| `false` | `true` | Build, deploy app to test, production approval, deploy the same image to production | +| `true` | `true` | Build, test infrastructure, test app, production approval, production infrastructure, production app | -Qualifying merges to `main` automatically deploy **code only to test**. Production remains opt-in: manually queue a commit merged to `main` with `deployToProd=true`. Approval rejects on timeout and the requester cannot self-approve. An infrastructure failure blocks the corresponding code stage; a code failure never invokes infrastructure rollback. +Qualifying merges to `main` automatically deploy **the app to test without infrastructure reconciliation**. Production remains opt-in: manually queue a commit merged to `main` with `deployToProd=true`. Approval rejects on timeout and the requester cannot self-approve. All stages remain in the run graph: infrastructure stages show **Skipped** when `deployInfra=false`, just as production stages do when production is disabled. App deployment accepts that intentional skip, but an infrastructure failure or cancellation blocks it. -#### Code-only deployment +#### Shared application deployment -`infra/pipelines/deploy_code.py` updates only the existing `pyrit-gui` container image. It validates the subscription, registry digest, environment, and single-container/single-revision topology before writing. Application configuration, identity, networking, NAT, and Front Door are not redeployed. +Both stages use `infra/pipelines/deploy_gui.sh` through the existing `AzureCLI@2` Bash `scriptPath` mechanism. The infrastructure stage passes `deployInfra=true, deployApp=false` to `main.bicep`; the app stage passes `deployInfra=false, deployApp=true`. Only the app stage applies the Container App definition, once per environment. There is no separate image-update implementation. -The script verifies the exact new revision and its existing access mode: direct ACA `/api/health` when public access is enabled, or Front Door `/api/health` when public access is disabled. It reports the mode explicitly and never falls back from a failed private path to public access. A public-mode success is **not** certification of Private Link readiness. The data-plane check has a five-minute budget after revision readiness. +**App-only is not image-only:** it reconciles the image and Bicep-defined application configuration, including environment variables, identity attachment, ingress, and scaling. It requires the same variable groups and existing topology as infrastructure-enabled deployment. Shared infrastructure resources are referenced rather than redeployed, and the app-only preview rejects writes outside the existing Container App. The existing ACA environment public/private access mode and Front Door resources are preserved. -Code deployment does not downgrade or create a database; application startup still follows the image's normal migration behavior. Failures are reported without automatic image or database rollback because migrations may make the previous image incompatible. The previous image digest is logged for an explicit recovery decision. +The script verifies the exact requested revision and its access mode: direct ACA `/api/health` when public access is enabled, or Front Door `/api/health` when public access is disabled. It never falls back from a failed private path to public access. A public-mode success is **not** certification of Private Link readiness. App-only data-plane verification has a five-minute budget after revision readiness. + +App deployment does not downgrade or create a database; application startup still follows the image's normal migration behavior. App-stage failures do not invoke infrastructure, image, or database rollback because migrations may make the previous image incompatible. The previous image digest is logged for an explicit recovery decision. + +Direct community deployments keep their existing behavior: `main.bicep` defaults both `deployInfra` and `deployApp` to `true`, and requires `containerImage` when deploying the app. Separate phases use **Incremental** deployment mode so omitted resources are not deleted. The internal app stage requires existing infrastructure, a managed identity, and a registry. #### Optional infrastructure deployment -Set `deployInfra=true` when changing Bicep-managed configuration or networking. Each infrastructure stage runs **before** the new code, using the image already deployed to that environment, not the Build output. Infrastructure rollback therefore cannot install a newly built application that has not passed code verification. +Set `deployInfra=true` when changing shared infrastructure or networking. Each infrastructure stage runs **before** the app deployment, leaving the existing Container App image and settings untouched. It supplies no image to Bicep and rejects previewed writes to the app or its child resources. Its health checks verify routing to the existing image and confirm that the app revision did not change. The following app stage then deploys the built digest and application configuration without further infrastructure reconciliation. + +Front Door routes the GUI and its relative `/api` requests on the same origin. Infrastructure readiness checks therefore do not need to redeploy the app's CORS settings; those settings are reconciled in the app stage. Entra redirect URI registration remains an external prerequisite, and infrastructure health is not a browser sign-in check. Cross-origin clients need the app stage's updated CORS configuration before using a newly introduced origin. -`infra/pipelines/deploy_public_nat.sh` retains the existing infrastructure safeguards: +`infra/pipelines/deploy_gui.sh` retains the existing infrastructure safeguards: 1. Build the source image and push a commit-SHA tag to ACR. -2. Capture the exact pushed digest for the code stages; infrastructure retains the currently deployed digest. +2. Capture the exact pushed digest for the app stages; infrastructure reads the current image only to verify that the running app remains healthy. 3. Require the existing app, environment, VNet, subnet, NAT, and reserved PIP; validate their IDs, prefixes, tags, SKU, allocation, and attachments. 4. Run a full ARM `what-if` through a fail-closed validator; reject malformed results, deletions, cross-resource-group writes, protected-network deltas other than the documented read-only NAT/PIP normalization, and core network, app, or Log Analytics workspace creates. The expected PIP protection lock may be created. 5. Preserve policy-managed PIP tags and deploy with Front Door Private Link, ACA public access disabled, and PIP protection enabled. 6. Validate the AFD origin targets the expected ACA environment, approve only active requests with the deterministic message, and require the ACA-side connection to report `Approved`. AFD can continue to display `Pending` after approval, so successful AFD health is the data-plane readiness signal. -7. Allow up to 30 minutes for Front Door propagation, then verify ACA public access is disabled, the retained-image revision and Front Door `/api/health` are healthy, direct ACA access is unavailable, and the PIP resource ID/address is unchanged. -8. If cutover validation fails, redeploy the prior public AFD origin and re-enable ACA public access; otherwise print the Front Door URL and static egress IPv4. +7. Allow up to 30 minutes for Front Door propagation, then verify ACA public access is disabled, the unchanged app revision and Front Door `/api/health` are healthy, direct ACA access is unavailable, and the PIP resource ID/address is unchanged. +8. If cutover validation fails, restore the public AFD origin and re-enable ACA public access without redeploying the app. Otherwise print the verified Front Door URL and static egress IPv4. App-stage failures do not trigger infrastructure rollback. `copyrit-gui-common` supplies the shared image settings: @@ -462,17 +468,17 @@ Both `copyrit-gui-test` and `copyrit-gui-prod` supply: | `keyVaultResourceId`, `envSecretName` | Existing runtime configuration secret | | `acrResourceId`, `enableOtel` | Registry resource ID and observability setting | -The container image is not a library variable. The Build stage publishes the exact pushed digest as `immutableImage`, and both code stages consume that output. Infrastructure stages discover the existing image directly from ACA. Code-only runs consume only `deploymentResourceGroup`, `deploymentAppName`, and `acrResourceId` from each environment group; changes to the other configuration values require `deployInfra=true`. Do not add the legacy `image`, `resourceGroup`, `appName`, or `enablePrivateEndpoint` variables; the current workflow does not consume them. +The container image is not a library variable. The Build stage publishes the exact pushed digest as `immutableImage`, and both app stages consume that output. Infrastructure stages discover the existing image directly from ACA for health verification only. All deployment stages consume the environment configuration above; app configuration changes do not require `deployInfra=true`, but shared infrastructure changes do. Do not add the legacy `image`, `resourceGroup`, `appName`, or `enablePrivateEndpoint` variables; the current workflow does not consume them. Pipeline definition 139 reads `gui-deploy.yml` from the GitHub commit being queued. Treat YAML and variable-group contract changes as one release: do not remove old keys before the commit that consumes the replacement keys reaches the target branch. Otherwise ADO leaves unresolved `$(name)` text in Bash, where it is interpreted as command substitution. `copyrit-gui-prod` must additionally define `prodApprovers` as the users or ADO groups allowed to approve `ManualValidation@1`. Protect the production variable group with ADO permissions; the approver list is authorization configuration, not a secret. -The resource group, registry, image-pull authorization, managed identity, Key Vault secret and access path, SQL user/roles and network path, and provider permissions must exist before the first pipeline run. The pipeline does not bootstrap those dependencies or update Entra redirect URIs. Setting `enableOtel=true` creates Application Insights and configures the app endpoint, but the managed agent still requires the post-deployment command in Notes. +The resource group, registry, image-pull authorization, managed identity, Key Vault secret and access path, SQL user/roles and network path, and provider permissions must exist before the first pipeline run. The pipeline does not bootstrap those dependencies or update Entra redirect URIs. Setting `enableOtel=true` requires an infrastructure-enabled run to create Application Insights before app-only runs can reference it. The managed agent still requires the post-deployment command in Notes. The internal workflow is update-only for networking: its app name and prefixes must resolve to the existing app/environment/VNet/subnet/NAT/PIP. It records the current PIP resource ID and address before preview, requires protected resources to remain unchanged except Azure read-only normalization, and verifies the same PIP/address after deployment. -The optional infrastructure stage also creates a `CanNotDelete` lock scoped to the reserved PIP. Its validated Front Door origin uses Private Link to the ACA environment, and the ACA public endpoint is disabled after a successful infrastructure deployment. Code-only runs preserve the existing configuration, including an existing public-access fallback. +The optional infrastructure stage also creates a `CanNotDelete` lock scoped to the reserved PIP. Its validated Front Door origin uses Private Link to the ACA environment, and the ACA public endpoint is disabled after a successful infrastructure deployment. App-only runs preserve that environment access mode, including an existing public-access fallback. ## Post-Deployment @@ -625,7 +631,7 @@ Supported Azure integrations, including OpenAI, Content Safety, and Speech, can ## Notes -- **Network topology**: Public ACA-managed HTTPS ingress with optional `allowedCidr` plus VNet-integrated fixed NAT egress is the base topology. Front Door Premium is an optional inbound layer. Private Link plus disabled ACA public access makes Front Door the only public application path. The team ADO infrastructure stage enables this isolated-origin mode; default code-only runs preserve existing networking. `allowedCidr` must be empty when Front Door is enabled; Bicep rejects the combination. +- **Network topology**: Public ACA-managed HTTPS ingress with optional `allowedCidr` plus VNet-integrated fixed NAT egress is the base topology. Front Door Premium is an optional inbound layer. Private Link plus disabled ACA public access makes Front Door the only public application path. The team ADO infrastructure stage enables this isolated-origin mode; default app-only runs preserve shared networking and the environment access mode. `allowedCidr` must be empty when Front Door is enabled; Bicep rejects the combination. - **Ingress vs. egress**: Front Door affects inbound requests only. The reserved NAT public IP remains the source for ACA-originated outbound connections. - **NAT routing**: NAT Gateway supplies the outbound source IP only while the subnet's effective default route remains `Internet`. A UDR or propagated BGP `0.0.0.0/0` route to a firewall or gateway takes precedence; in that topology, allow-list the egress device's public IP instead. - **Network outputs**: `egressPublicIpAddress`, `natGatewayId`, `acaInfrastructureSubnetId`, and `vnetName` describe the created network. @@ -641,7 +647,7 @@ Supported Azure integrations, including OpenAI, Content Safety, and Speech, can az containerapp env telemetry app-insights set \ --name -env -g --connection-string "$AI_CONN" ``` -- **Existing resources**: Log Analytics, ACR, and a UAMI can be supplied as existing resources; Key Vault must be supplied. The template always creates its dedicated VNet, ACA subnet, NAT Gateway, and egress public IP. Although Bicep can declare an ACR when no registry is supplied, a separate bootstrap is required to push the image and authorize its identity before the app can run. +- **Existing resources**: Log Analytics, ACR, and a UAMI can be supplied as existing resources; Key Vault must be supplied. With `deployInfra=true`, the template creates its dedicated VNet, ACA subnet, NAT Gateway, and egress public IP; app-only deployment leaves these existing resources untouched. Although Bicep can declare an ACR when no registry is supplied, a separate bootstrap is required to push the image and authorize its identity before the app can run. - **Azure CLI**: Version 2.84+ required (2.77 has a known bug). ## Teardown and Redeployment diff --git a/infra/main.bicep b/infra/main.bicep index 0e2ee20d63..e4e5964e8b 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -41,9 +41,19 @@ param appName string = 'pyrit-gui' @description('Azure region for all resources') param location string = resourceGroup().location -@description('Container image โ€” must use a unique tag (commit SHA) or digest, never :latest. Enforce in CI pipeline.') +@description('Container image, required when deployApp is true โ€” must use a unique tag (commit SHA) or digest, never :latest. Enforce in CI pipeline.') @metadata({ example: 'myacr.azurecr.io/pyrit:a1b2c3d or myacr.azurecr.io/pyrit@sha256:...' }) -param containerImage string +param containerImage string = '' + +@description('Reconcile shared infrastructure. False requires an existing environment, managed identity, and registry.') +param deployInfra bool = true + +@description('Deploy the Container App image and configuration. False leaves the existing application untouched.') +param deployApp bool = true + +var effectiveContainerImage = deployApp && empty(containerImage) + ? fail('containerImage is required when deployApp is true') + : containerImage @description('Entra ID tenant ID') param entraTenantId string @@ -179,20 +189,26 @@ var effectiveAllowedCidr = enableFrontDoor && !empty(allowedCidr) var effectiveFrontDoorPrivateLink = enableFrontDoorPrivateLink && !enableFrontDoor ? fail('enableFrontDoor must be true when enableFrontDoorPrivateLink is true') : enableFrontDoorPrivateLink -var effectiveContainerAppsPublicAccess = disableContainerAppsPublicAccess - ? (effectiveFrontDoorPrivateLink ? 'Disabled' : fail('Front Door Private Link is required before ACA public access can be disabled')) - : 'Enabled' +var effectiveContainerAppsPublicAccess = deployInfra + ? (disableContainerAppsPublicAccess + ? (effectiveFrontDoorPrivateLink ? 'Disabled' : fail('Front Door Private Link is required before ACA public access can be disabled')) + : 'Enabled') + : existingAcaEnvironment!.properties.publicNetworkAccess var createLogAnalytics = logAnalyticsWorkspaceId == '' var createAcr = acrResourceId == '' && acrName == '' + ? (deployInfra ? true : fail('App-only deployment requires an existing registry')) + : false var useInlineEnvFile = !empty(envFileContents) var createManagedIdentity = empty(existingManagedIdentityResourceId) + ? (deployInfra ? true : fail('App-only deployment requires existingManagedIdentityResourceId')) + : false var generatedAcrName = '${padLeft(replace(appName, '-', ''), 2, 'p')}acr' var existingManagedIdentitySegments = split(existingManagedIdentityResourceId, '/') var existingManagedIdentitySubscriptionId = createManagedIdentity ? subscription().subscriptionId : existingManagedIdentitySegments[2] var existingManagedIdentityResourceGroupName = createManagedIdentity ? resourceGroup().name : existingManagedIdentitySegments[4] var existingManagedIdentityName = createManagedIdentity ? '' : last(existingManagedIdentitySegments) -module acaNatNetwork './modules/aca_nat_network.bicep' = { +module acaNatNetwork './modules/aca_nat_network.bicep' = if (deployInfra) { name: '${appName}-aca-nat-network' params: { namePrefix: appName @@ -232,7 +248,7 @@ var effectiveAcrServer = '${effectiveAcrName}.azurecr.io' // The key is used during deployment for log ingestion config only โ€” it is NOT // injected into the container or accessible to application code. // ============================================================================ -resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = if (createLogAnalytics) { +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = if (deployInfra && createLogAnalytics) { name: '${appName}-logs' location: location tags: tags @@ -250,7 +266,7 @@ var effectiveLogAnalyticsKeyValue = createLogAnalytics ? logAnalytics!.listKeys( // ============================================================================ // Application Insights (created when OTel is enabled โ€” destination for traces/logs) // ============================================================================ -resource appInsights 'Microsoft.Insights/components@2020-02-02' = if (enableOtel) { +resource appInsights 'Microsoft.Insights/components@2020-02-02' = if (deployInfra && enableOtel) { name: '${appName}-ai' location: location tags: tags @@ -312,7 +328,7 @@ var keyVaultName = last(split(keyVaultResourceId, '/')) // OTel: When enableOtel=true, configure the managed OTel agent // as a post-deploy CLI step (2024-03-01 schema does not support it natively). // ============================================================================ -resource acaEnvironment 'Microsoft.App/managedEnvironments@2024-10-02-preview' = { +resource acaEnvironment 'Microsoft.App/managedEnvironments@2024-10-02-preview' = if (deployInfra) { name: '${appName}-env' location: location tags: tags @@ -349,9 +365,16 @@ resource acaEnvironment 'Microsoft.App/managedEnvironments@2024-10-02-preview' = } } -var acaOriginHostName = '${appName}.${acaEnvironment.properties.defaultDomain}' +resource existingAcaEnvironment 'Microsoft.App/managedEnvironments@2024-10-02-preview' existing = if (!deployInfra) { + name: '${appName}-env' +} + +var environmentDefaultDomain = deployInfra + ? acaEnvironment!.properties.defaultDomain + : existingAcaEnvironment!.properties.defaultDomain +var acaOriginHostName = '${appName}.${environmentDefaultDomain}' -module acaFrontDoor './modules/aca_front_door.bicep' = if (enableFrontDoor) { +module acaFrontDoor './modules/aca_front_door.bicep' = if (deployInfra && enableFrontDoor) { name: '${appName}-aca-front-door' params: { namePrefix: appName @@ -364,6 +387,22 @@ module acaFrontDoor './modules/aca_front_door.bicep' = if (enableFrontDoor) { } } +resource existingFrontDoorEndpoint 'Microsoft.Cdn/profiles/afdEndpoints@2024-09-01' existing = if (!deployInfra && enableFrontDoor) { + name: '${appName}-afd/${appName}-${take(uniqueString(subscription().id, resourceGroup().id, appName), 8)}' +} + +var frontDoorHostName = enableFrontDoor + ? (deployInfra ? acaFrontDoor!.outputs.endpointHostName : existingFrontDoorEndpoint!.properties.hostName) + : '' + +resource existingEgressPublicIp 'Microsoft.Network/publicIPAddresses@2024-05-01' existing = if (!deployInfra) { + name: '${appName}-egress-pip' +} + +resource existingAppInsights 'Microsoft.Insights/components@2020-02-02' existing = if (!deployInfra && enableOtel) { + name: '${appName}-ai' +} + // NOTE: When enableOtel=true, configure the OpenTelemetry managed agent on the // environment as a post-deployment step using az CLI: // az containerapp env telemetry app-insights set \ @@ -374,7 +413,7 @@ module acaFrontDoor './modules/aca_front_door.bicep' = if (enableFrontDoor) { // ============================================================================ // Container App โ€” PyRIT GUI // ============================================================================ -resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { +resource containerApp 'Microsoft.App/containerApps@2024-03-01' = if (deployApp) { name: appName location: location tags: tags @@ -437,7 +476,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { containers: [ { name: 'pyrit-gui' - image: containerImage + image: effectiveContainerImage resources: { cpu: json(cpuCores) memory: '${memoryGb}Gi' @@ -517,8 +556,8 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { name: 'PYRIT_CORS_ORIGINS' value: enableFrontDoor ? (effectiveContainerAppsPublicAccess == 'Disabled' - ? 'https://${acaFrontDoor!.outputs.endpointHostName}' - : 'https://${acaOriginHostName},https://${acaFrontDoor!.outputs.endpointHostName}') + ? 'https://${frontDoorHostName}' + : 'https://${acaOriginHostName},https://${frontDoorHostName}') : 'https://${acaOriginHostName}' } ] @@ -532,6 +571,14 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { } } +resource existingContainerApp 'Microsoft.App/containerApps@2024-03-01' existing = if (!deployApp) { + name: appName +} + +var appHostName = deployApp + ? containerApp!.properties.configuration.ingress.fqdn + : existingContainerApp!.properties.configuration.ingress.fqdn + // ============================================================================ // NOTE: Easy Auth (authConfigs) is intentionally NOT used. // The tenant's credential policy blocks client secrets and trusted-CA-only @@ -548,16 +595,16 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { // ============================================================================ @description('The generated ACA FQDN; inaccessible when ACA public network access is disabled') -output appFqdn string = containerApp.properties.configuration.ingress.fqdn +output appFqdn string = appHostName @description('The Azure Front Door managed HTTPS hostname') -output frontDoorFqdn string = enableFrontDoor ? acaFrontDoor!.outputs.endpointHostName : '' +output frontDoorFqdn string = frontDoorHostName @description('The Azure Front Door public URL') -output frontDoorUrl string = enableFrontDoor ? 'https://${acaFrontDoor!.outputs.endpointHostName}' : '' +output frontDoorUrl string = enableFrontDoor ? 'https://${frontDoorHostName}' : '' @description('The deterministic ACA Private Link approval request message; empty when Private Link is disabled') -output frontDoorPrivateLinkRequestMessage string = effectiveFrontDoorPrivateLink +output frontDoorPrivateLinkRequestMessage string = deployInfra && effectiveFrontDoorPrivateLink ? acaFrontDoor!.outputs.privateLinkRequestMessage : '' @@ -565,19 +612,19 @@ output frontDoorPrivateLinkRequestMessage string = effectiveFrontDoorPrivateLink output containerAppsPublicNetworkAccess string = effectiveContainerAppsPublicAccess @description('The public application FQDN selected for this deployment') -output publicFqdn string = enableFrontDoor ? acaFrontDoor!.outputs.endpointHostName : containerApp.properties.configuration.ingress.fqdn +output publicFqdn string = enableFrontDoor ? frontDoorHostName : appHostName @description('The default domain of the ACA environment') -output environmentDefaultDomain string = acaEnvironment.properties.defaultDomain +output environmentDefaultDomain string = environmentDefaultDomain @description('Static outbound IPv4 address') -output egressPublicIpAddress string = acaNatNetwork!.outputs.egressPublicIpAddress +output egressPublicIpAddress string = deployInfra ? acaNatNetwork!.outputs.egressPublicIpAddress : existingEgressPublicIp!.properties.ipAddress @description('NAT Gateway resource ID') -output natGatewayId string = acaNatNetwork!.outputs.natGatewayId +output natGatewayId string = deployInfra ? acaNatNetwork!.outputs.natGatewayId : resourceId('Microsoft.Network/natGateways', '${appName}-nat') @description('ACA infrastructure subnet resource ID') -output acaInfrastructureSubnetId string = acaNatNetwork!.outputs.infrastructureSubnetId +output acaInfrastructureSubnetId string = deployInfra ? acaNatNetwork!.outputs.infrastructureSubnetId : resourceId('Microsoft.Network/virtualNetworks/subnets', '${appName}-vnet', '${appName}-aca-subnet') @description('The principal ID of the user-assigned managed identity โ€” grant this Cognitive Services OpenAI User on your AOAI instances and db_datareader/db_datawriter on Azure SQL') output managedIdentityPrincipalId string = effectiveManagedIdentityPrincipalId @@ -597,7 +644,9 @@ output keyVaultName string = keyVaultName output acrLoginServer string = effectiveAcrServer @description('Virtual network name') -output vnetName string = acaNatNetwork!.outputs.vnetName +output vnetName string = deployInfra ? acaNatNetwork!.outputs.vnetName : '${appName}-vnet' @description('Application Insights connection string (if OTel enabled)') -output appInsightsConnectionString string = enableOtel ? appInsights!.properties.ConnectionString : 'N/A (OTel disabled)' +output appInsightsConnectionString string = enableOtel + ? (deployInfra ? appInsights!.properties.ConnectionString : existingAppInsights!.properties.ConnectionString) + : 'N/A (OTel disabled)' diff --git a/infra/pipelines/deploy-infra.yml b/infra/pipelines/deploy-infra.yml index 143b2e9ac0..63d452b2a1 100644 --- a/infra/pipelines/deploy-infra.yml +++ b/infra/pipelines/deploy-infra.yml @@ -8,11 +8,15 @@ parameters: values: - test - prod + - name: deployInfra + type: boolean + default: false stages: - stage: ${{ parameters.stageName }} displayName: 'Deploy ${{ parameters.slot }} infrastructure' dependsOn: ${{ parameters.dependsOn }} + condition: and(succeeded(), eq('${{ parameters.deployInfra }}', 'true')) variables: - group: copyrit-gui-common - group: copyrit-gui-${{ parameters.slot }} @@ -20,7 +24,7 @@ stages: vmImage: 'ubuntu-latest' jobs: - deployment: DeployInfrastructure - displayName: 'Preview, deploy, and verify infrastructure with the current image' + displayName: 'Preview, deploy, and verify infrastructure without redeploying the app' timeoutInMinutes: 120 environment: 'copyrit-${{ parameters.slot }}' strategy: @@ -33,6 +37,7 @@ stages: displayName: 'Preview, deploy, and verify ${{ parameters.slot }} infrastructure' env: PYRIT_SLOT: ${{ parameters.slot }} + PYRIT_DEPLOY_INFRA: 'true' PYRIT_BUILD_ID: $(Build.BuildId) PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) @@ -57,4 +62,4 @@ stages: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' scriptLocation: 'scriptPath' - scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' + scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_gui.sh' diff --git a/infra/pipelines/deploy_code.py b/infra/pipelines/deploy_code.py deleted file mode 100644 index d2803f623a..0000000000 --- a/infra/pipelines/deploy_code.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. -"""Update only the existing CoPyRIT container image and verify its current access mode.""" - -import os -import re -import subprocess -import sys -import time - - -def _az(*arguments: str) -> str: - return subprocess.run( - ["az", *arguments, "--only-show-errors"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - - -def _value(*arguments: str, query: str) -> str: - value = _az(*arguments, "--query", query, "--output", "tsv") - if not value or value in {"None", "null"} or "\n" in value or "\t" in value: - raise ValueError(f"Azure returned no single value for {query}") - return value - - -def _health_url(*, app_arguments: tuple[str, ...], environment_id: str, app_name: str) -> tuple[str, str]: - access = _value("containerapp", "env", "show", "--ids", environment_id, query="properties.publicNetworkAccess") - if access == "Enabled": - hostname = _value("containerapp", "show", *app_arguments, query="properties.configuration.ingress.fqdn") - suffix = ".azurecontainerapps.io" - elif access == "Disabled": - resource_group_id = environment_id.split("/providers/")[0] - hostname = _value( - "rest", - "--method", - "get", - "--url", - f"https://management.azure.com{resource_group_id}/providers/Microsoft.Cdn/profiles/" - f"{app_name}-afd/afdEndpoints?api-version=2024-09-01", - query="value[?properties.enabledState=='Enabled'].properties.hostName", - ) - suffix = ".azurefd.net" - else: - raise ValueError(f"Unsupported ACA public network access state: {access}") - if not re.fullmatch(r"[a-z0-9][a-z0-9.-]*", hostname) or not hostname.endswith(suffix): - raise ValueError("Azure returned an unexpected health-check hostname") - return f"https://{hostname}/api/health", access - - -def _wait_for_revision(*, app_arguments: tuple[str, ...], revision: str, image: str) -> None: - arguments = ("containerapp", "revision", "show", *app_arguments, "--revision", revision) - if _value(*arguments, query="properties.template.containers[0].image") != image: - raise ValueError("The deployed revision does not contain the requested image") - for attempt in range(1, 6): - health = _az(*arguments, "--query", "properties.healthState", "--output", "tsv") - print(f"Revision {revision} health attempt {attempt}/5: {health or ''}", flush=True) - if health == "Healthy": - return - if attempt < 5: - time.sleep(120) - raise RuntimeError("Deployed revision did not become healthy; networking was not changed") - - -def _wait_for_http_health(url: str) -> None: - deadline = time.monotonic() + 300 - while (remaining := deadline - time.monotonic()) > 0: - response = subprocess.run( - [ - "curl", - "--silent", - "--show-error", - "--output", - os.devnull, - "--write-out", - "%{http_code}", - "--max-time", - str(min(30, remaining)), - url, - ], - capture_output=True, - text=True, - check=False, - ) - status = response.stdout.strip() - print(f"Application health at {url}: {status or ''}", flush=True) - if response.returncode: - print(f"Health request failed: {response.stderr.strip()}", file=sys.stderr) - elif status == "200": - return - time.sleep(max(0, min(30, deadline - time.monotonic()))) - raise RuntimeError("Application health check failed; networking was not changed") - - -def deploy_code(*, slot: str, resource_group: str, app_name: str, acr_resource_id: str, image: str) -> None: - """Deploy an immutable image without applying templates or changing access settings.""" - if slot not in {"test", "prod"}: - raise ValueError("Invalid deployment slot") - if not re.fullmatch(r"[a-zA-Z0-9_.()-]{1,90}", resource_group) or resource_group.endswith("."): - raise ValueError("Invalid deployment resource group") - if not re.fullmatch(r"[a-z][a-z0-9-]{0,30}[a-z0-9]", app_name): - raise ValueError("Invalid container app name") - acr = re.fullmatch( - r"/subscriptions/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})" - r"/resourcegroups/[^/]+/providers/microsoft\.containerregistry/registries/([a-z0-9]{5,50})", - acr_resource_id.casefold(), - ) - if acr is None: - raise ValueError("ACR resource ID is not canonical") - subscription, registry = acr.groups() - if not re.fullmatch( - rf"{registry}\.azurecr\.io/[a-z0-9]+(?:[._-][a-z0-9]+)*" - r"(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*@sha256:[0-9a-fA-F]{64}", - image, - ): - raise ValueError("Image must be an immutable digest from the configured ACR") - if _value("account", "show", query="id").casefold() != subscription: - raise ValueError("Azure subscription does not match ACR") - - app_arguments = ("--resource-group", resource_group, "--name", app_name) - show = ("containerapp", "show", *app_arguments) - resource_group_id = f"/subscriptions/{subscription}/resourceGroups/{resource_group}" - environment_id = f"{resource_group_id}/providers/Microsoft.App/managedEnvironments/{app_name}-env" - if _value(*show, query="properties.managedEnvironmentId").casefold() != environment_id.casefold(): - raise ValueError("Existing app does not belong to the expected environment") - if _value(*show, query="properties.configuration.activeRevisionsMode") != "Single": - raise ValueError("Code-only deployment requires the existing single-revision topology") - if ( - _value(*show, query="length(properties.template.containers)") != "1" - or _value(*show, query="properties.template.containers[0].name") != "pyrit-gui" - ): - raise ValueError("Code-only deployment requires the existing pyrit-gui container") - url, access = _health_url(app_arguments=app_arguments, environment_id=environment_id, app_name=app_name) - previous_image = _value(*show, query="properties.template.containers[0].image") - print(f"Code-only {slot} deployment; ACA public access: {access}; verification URL: {url}", flush=True) - print(f"Previous image (not automatically restored): {previous_image}", flush=True) - if previous_image != image: - _az( - "containerapp", - "update", - *app_arguments, - "--container-name", - "pyrit-gui", - "--image", - image, - "--output", - "none", - ) - revision = _value(*show, query="properties.latestRevisionName") - _wait_for_revision(app_arguments=app_arguments, revision=revision, image=image) - _wait_for_http_health(url) - if ( - _value(*show, query="properties.latestRevisionName") != revision - or _value(*show, query="properties.latestReadyRevisionName") != revision - or _value(*show, query="properties.template.containers[0].image") != image - ): - raise RuntimeError("The verified image revision is not the current ready revision") - final_url, final_access = _health_url(app_arguments=app_arguments, environment_id=environment_id, app_name=app_name) - if (final_url, final_access) != (url, access): - raise RuntimeError("The application's access mode changed during code-only deployment") - print(f"Code deployment healthy: {revision}; networking unchanged; verified {url}", flush=True) - - -def main() -> int: - """Read pipeline inputs and fail without rolling back infrastructure.""" - inputs = { - "slot": "PYRIT_SLOT", - "resource_group": "PYRIT_DEPLOYMENT_RESOURCE_GROUP", - "app_name": "PYRIT_APP_NAME", - "acr_resource_id": "PYRIT_ACR_RESOURCE_ID", - "image": "PYRIT_CONTAINER_IMAGE", - } - try: - values = {} - for name, variable in inputs.items(): - value = os.environ.get(variable, "") - if not value or value.startswith("$("): - raise ValueError(f"Required deployment value is missing or unresolved: {variable}") - values[name] = value - deploy_code(**values) - except (ValueError, RuntimeError, OSError, subprocess.CalledProcessError) as error: - print(f"##vso[task.logissue type=error]Code deployment failed: {error}", file=sys.stderr) - if isinstance(error, subprocess.CalledProcessError) and error.stderr: - print(error.stderr, file=sys.stderr) - print("No automatic image, database, or infrastructure rollback was attempted.", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/infra/pipelines/deploy_public_nat.sh b/infra/pipelines/deploy_gui.sh similarity index 56% rename from infra/pipelines/deploy_public_nat.sh rename to infra/pipelines/deploy_gui.sh index a286b6240d..16abbb8da4 100644 --- a/infra/pipelines/deploy_public_nat.sh +++ b/infra/pipelines/deploy_gui.sh @@ -10,6 +10,7 @@ lowercase() { required_variables=( PYRIT_SLOT + PYRIT_DEPLOY_INFRA PYRIT_BUILD_ID PYRIT_SOURCE_DIRECTORY PYRIT_AGENT_TEMP_DIRECTORY @@ -50,8 +51,9 @@ if [[ -n "${PYRIT_ALLOWED_CLIENT_CIDR:-}" ]]; then exit 1 fi -if [[ ! "$PYRIT_SLOT" =~ ^(test|prod)$ || ! "$PYRIT_BUILD_ID" =~ ^[0-9]+$ ]]; then - echo "##vso[task.logissue type=error]Invalid slot or build ID" +if [[ ! "$PYRIT_SLOT" =~ ^(test|prod)$ || ! "$PYRIT_BUILD_ID" =~ ^[0-9]+$ || + ! "$PYRIT_DEPLOY_INFRA" =~ ^(true|false)$ ]]; then + echo "##vso[task.logissue type=error]Invalid slot, build ID, or deployInfra value" exit 1 fi @@ -79,7 +81,7 @@ if ! python3 - \ "$PYRIT_ENTRA_CLIENT_ID" \ "$PYRIT_ALLOWED_GROUP_OBJECT_IDS" \ "$PYRIT_ADMIN_GROUP_OBJECT_ID" \ - "${PYRIT_CONFIG_FILE_URI:-}" <<'PY' + "${PYRIT_CONFIG_FILE_URI:-}" << 'PY'; then import ipaddress import sys import uuid @@ -124,8 +126,7 @@ try: except (ValueError, IndexError): raise SystemExit(1) PY -then - echo "##vso[task.logissue type=error]Invalid network prefix, subnet sizing, Entra ID, group ID, or config URI" + echo "##vso[task.logissue type=error]Invalid network prefix, subnet sizing, Entra ID, group ID, or config URI" exit 1 fi @@ -155,19 +156,19 @@ if [[ ! "$normalized_key_vault_resource_id" =~ ^/subscriptions/($guid_pattern)/r echo "##vso[task.logissue type=error]Key Vault resource ID is not canonical or is in another subscription" exit 1 fi -if [[ ! "$PYRIT_SQL_SERVER_FQDN" =~ ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\.database\.windows\.net$ \ - || ! "$PYRIT_ENV_SECRET_NAME" =~ ^[a-zA-Z0-9-]{1,127}$ \ - || ! "$PYRIT_ENABLE_OTEL" =~ ^(true|false)$ ]]; then +if [[ ! "$PYRIT_SQL_SERVER_FQDN" =~ ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\.database\.windows\.net$ || + ! "$PYRIT_ENV_SECRET_NAME" =~ ^[a-zA-Z0-9-]{1,127}$ || + ! "$PYRIT_ENABLE_OTEL" =~ ^(true|false)$ ]]; then echo "##vso[task.logissue type=error]Invalid SQL FQDN, Key Vault secret name, or enableOtel value" exit 1 fi -if ! az resource show --ids "$PYRIT_MANAGED_IDENTITY_RESOURCE_ID" --api-version 2023-01-31 -o none 2>/dev/null; then +if ! az resource show --ids "$PYRIT_MANAGED_IDENTITY_RESOURCE_ID" --api-version 2023-01-31 -o none 2> /dev/null; then echo "##vso[task.logissue type=error]Managed identity does not exist or is not readable" exit 1 fi deployment_resource_group_id=$(az group show \ - --name "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" --query id -o tsv 2>/dev/null || true) + --name "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" --query id -o tsv 2> /dev/null || true) if [[ -z "$deployment_resource_group_id" ]]; then echo "##vso[task.logissue type=error]Deployment resource group must already exist" exit 1 @@ -194,31 +195,31 @@ normalized_expected_pip_id=$(lowercase "$expected_pip_id") existing_app=$(az containerapp show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME" \ - --query '{id:id,environmentId:properties.managedEnvironmentId,tags:tags,containers:properties.template.containers[].{name:name,image:image}}' -o json 2>/dev/null || true) + --query '{id:id,environmentId:properties.managedEnvironmentId,tags:tags,mode:properties.configuration.activeRevisionsMode,revision:properties.latestRevisionName,containers:properties.template.containers[].{name:name,image:image}}' -o json 2> /dev/null || true) existing_environment=$(az containerapp env show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-env" \ - --query '{id:id,publicNetworkAccess:properties.publicNetworkAccess}' -o json 2>/dev/null || true) + --query '{id:id,publicNetworkAccess:properties.publicNetworkAccess}' -o json 2> /dev/null || true) existing_vnet=$(az network vnet show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-vnet" \ - --query '{id:id,prefix:addressSpace.addressPrefixes[0],tags:tags}' -o json 2>/dev/null || true) + --query '{id:id,prefix:addressSpace.addressPrefixes[0],tags:tags}' -o json 2> /dev/null || true) existing_subnet=$(az network vnet subnet show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --vnet-name "$PYRIT_APP_NAME-vnet" \ --name "$PYRIT_APP_NAME-aca-subnet" \ - --query '{id:id,prefix:addressPrefix,natId:natGateway.id}' -o json 2>/dev/null || true) + --query '{id:id,prefix:addressPrefix,natId:natGateway.id}' -o json 2> /dev/null || true) existing_nat=$(az network nat gateway show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-nat" \ - --query '{id:id,pipId:publicIpAddresses[0].id,tags:tags}' -o json 2>/dev/null || true) + --query '{id:id,pipId:publicIpAddresses[0].id,tags:tags}' -o json 2> /dev/null || true) existing_pip=$(az network public-ip show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-egress-pip" \ - --query '{id:id,ip:ipAddress,allocation:publicIPAllocationMethod,sku:sku.name,tags:tags}' -o json 2>/dev/null || true) + --query '{id:id,ip:ipAddress,allocation:publicIPAllocationMethod,sku:sku.name,tags:tags}' -o json 2> /dev/null || true) -if [[ -z "$existing_app" || -z "$existing_environment" || -z "$existing_vnet" || -z "$existing_subnet" \ - || -z "$existing_nat" || -z "$existing_pip" ]]; then +if [[ -z "$existing_app" || -z "$existing_environment" || -z "$existing_vnet" || -z "$existing_subnet" || + -z "$existing_nat" || -z "$existing_pip" ]]; then echo "##vso[task.logissue type=error]Internal deployments must adopt an existing app, environment, VNet, subnet, NAT, and egress PIP" exit 1 fi @@ -232,61 +233,101 @@ existing_pip_ip_tags=$(az network public-ip show \ --name "$PYRIT_APP_NAME-egress-pip" --query 'ipTags || `[]`' -o json | jq -c .) expected_egress_ip=$(jq -r '.ip // empty' <<< "$existing_pip") -if [[ "$(jq -r '.id | ascii_downcase' <<< "$existing_app")" != "$normalized_expected_app_id" \ - || "$(jq -r '.environmentId | ascii_downcase' <<< "$existing_app")" != "$normalized_expected_environment_id" \ - || "$(jq -r '.id | ascii_downcase' <<< "$existing_environment")" != "$normalized_expected_environment_id" \ - || ! "$(jq -r '.publicNetworkAccess' <<< "$existing_environment")" =~ ^(Enabled|Disabled)$ \ - || "$(jq -r '.id | ascii_downcase' <<< "$existing_vnet")" != "$normalized_expected_vnet_id" \ - || "$(jq -r '.id | ascii_downcase' <<< "$existing_subnet")" != "$normalized_expected_subnet_id" \ - || "$(jq -r '.id | ascii_downcase' <<< "$existing_nat")" != "$normalized_expected_nat_id" \ - || "$(jq -r '.id | ascii_downcase' <<< "$existing_pip")" != "$normalized_expected_pip_id" \ - || "$(jq -r '.natId | ascii_downcase' <<< "$existing_subnet")" != "$normalized_expected_nat_id" \ - || "$(jq -r '.pipId | ascii_downcase' <<< "$existing_nat")" != "$normalized_expected_pip_id" \ - || "$(jq -r '.prefix' <<< "$existing_vnet")" != "$PYRIT_VNET_ADDRESS_PREFIX" \ - || "$(jq -r '.prefix' <<< "$existing_subnet")" != "$PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX" \ - || "$(jq -r '.allocation' <<< "$existing_pip")" != "Static" \ - || "$(jq -r '.sku' <<< "$existing_pip")" != "Standard" \ - || -z "$expected_egress_ip" ]]; then +if [[ "$(jq -r '.id | ascii_downcase' <<< "$existing_app")" != "$normalized_expected_app_id" || +"$(jq -r '.environmentId | ascii_downcase' <<< "$existing_app")" != "$normalized_expected_environment_id" || +"$(jq -r '.id | ascii_downcase' <<< "$existing_environment")" != "$normalized_expected_environment_id" || +! "$(jq -r '.publicNetworkAccess' <<< "$existing_environment")" =~ ^(Enabled|Disabled)$ || +"$(jq -r '.id | ascii_downcase' <<< "$existing_vnet")" != "$normalized_expected_vnet_id" || +"$(jq -r '.id | ascii_downcase' <<< "$existing_subnet")" != "$normalized_expected_subnet_id" || +"$(jq -r '.id | ascii_downcase' <<< "$existing_nat")" != "$normalized_expected_nat_id" || +"$(jq -r '.id | ascii_downcase' <<< "$existing_pip")" != "$normalized_expected_pip_id" || +"$(jq -r '.natId | ascii_downcase' <<< "$existing_subnet")" != "$normalized_expected_nat_id" || +"$(jq -r '.pipId | ascii_downcase' <<< "$existing_nat")" != "$normalized_expected_pip_id" || +"$(jq -r '.prefix' <<< "$existing_vnet")" != "$PYRIT_VNET_ADDRESS_PREFIX" || +"$(jq -r '.prefix' <<< "$existing_subnet")" != "$PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX" || +"$(jq -r '.allocation' <<< "$existing_pip")" != "Static" || +"$(jq -r '.sku' <<< "$existing_pip")" != "Standard" || +-z "$expected_egress_ip" ]]; then echo "##vso[task.logissue type=error]Deployment variables do not match the existing protected topology" exit 1 fi -if [[ "$deployment_tags" == *'<'* || "$deployment_tags" == "null" \ - || "$deployment_tags" != "$pip_tags" || "$deployment_tags" != "$nat_tags" \ - || "$deployment_tags" != "$vnet_tags" ]]; then +if [[ "$deployment_tags" == *'<'* || "$deployment_tags" == "null" || + "$deployment_tags" != "$pip_tags" || "$deployment_tags" != "$nat_tags" || + "$deployment_tags" != "$vnet_tags" ]]; then echo "##vso[task.logissue type=error]Protected resource tags are missing, placeholders, or inconsistent" exit 1 fi -if [[ "$(jq '.containers | length' <<< "$existing_app")" != "1" \ - || "$(jq -r '.containers[0].name' <<< "$existing_app")" != "pyrit-gui" ]]; then - echo "##vso[task.logissue type=error]Infrastructure deployment requires the existing pyrit-gui container" +if [[ "$(jq '.containers | length' <<< "$existing_app")" != "1" || +"$(jq -r '.containers[0].name' <<< "$existing_app")" != "pyrit-gui" || +"$(jq -r '.mode' <<< "$existing_app")" != "Single" ]]; then + echo "##vso[task.logissue type=error]Deployment requires the existing single-revision pyrit-gui container" exit 1 fi current_image=$(jq -r '.containers[0].image // empty' <<< "$existing_app") -if [[ ! "$current_image" =~ ^([^/]+)/(.+)@(sha256:[0-9a-fA-F]{64})$ ]]; then - echo "##vso[task.logissue type=error]Current image must be an immutable registry digest" +current_revision=$(jq -r '.revision // empty' <<< "$existing_app") +echo "Current image (not automatically restored after an app failure): $current_image" +deploy_app=true +if [[ "$PYRIT_DEPLOY_INFRA" == "true" ]]; then + deploy_app=false + requested_image=$current_image + if [[ -z "$current_revision" ]]; then + echo "##vso[task.logissue type=error]Infrastructure deployment requires an existing app revision" + exit 1 + fi + echo "Reconciling infrastructure only; leaving the running application unchanged" +else + requested_image=${PYRIT_CONTAINER_IMAGE:-} +fi +if [[ ! "$requested_image" =~ ^([^/]+)/(.+)@(sha256:[0-9a-fA-F]{64})$ ]]; then + echo "##vso[task.logissue type=error]Requested image must be an immutable registry digest" exit 1 fi registry_server=${BASH_REMATCH[1]} repository=${BASH_REMATCH[2]} digest=${BASH_REMATCH[3]} if [[ "$registry_server" != "$acr_name.azurecr.io" ]]; then - echo "##vso[task.logissue type=error]Current image registry does not match ACR resource ID" + echo "##vso[task.logissue type=error]Requested image registry does not match ACR resource ID" exit 1 fi repository_pattern='^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*$' if [[ ! "$repository" =~ $repository_pattern ]]; then - echo "##vso[task.logissue type=error]Current image repository is invalid" + echo "##vso[task.logissue type=error]Requested image repository is invalid" exit 1 fi immutable_image="$registry_server/$repository@$digest" -echo "Infrastructure-only deployment; retaining current image: $immutable_image" private_link_request_message="Azure Front Door private access to $PYRIT_APP_NAME" +enable_front_door=true +enable_private_link=true +disable_public_access=true +expected_public_access=Disabled +if [[ "$PYRIT_DEPLOY_INFRA" == "false" ]]; then + expected_public_access=$(jq -r '.publicNetworkAccess' <<< "$existing_environment") + front_door_count=$(az resource list \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --resource-type Microsoft.Cdn/profiles --name "$PYRIT_APP_NAME-afd" --query 'length(@)' -o tsv) + case "$front_door_count" in + 0) enable_front_door=false ;; + 1) enable_front_door=true ;; + *) + echo "##vso[task.logissue type=error]Could not identify the existing Front Door profile" + exit 1 + ;; + esac + if [[ "$expected_public_access" == "Enabled" ]]; then + enable_private_link=false + disable_public_access=false + elif [[ "$enable_front_door" != "true" ]]; then + echo "##vso[task.logissue type=error]Private ACA access requires the existing Front Door profile" + exit 1 + fi +fi parameters=( "appName=$PYRIT_APP_NAME" - "containerImage=$immutable_image" + "deployInfra=$PYRIT_DEPLOY_INFRA" + "deployApp=$deploy_app" "entraTenantId=$PYRIT_ENTRA_TENANT_ID" "entraClientId=$PYRIT_ENTRA_CLIENT_ID" "allowedGroupObjectIds=$PYRIT_ALLOWED_GROUP_OBJECT_IDS" @@ -300,16 +341,19 @@ parameters=( "enableOtel=$PYRIT_ENABLE_OTEL" "envSecretName=$PYRIT_ENV_SECRET_NAME" "pyritConfigFileUri=${PYRIT_CONFIG_FILE_URI:-}" - "enableFrontDoor=true" - "enableFrontDoorPrivateLink=true" + "enableFrontDoor=$enable_front_door" + "enableFrontDoorPrivateLink=$enable_private_link" "frontDoorPrivateLinkRequestMessage=$private_link_request_message" - "disableContainerAppsPublicAccess=true" + "disableContainerAppsPublicAccess=$disable_public_access" "vnetAddressPrefix=$PYRIT_VNET_ADDRESS_PREFIX" "infrastructureSubnetAddressPrefix=$PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX" "egressPublicIpTags=$existing_pip_ip_tags" "protectEgressPublicIp=true" "tags=$deployment_tags" ) +if [[ "$deploy_app" == "true" ]]; then + parameters+=("containerImage=$immutable_image") +fi rollback_parameters=() for parameter in "${parameters[@]}"; do @@ -320,12 +364,14 @@ for parameter in "${parameters[@]}"; do esac done -deployment_name="pyrit-$PYRIT_SLOT-$PYRIT_BUILD_ID" +deployment_name="pyrit-$PYRIT_SLOT-$PYRIT_BUILD_ID-app" +[[ "$PYRIT_DEPLOY_INFRA" == "true" ]] && deployment_name="pyrit-$PYRIT_SLOT-$PYRIT_BUILD_ID-infra" what_if_file="$PYRIT_AGENT_TEMP_DIRECTORY/$deployment_name-what-if.json" az deployment group what-if \ --name "$deployment_name-preview" \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --template-file "$PYRIT_SOURCE_DIRECTORY/infra/main.bicep" \ + --mode Incremental \ --parameters "${parameters[@]}" \ --result-format FullResourcePayloads --no-pretty-print -o json > "$what_if_file" @@ -343,6 +389,25 @@ if ! python3 "$PYRIT_SOURCE_DIRECTORY/infra/pipelines/validate_what_if.py" \ exit 1 fi +if [[ "$PYRIT_DEPLOY_INFRA" == "false" ]] && ! jq -e --arg app "$normalized_expected_app_id" ' + .changes | all(.[]; + .changeType == "Ignore" or .changeType == "NoChange" or + (.changeType == "Modify" and (.resourceId | ascii_downcase) == $app)) +' "$what_if_file" > /dev/null; then + echo "##vso[task.logissue type=error]App-only preview must not write infrastructure" + exit 1 +fi + +if [[ "$PYRIT_DEPLOY_INFRA" == "true" ]] && ! jq -e --arg app "$normalized_expected_app_id" ' + .changes | all(.[]; + .changeType == "Ignore" or .changeType == "NoChange" or + ((.resourceId | ascii_downcase | rtrimstr("/")) as $id | + $id != $app and ($id | startswith($app + "/") | not))) +' "$what_if_file" > /dev/null; then + echo "##vso[task.logissue type=error]Infrastructure-only preview must not write the Container App" + exit 1 +fi + cutover_in_progress=false rollback_public_origin() { local exit_code=$? @@ -360,10 +425,10 @@ rollback_public_origin() { --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --template-file "$PYRIT_SOURCE_DIRECTORY/infra/modules/aca_front_door.bicep" \ --parameters \ - "namePrefix=$PYRIT_APP_NAME" \ - "originHostName=$rollback_origin_host" \ - "tags=$deployment_tags" \ - "enablePrivateLink=false" || true + "namePrefix=$PYRIT_APP_NAME" \ + "originHostName=$rollback_origin_host" \ + "tags=$deployment_tags" \ + "enablePrivateLink=false" || true fi local rollback_connections @@ -372,7 +437,7 @@ rollback_public_origin() { rollback_connections=$(az network private-endpoint-connection list \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-env" \ - --type Microsoft.App/managedEnvironments -o json 2>/dev/null || true) + --type Microsoft.App/managedEnvironments -o json 2> /dev/null || true) if [[ -n "$rollback_connections" ]]; then while IFS= read -r connection_id; do [[ -z "$connection_id" ]] && continue @@ -390,7 +455,7 @@ rollback_public_origin() { rollback_connections=$(az network private-endpoint-connection list \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-env" \ - --type Microsoft.App/managedEnvironments -o json 2>/dev/null || true) + --type Microsoft.App/managedEnvironments -o json 2> /dev/null || true) if [[ -z "$rollback_connections" ]]; then rollback_connection_count=-1 [[ "$attempt" -lt 20 ]] && sleep 15 @@ -412,6 +477,7 @@ rollback_public_origin() { --name "$deployment_name-rollback" \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --template-file "$PYRIT_SOURCE_DIRECTORY/infra/main.bicep" \ + --mode Incremental \ --parameters "${rollback_parameters[@]}"; then echo "##vso[task.logissue type=warning]Public ACA origin rollback completed" else @@ -420,114 +486,134 @@ rollback_public_origin() { fi exit "$exit_code" } -trap rollback_public_origin EXIT -trap 'exit 143' TERM -trap 'exit 130' INT -cutover_in_progress=true +if [[ "$PYRIT_DEPLOY_INFRA" == "true" ]]; then + trap rollback_public_origin EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + cutover_in_progress=true +fi az deployment group create \ --name "$deployment_name" \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --template-file "$PYRIT_SOURCE_DIRECTORY/infra/main.bicep" \ + --mode Incremental \ --parameters "${parameters[@]}" -deployed_private_link_request_message=$(az deployment group show \ - --name "$deployment_name" --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ - --query properties.outputs.frontDoorPrivateLinkRequestMessage.value -o tsv) -if [[ "$deployed_private_link_request_message" != "$private_link_request_message" ]]; then - echo "##vso[task.logissue type=error]Deployment Private Link request message does not match the approved pipeline value" - exit 1 -fi -origin_resource_url="https://management.azure.com${deployment_resource_group_id}/providers/Microsoft.Cdn/profiles/$PYRIT_APP_NAME-afd/originGroups/$PYRIT_APP_NAME-origin-group/origins/$PYRIT_APP_NAME-aca-origin?api-version=2024-09-01" -origin_private_link=$(az rest --method get --url "$origin_resource_url" \ - --query '{status:properties.sharedPrivateLinkResource.status,resourceId:properties.sharedPrivateLinkResource.privateLink.id}' -o json) -private_link_status=$(jq -r '.status // empty' <<< "$origin_private_link") -private_link_resource_id=$(jq -r '.resourceId // empty | ascii_downcase' <<< "$origin_private_link") -if [[ "$private_link_resource_id" != "$normalized_expected_environment_id" \ - || ! "$private_link_status" =~ ^(Pending|Approved)$ ]]; then - echo "##vso[task.logissue type=error]Front Door Private Link does not target the expected ACA environment" - exit 1 -fi +if [[ "$PYRIT_DEPLOY_INFRA" == "true" ]]; then + deployed_private_link_request_message=$(az deployment group show \ + --name "$deployment_name" --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --query properties.outputs.frontDoorPrivateLinkRequestMessage.value -o tsv) + if [[ "$deployed_private_link_request_message" != "$private_link_request_message" ]]; then + echo "##vso[task.logissue type=error]Deployment Private Link request message does not match the approved pipeline value" + exit 1 + fi + origin_resource_url="https://management.azure.com${deployment_resource_group_id}/providers/Microsoft.Cdn/profiles/$PYRIT_APP_NAME-afd/originGroups/$PYRIT_APP_NAME-origin-group/origins/$PYRIT_APP_NAME-aca-origin?api-version=2024-09-01" + origin_private_link=$(az rest --method get --url "$origin_resource_url" \ + --query '{status:properties.sharedPrivateLinkResource.status,resourceId:properties.sharedPrivateLinkResource.privateLink.id}' -o json) + private_link_status=$(jq -r '.status // empty' <<< "$origin_private_link") + private_link_resource_id=$(jq -r '.resourceId // empty | ascii_downcase' <<< "$origin_private_link") + if [[ "$private_link_resource_id" != "$normalized_expected_environment_id" || + ! "$private_link_status" =~ ^(Pending|Approved)$ ]]; then + echo "##vso[task.logissue type=error]Front Door Private Link does not target the expected ACA environment" + exit 1 + fi -matching_connections='' -for attempt in {1..20}; do - connections=$(az network private-endpoint-connection list \ - --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ - --name "$PYRIT_APP_NAME-env" \ - --type Microsoft.App/managedEnvironments -o json || echo '[]') - matching_connections=$(jq -c --arg message "$private_link_request_message" \ - '[.[] | select( + matching_connections='' + for attempt in {1..20}; do + connections=$(az network private-endpoint-connection list \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-env" \ + --type Microsoft.App/managedEnvironments -o json || echo '[]') + matching_connections=$(jq -c --arg message "$private_link_request_message" \ + '[.[] | select( .properties.privateLinkServiceConnectionState.description == $message and (.properties.privateLinkServiceConnectionState.status == "Pending" or .properties.privateLinkServiceConnectionState.status == "Approved"))]' <<< "$connections") - connection_count=$(jq 'length' <<< "$matching_connections") - echo "Private Link request discovery attempt $attempt/20: $connection_count active connection(s)" - [[ "$connection_count" -gt 0 ]] && break - [[ "$attempt" -lt 20 ]] && sleep 15 -done -if [[ "$(jq 'length' <<< "$matching_connections")" == "0" ]]; then - echo "##vso[task.logissue type=error]Front Door did not create the expected ACA Private Link request" - exit 1 -fi - -while IFS=$'\t' read -r connection_id connection_status; do - normalized_connection_id=$(lowercase "$connection_id") - if [[ "$normalized_connection_id" != "$normalized_expected_environment_id/privateendpointconnections/"* ]]; then - echo "##vso[task.logissue type=error]Private Link request is outside the expected ACA environment" + connection_count=$(jq 'length' <<< "$matching_connections") + echo "Private Link request discovery attempt $attempt/20: $connection_count active connection(s)" + [[ "$connection_count" -gt 0 ]] && break + [[ "$attempt" -lt 20 ]] && sleep 15 + done + if [[ "$(jq 'length' <<< "$matching_connections")" == "0" ]]; then + echo "##vso[task.logissue type=error]Front Door did not create the expected ACA Private Link request" exit 1 fi - if [[ "$connection_status" == "Pending" ]]; then - connection_name=${connection_id##*/} - connection_suffix=${connection_name:0:8} - az deployment group create \ - --name "$deployment_name-private-link-approval-$connection_suffix" \ - --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ - --template-file "$PYRIT_SOURCE_DIRECTORY/infra/modules/aca_private_endpoint_approval.bicep" \ - --parameters \ + + while IFS=$'\t' read -r connection_id connection_status; do + normalized_connection_id=$(lowercase "$connection_id") + if [[ "$normalized_connection_id" != "$normalized_expected_environment_id/privateendpointconnections/"* ]]; then + echo "##vso[task.logissue type=error]Private Link request is outside the expected ACA environment" + exit 1 + fi + if [[ "$connection_status" == "Pending" ]]; then + connection_name=${connection_id##*/} + connection_suffix=${connection_name:0:8} + az deployment group create \ + --name "$deployment_name-private-link-approval-$connection_suffix" \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --template-file "$PYRIT_SOURCE_DIRECTORY/infra/modules/aca_private_endpoint_approval.bicep" \ + --parameters \ "environmentName=$PYRIT_APP_NAME-env" \ "connectionName=$connection_name" \ "approvalDescription=$private_link_request_message" -o none - fi -done < <(jq -r '.[] | [.id, .properties.privateLinkServiceConnectionState.status] | @tsv' \ - <<< "$matching_connections") + fi + done < <(jq -r '.[] | [.id, .properties.privateLinkServiceConnectionState.status] | @tsv' \ + <<< "$matching_connections") -approved_connection_count=0 -for attempt in {1..20}; do - connections=$(az network private-endpoint-connection list \ - --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ - --name "$PYRIT_APP_NAME-env" \ - --type Microsoft.App/managedEnvironments -o json || echo '[]') - approved_connection_count=$(jq --arg message "$private_link_request_message" \ - '[.[] | select( + approved_connection_count=0 + for attempt in {1..20}; do + connections=$(az network private-endpoint-connection list \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-env" \ + --type Microsoft.App/managedEnvironments -o json || echo '[]') + approved_connection_count=$(jq --arg message "$private_link_request_message" \ + '[.[] | select( .properties.privateLinkServiceConnectionState.description == $message and .properties.privateLinkServiceConnectionState.status == "Approved")] | length' <<< "$connections") - echo "ACA Private Link approval attempt $attempt/20: $approved_connection_count approved connection(s)" - [[ "$approved_connection_count" -gt 0 ]] && break - [[ "$attempt" -lt 20 ]] && sleep 15 -done -if [[ "$approved_connection_count" == "0" ]]; then - echo "##vso[task.logissue type=error]ACA Private Link connection did not become approved" - exit 1 + echo "ACA Private Link approval attempt $attempt/20: $approved_connection_count approved connection(s)" + [[ "$approved_connection_count" -gt 0 ]] && break + [[ "$attempt" -lt 20 ]] && sleep 15 + done + if [[ "$approved_connection_count" == "0" ]]; then + echo "##vso[task.logissue type=error]ACA Private Link connection did not become approved" + exit 1 + fi + echo "AFD origin status is ${private_link_status}; ACA approval and AFD health determine readiness" fi -echo "AFD origin status is ${private_link_status}; ACA approval and AFD health determine readiness" public_network_access=$(az containerapp env show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-env" \ --query properties.publicNetworkAccess -o tsv) -if [[ "$public_network_access" != "Disabled" ]]; then - echo "##vso[task.logissue type=error]ACA environment public network access remains enabled" +if [[ "$public_network_access" != "$expected_public_access" ]]; then + echo "##vso[task.logissue type=error]ACA environment public network access differs from the expected mode" exit 1 fi +revision=$(az containerapp show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME" --query properties.latestRevisionName -o tsv) +if [[ -z "$revision" || "$revision" == "null" || "$revision" == "None" ]]; then + echo "##vso[task.logissue type=error]Container App did not report a revision" + exit 1 +fi +if [[ "$PYRIT_DEPLOY_INFRA" == "true" && "$revision" != "$current_revision" ]]; then + echo "##vso[task.logissue type=error]Infrastructure-only deployment changed the running app revision" + exit 1 +fi health="" for attempt in {1..5}; do - health=$(az containerapp revision list \ + revision_state=$(az containerapp revision show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ - --name "$PYRIT_APP_NAME" \ - --query "[?properties.template.containers[0].image=='$immutable_image'] | sort_by(@,&properties.createdTime)[-1].properties.healthState" \ - -o tsv || true) - echo "Revision health attempt $attempt/5: ${health:-}" + --name "$PYRIT_APP_NAME" --revision "$revision" \ + --query '{image:properties.template.containers[0].image,health:properties.healthState}' -o json) + if [[ "$(jq -r '.image' <<< "$revision_state")" != "$immutable_image" ]]; then + echo "##vso[task.logissue type=error]Deployed revision does not contain the requested image" + exit 1 + fi + health=$(jq -r '.health // empty' <<< "$revision_state") + echo "Revision $revision health attempt $attempt/5: ${health:-}" [[ "$health" == "Healthy" ]] && break [[ "$attempt" -lt 5 ]] && sleep 120 done @@ -549,40 +635,74 @@ actual_pip_id=$(az network public-ip show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-egress-pip" --query id -o tsv) normalized_actual_pip_id=$(lowercase "$actual_pip_id") -if [[ "$egress_ip" != "$expected_egress_ip" \ - || "$normalized_actual_pip_id" != "$normalized_expected_pip_id" ]]; then +if [[ "$egress_ip" != "$expected_egress_ip" || + "$normalized_actual_pip_id" != "$normalized_expected_pip_id" ]]; then echo "##vso[task.logissue type=error]Reserved egress PIP identity or address changed" exit 1 fi -front_door_health="" -front_door_health_timeout_seconds=1800 -front_door_health_deadline=$((SECONDS + front_door_health_timeout_seconds)) +health_url="https://$app_fqdn/api/health" +if [[ "$expected_public_access" == "Disabled" ]]; then + if [[ ! "$front_door_fqdn" =~ ^[a-z0-9][a-z0-9.-]*\.azurefd\.net$ ]]; then + echo "##vso[task.logissue type=error]Private ACA access requires a valid Front Door hostname" + exit 1 + fi + health_url="https://$front_door_fqdn/api/health" +fi +if [[ ! "$app_fqdn" =~ ^[a-z0-9][a-z0-9.-]*\.azurecontainerapps\.io$ ]]; then + echo "##vso[task.logissue type=error]Deployment returned an invalid ACA hostname" + exit 1 +fi +application_health="" +health_timeout_seconds=300 +[[ "$PYRIT_DEPLOY_INFRA" == "true" ]] && health_timeout_seconds=1800 +health_deadline=$((SECONDS + health_timeout_seconds)) attempt=0 -while ((SECONDS < front_door_health_deadline)); do +while ((SECONDS < health_deadline)); do ((attempt += 1)) - remaining_seconds=$((front_door_health_deadline - SECONDS)) + remaining_seconds=$((health_deadline - SECONDS)) request_timeout=$((remaining_seconds < 30 ? remaining_seconds : 30)) - front_door_health=$(curl \ + if ! application_health=$(curl \ --silent --show-error --output /dev/null --write-out '%{http_code}' \ - --max-time "$request_timeout" "https://$front_door_fqdn/api/health" || true) - echo "Front Door health attempt $attempt (${remaining_seconds}s budget before request): ${front_door_health:-}" - [[ "$front_door_health" == "200" ]] && break - remaining_seconds=$((front_door_health_deadline - SECONDS)) + --max-time "$request_timeout" "$health_url"); then + application_health="" + fi + echo "Application health at $health_url attempt $attempt (${remaining_seconds}s budget before request): ${application_health:-}" + [[ "$application_health" == "200" ]] && break + remaining_seconds=$((health_deadline - SECONDS)) ((remaining_seconds > 0)) || break sleep_seconds=$((remaining_seconds < 30 ? remaining_seconds : 30)) sleep "$sleep_seconds" done -if [[ "$front_door_health" != "200" ]]; then - echo "##vso[task.logissue type=error]Front Door did not route a healthy response" +if [[ "$application_health" != "200" ]]; then + echo "##vso[task.logissue type=error]Application endpoint did not return a healthy response" exit 1 fi -direct_aca_health=$(curl \ - --silent --show-error --output /dev/null --write-out '%{http_code}' \ - --max-time 15 "https://$app_fqdn/api/health" || true) -if [[ "$direct_aca_health" == "200" ]]; then - echo "##vso[task.logissue type=error]Direct ACA public access remains reachable" +if [[ "$expected_public_access" == "Disabled" ]]; then + direct_aca_health=$(curl \ + --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --max-time 15 "https://$app_fqdn/api/health" || true) + if [[ "$direct_aca_health" == "200" ]]; then + echo "##vso[task.logissue type=error]Direct ACA public access remains reachable" + exit 1 + fi +fi +final_app=$(az containerapp show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" --name "$PYRIT_APP_NAME" \ + --query '{latest:properties.latestRevisionName,ready:properties.latestReadyRevisionName,image:properties.template.containers[0].image}' -o json) +final_access=$(az containerapp env show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" --name "$PYRIT_APP_NAME-env" \ + --query properties.publicNetworkAccess -o tsv) +if [[ "$(jq -r '.latest' <<< "$final_app")" != "$revision" || +"$(jq -r '.ready' <<< "$final_app")" != "$revision" || +"$(jq -r '.image' <<< "$final_app")" != "$immutable_image" || +"$final_access" != "$expected_public_access" ]]; then + echo "##vso[task.logissue type=error]Verified image is not the current ready revision or the access mode changed" exit 1 fi cutover_in_progress=false trap - EXIT TERM INT -echo "Deployment healthy; public URL: https://$front_door_fqdn; ACA public access: disabled; egress IPv4: $egress_ip" +if [[ "$PYRIT_DEPLOY_INFRA" == "true" ]]; then + echo "Infrastructure healthy; app revision unchanged: $revision; verified $health_url; egress IPv4: $egress_ip" +else + echo "Deployment healthy: $revision; verified $health_url; ACA public access: $expected_public_access; egress IPv4: $egress_ip" +fi diff --git a/tests/unit/infra/test_bicep_topology.py b/tests/unit/infra/test_bicep_topology.py index 4a99cf014f..2971e08db1 100644 --- a/tests/unit/infra/test_bicep_topology.py +++ b/tests/unit/infra/test_bicep_topology.py @@ -107,7 +107,7 @@ def test_main_has_one_public_nat_topology(self): assert len(modules) == 2 network_module = next(module for module in modules if "aca-nat-network" in module["name"]) front_door_module = next(module for module in modules if "aca-front-door" in module["name"]) - assert "condition" not in network_module + assert network_module["condition"] == "[parameters('deployInfra')]" assert "parameters('enableFrontDoor')" in front_door_module["condition"] assert ( "effectiveFrontDoorPrivateLink" @@ -142,11 +142,14 @@ def test_main_has_one_public_nat_topology(self): environment = _resources(template, "Microsoft.App/managedEnvironments")[0] environment_properties = environment["properties"] - assert environment_properties["publicNetworkAccess"] == "[variables('effectiveContainerAppsPublicAccess')]" - effective_public_access = template["variables"]["effectiveContainerAppsPublicAccess"] + assert environment["condition"] == "[parameters('deployInfra')]" + effective_public_access = environment_properties["publicNetworkAccess"] assert "disableContainerAppsPublicAccess" in effective_public_access assert "effectiveFrontDoorPrivateLink" in effective_public_access assert "fail(" in effective_public_access + assert "parameters('deployInfra')" in effective_public_access + assert "reference(resourceId('Microsoft.App/managedEnvironments'" in effective_public_access + assert effective_public_access.endswith(".publicNetworkAccess)]") assert environment_properties["vnetConfiguration"]["internal"] is False assert ( environment_properties["appLogsConfiguration"]["logAnalyticsConfiguration"]["dynamicJsonColumns"] is False @@ -182,6 +185,41 @@ def test_main_has_one_public_nat_topology(self): ) assert "aca-front-door" in cors_value assert "outputs.endpointHostName.value" in cors_value + assert "Microsoft.Cdn/profiles/afdEndpoints" in cors_value + assert ".publicNetworkAccess" in cors_value + + def test_main_gates_application_and_infrastructure_independently(self) -> None: + template = _compile_bicep(MAIN_BICEP, self.output_directory / "app-only.json") + + assert template["parameters"]["deployInfra"]["defaultValue"] is True + assert template["parameters"]["deployApp"]["defaultValue"] is True + assert template["parameters"]["containerImage"]["defaultValue"] == "" + apps = _resources(template, "Microsoft.App/containerApps") + assert len(apps) == 1 + assert apps[0]["condition"] == "[parameters('deployApp')]" + assert apps[0]["properties"]["template"]["containers"][0]["image"] == "[variables('effectiveContainerImage')]" + image = template["variables"]["effectiveContainerImage"] + assert "and(parameters('deployApp'), empty(parameters('containerImage')))" in image + assert "fail('containerImage is required when deployApp is true')" in image + for resource in template["resources"]: + if resource["type"] == "Microsoft.App/containerApps": + continue + condition = resource["condition"] + if resource["type"] == "Microsoft.ContainerRegistry/registries": + condition = template["variables"]["createAcr"] + assert "fail('App-only deployment requires an existing registry')" in condition + elif resource["type"] == "Microsoft.ManagedIdentity/userAssignedIdentities": + condition = template["variables"]["createManagedIdentity"] + assert "fail('App-only deployment requires existingManagedIdentityResourceId')" in condition + assert "parameters('deployInfra')" in condition + + outputs = template["outputs"] + assert outputs["egressPublicIpAddress"]["value"].startswith("[if(parameters('deployInfra'),") + assert "Microsoft.Network/publicIPAddresses" in outputs["egressPublicIpAddress"]["value"] + assert "Microsoft.Cdn/profiles/afdEndpoints" in outputs["frontDoorFqdn"]["value"] + assert "parameters('deployInfra')" in outputs["frontDoorFqdn"]["value"] + assert outputs["appFqdn"]["value"].startswith("[if(parameters('deployApp'),") + assert "reference(resourceId('Microsoft.App/containerApps'" in outputs["appFqdn"]["value"] def test_aca_nat_network_is_static_and_delegated(self): template = _compile_bicep(NETWORK_BICEP, self.output_directory / "network.json") diff --git a/tests/unit/infra/test_code_deployment.py b/tests/unit/infra/test_code_deployment.py index 283b6892b0..25c152910e 100644 --- a/tests/unit/infra/test_code_deployment.py +++ b/tests/unit/infra/test_code_deployment.py @@ -1,203 +1,463 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Exercise code-only deployment with no Azure or network calls.""" +"""Exercise the shared Bash deployment flow without Azure or network calls.""" +import json import os +import shlex +import shutil import subprocess +import sys +import tempfile import unittest -from unittest.mock import patch - -from infra.pipelines import deploy_code +from pathlib import Path +REPO_ROOT = Path(__file__).resolve().parents[3] +DEPLOY_SCRIPT = REPO_ROOT / "infra" / "pipelines" / "deploy_gui.sh" SUBSCRIPTION = "11111111-1111-1111-1111-111111111111" -ENVIRONMENT = ( - f"/subscriptions/{SUBSCRIPTION}/resourceGroups/copyrit-test" - "/providers/Microsoft.App/managedEnvironments/copyrit-test-env" -) +RESOURCE_GROUP = f"/subscriptions/{SUBSCRIPTION}/resourceGroups/copyrit-test" +APP = f"{RESOURCE_GROUP}/providers/Microsoft.App/containerApps/copyrit-test" +ENVIRONMENT = f"{RESOURCE_GROUP}/providers/Microsoft.App/managedEnvironments/copyrit-test-env" +VNET = f"{RESOURCE_GROUP}/providers/Microsoft.Network/virtualNetworks/copyrit-test-vnet" +SUBNET = f"{VNET}/subnets/copyrit-test-aca-subnet" +NAT = f"{RESOURCE_GROUP}/providers/Microsoft.Network/natGateways/copyrit-test-nat" +PIP = f"{RESOURCE_GROUP}/providers/Microsoft.Network/publicIPAddresses/copyrit-test-egress-pip" IMAGE = f"copyritacr.azurecr.io/pyrit@sha256:{'a' * 64}" PREVIOUS_IMAGE = f"copyritacr.azurecr.io/pyrit@sha256:{'b' * 64}" ACA_HOST = "copyrit-test.example.westus2.azurecontainerapps.io" AFD_HOST = "copyrit-test.example.azurefd.net" REVISION = "copyrit-test--0000002" +PREVIOUS_REVISION = "copyrit-test--0000001" + + +def _find_bash() -> str | None: + if os.name != "nt": + return shutil.which("bash") + candidates = [ + Path(os.environ.get("PROGRAMFILES", r"C:\Program Files")) / "Git" / "bin" / "bash.exe", + Path(os.environ.get("LOCALAPPDATA", "")) / "Programs" / "Git" / "bin" / "bash.exe", + ] + return next((str(candidate) for candidate in candidates if candidate.is_file()), None) + + +BASH = _find_bash() +JQ = shutil.which("jq") + +# Only the external services and elapsed time are replaced; source, jq, and Python run normally. +HARNESS = r""" +set -euo pipefail +az() { + printf '%s\t' "$@" >> "$MOCK_DIRECTORY/az.log"; printf '\n' >> "$MOCK_DIRECTORY/az.log" + if [[ -n "$MOCK_FAIL" && "$*" == "$MOCK_FAIL"* ]]; then return 23; fi + local query='' previous='' argument deploy_app=false deploy_infra=false + for argument in "$@"; do + [[ "$previous" != --query ]] || query=$argument + previous=$argument + case "$argument" in + deployApp=true) deploy_app=true ;; + deployInfra=true) deploy_infra=true ;; + esac + done + case "$1 $2 ${3:-}" in + 'account show --query') printf '%s\n' "$MOCK_SUBSCRIPTION" ;; + 'resource show --ids') : ;; + 'resource list --resource-group') printf '%s\n' "$MOCK_FRONT_DOOR_COUNT" ;; + 'group show --name') printf '%s\n' "$MOCK_RESOURCE_GROUP" ;; + 'containerapp show --resource-group') + case "$query" in + '{id:'*) printf '%s\n' "$MOCK_APP" ;; + properties.latestRevisionName) printf '%s\n' "${MOCK_REVISION:-$(cat "$MOCK_DIRECTORY/revision")}" ;; + properties.configuration.ingress.fqdn) printf '%s\n' "$MOCK_ACA_HOST" ;; + '{latest:'*) + touch "$MOCK_DIRECTORY/final-read" + jq -cn --arg latest "${MOCK_FINAL_LATEST:-$(cat "$MOCK_DIRECTORY/revision")}" \ + --arg ready "${MOCK_FINAL_READY:-$(cat "$MOCK_DIRECTORY/revision")}" \ + --arg image "${MOCK_FINAL_IMAGE:-$(cat "$MOCK_DIRECTORY/image")}" \ + '{latest:$latest,ready:$ready,image:$image}' ;; + *) echo "Unexpected app query: $query" >&2; return 97 ;; + esac ;; + 'containerapp env show') + if [[ "$query" == '{id:'* ]]; then printf '%s\n' "$MOCK_ENVIRONMENT" + elif [[ "$query" == properties.publicNetworkAccess ]]; then + if [[ -e "$MOCK_DIRECTORY/final-read" && -n "$MOCK_FINAL_ACCESS" ]]; then + printf '%s\n' "$MOCK_FINAL_ACCESS" + else cat "$MOCK_DIRECTORY/access"; fi + else echo "Unexpected environment query: $query" >&2; return 97; fi ;; + 'network vnet show') printf '%s\n' "$MOCK_VNET" ;; + 'network vnet subnet') printf '%s\n' "$MOCK_SUBNET" ;; + 'network nat gateway') printf '%s\n' "$MOCK_NAT" ;; + 'network public-ip show') + case "$query" in + '{id:'*) printf '%s\n' "$MOCK_PIP" ;; + 'ipTags || `[]`') printf '[]\n' ;; + id) printf '%s\n' "$MOCK_PIP_ID" ;; + *) echo "Unexpected PIP query: $query" >&2; return 97 ;; + esac ;; + 'deployment group what-if') printf '%s\n' "$MOCK_WHAT_IF" ;; + 'deployment group create') + [[ "$*" != *-rollback-origin* ]] || touch "$MOCK_DIRECTORY/rollback" + for argument in "$@"; do + case "$argument" in + containerImage=*) + if [[ "$deploy_app" == true ]]; then + printf '%s\n' "${argument#*=}" > "$MOCK_DIRECTORY/image" + printf '%s\n' "$MOCK_DEPLOYED_REVISION" > "$MOCK_DIRECTORY/revision" + fi ;; + disableContainerAppsPublicAccess=true) + if [[ "$deploy_infra" == true ]]; then printf 'Disabled\n' > "$MOCK_DIRECTORY/access"; fi ;; + disableContainerAppsPublicAccess=false) + if [[ "$deploy_infra" == true ]]; then printf 'Enabled\n' > "$MOCK_DIRECTORY/access"; fi ;; + esac + done ;; + 'deployment group show') + case "$query" in + properties.outputs.frontDoorPrivateLinkRequestMessage.value) printf '%s\n' "$MOCK_PL_MESSAGE" ;; + properties.outputs.appFqdn.value) printf '%s\n' "$MOCK_ACA_HOST" ;; + properties.outputs.frontDoorFqdn.value) printf '%s\n' "$MOCK_AFD_HOST" ;; + properties.outputs.egressPublicIpAddress.value) printf '203.0.113.10\n' ;; + *) echo "Unexpected deployment output: $query" >&2; return 97 ;; + esac ;; + 'containerapp revision show') + jq -cn --arg image "${MOCK_REVISION_IMAGE:-$(cat "$MOCK_DIRECTORY/image")}" \ + --arg health "$MOCK_HEALTH" '{image:$image,health:$health}' ;; + 'rest --method get') printf '%s\n' "$MOCK_ORIGIN" ;; + 'rest --method delete') : ;; + 'network private-endpoint-connection list') + if [[ -e "$MOCK_DIRECTORY/rollback" ]]; then printf '[]\n' + else printf '%s\n' "$MOCK_CONNECTIONS"; fi ;; + *) echo "Unexpected Azure call: $*" >&2; return 97 ;; + esac +} +curl() { + printf '%s\t' "$@" >> "$MOCK_DIRECTORY/curl.log"; printf '\n' >> "$MOCK_DIRECTORY/curl.log" + if [[ "${!#}" == "https://$MOCK_ACA_HOST/api/health" && + "$(cat "$MOCK_DIRECTORY/access")" == Disabled ]]; then + printf '403'; return 0 + fi + printf '%s' "$MOCK_HTTP_STATUS" + return "$MOCK_HTTP_EXIT" +} +sleep() { SECONDS=$((SECONDS + $1)); } +""" +@unittest.skipIf(BASH is None or JQ is None, "Native Bash and jq are required") class TestCodeDeployment(unittest.TestCase): def setUp(self) -> None: - self.arguments = { - "slot": "test", - "resource_group": "copyrit-test", - "app_name": "copyrit-test", - "acr_resource_id": ( - f"/subscriptions/{SUBSCRIPTION}/resourceGroups/shared" - "/providers/Microsoft.ContainerRegistry/registries/copyritacr" + tags = {"owner": "copyrit"} + self.environment = { + "PYRIT_SLOT": "test", + "PYRIT_DEPLOY_INFRA": "false", + "PYRIT_BUILD_ID": "42", + "PYRIT_SOURCE_DIRECTORY": REPO_ROOT.as_posix(), + "PYRIT_DEPLOYMENT_RESOURCE_GROUP": "copyrit-test", + "PYRIT_APP_NAME": "copyrit-test", + "PYRIT_CONTAINER_IMAGE": IMAGE, + "PYRIT_VNET_ADDRESS_PREFIX": "10.20.0.0/16", + "PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX": "10.20.0.0/23", + "PYRIT_ALLOWED_CLIENT_CIDR": "", + "PYRIT_MANAGED_IDENTITY_RESOURCE_ID": ( + f"{RESOURCE_GROUP}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/copyrit-id" ), - "image": IMAGE, + "PYRIT_ENTRA_TENANT_ID": SUBSCRIPTION, + "PYRIT_ENTRA_CLIENT_ID": SUBSCRIPTION, + "PYRIT_ALLOWED_GROUP_OBJECT_IDS": SUBSCRIPTION, + "PYRIT_ADMIN_GROUP_OBJECT_ID": SUBSCRIPTION, + "PYRIT_CONFIG_FILE_URI": "", + "PYRIT_SQL_SERVER_FQDN": "copyrit.database.windows.net", + "PYRIT_SQL_DATABASE_NAME": "copyrit", + "PYRIT_KEY_VAULT_RESOURCE_ID": f"{RESOURCE_GROUP}/providers/Microsoft.KeyVault/vaults/copyrit-kv", + "PYRIT_ACR_RESOURCE_ID": f"{RESOURCE_GROUP}/providers/Microsoft.ContainerRegistry/registries/copyritacr", + "PYRIT_ENABLE_OTEL": "false", + "PYRIT_ENV_SECRET_NAME": "pyrit-env", + "MOCK_SUBSCRIPTION": SUBSCRIPTION, + "MOCK_RESOURCE_GROUP": RESOURCE_GROUP, + "MOCK_FRONT_DOOR_COUNT": "0", + "MOCK_PIP_ID": PIP, + "MOCK_REVISION": "", + "MOCK_DEPLOYED_REVISION": REVISION, + "MOCK_REVISION_IMAGE": "", + "MOCK_FINAL_LATEST": "", + "MOCK_FINAL_READY": "", + "MOCK_FINAL_IMAGE": "", + "MOCK_FINAL_ACCESS": "", + "MOCK_HEALTH": "Healthy", + "MOCK_ACA_HOST": ACA_HOST, + "MOCK_AFD_HOST": AFD_HOST, + "MOCK_PL_MESSAGE": "Azure Front Door private access to copyrit-test", + "MOCK_HTTP_STATUS": "200", + "MOCK_HTTP_EXIT": "0", + "MOCK_FAIL": "", } - self.values = { - "id": SUBSCRIPTION, - "properties.managedEnvironmentId": ENVIRONMENT, - "properties.configuration.activeRevisionsMode": "Single", - "length(properties.template.containers)": "1", - "properties.template.containers[0].name": "pyrit-gui", - "properties.configuration.ingress.fqdn": ACA_HOST, - "properties.publicNetworkAccess": "Enabled", - "properties.latestRevisionName": REVISION, - "properties.latestReadyRevisionName": REVISION, - "properties.healthState": "Healthy", - "value[?properties.enabledState=='Enabled'].properties.hostName": AFD_HOST, + self.fixtures = { + "MOCK_APP": { + "id": APP, + "environmentId": ENVIRONMENT, + "tags": tags, + "mode": "Single", + "revision": PREVIOUS_REVISION, + "containers": [{"name": "pyrit-gui", "image": PREVIOUS_IMAGE}], + }, + "MOCK_ENVIRONMENT": {"id": ENVIRONMENT, "publicNetworkAccess": "Enabled"}, + "MOCK_VNET": {"id": VNET, "prefix": "10.20.0.0/16", "tags": tags}, + "MOCK_SUBNET": {"id": SUBNET, "prefix": "10.20.0.0/23", "natId": NAT}, + "MOCK_NAT": {"id": NAT, "pipId": PIP, "tags": tags}, + "MOCK_PIP": {"id": PIP, "ip": "203.0.113.10", "allocation": "Static", "sku": "Standard", "tags": tags}, + "MOCK_WHAT_IF": { + "changes": [ + {"changeType": "Modify", "resourceId": APP, "delta": [{"path": "properties.configuration"}]}, + {"changeType": "NoChange", "resourceId": ENVIRONMENT}, + ] + }, + "MOCK_ORIGIN": {"status": "Approved", "resourceId": ENVIRONMENT}, + "MOCK_CONNECTIONS": [ + { + "id": f"{ENVIRONMENT}/privateEndpointConnections/connection-1", + "properties": { + "privateLinkServiceConnectionState": { + "status": "Approved", + "description": self.environment["MOCK_PL_MESSAGE"], + } + }, + } + ], } - self.current_image = PREVIOUS_IMAGE - self.calls: list[tuple[str, ...]] = [] - self.az_patch = patch.object(deploy_code, "_az", side_effect=self._az) - self.az_patch.start() - self.addCleanup(self.az_patch.stop) - self.http = patch.object(deploy_code, "_wait_for_http_health").start() - self.addCleanup(patch.stopall) - - def _az(self, *arguments: str) -> str: - self.calls.append(arguments) - if arguments[:2] == ("containerapp", "update"): - self.current_image = arguments[arguments.index("--image") + 1] - return "" - query = arguments[arguments.index("--query") + 1] - if query == "properties.template.containers[0].image": - return self.current_image - return self.values[query] - - def test_public_mode_updates_only_image_and_checks_aca(self) -> None: - deploy_code.deploy_code(**self.arguments) - - updates = [call for call in self.calls if call[:2] == ("containerapp", "update")] - assert updates == [ - ( - "containerapp", - "update", - "--resource-group", - "copyrit-test", - "--name", - "copyrit-test", - "--container-name", - "pyrit-gui", - "--image", - IMAGE, - "--output", - "none", + + def _run(self, **overrides: str) -> subprocess.CompletedProcess[str]: + assert BASH is not None and JQ is not None + environment = ( + os.environ + | self.environment + | {name: json.dumps(value) for name, value in self.fixtures.items()} + | overrides + ) + environment["MSYS2_ARG_CONV_EXCL"] = "*" + environment.pop("BASH_ENV", None) + with tempfile.TemporaryDirectory(prefix=".deployment-test-", dir=REPO_ROOT) as directory: + fixture_dir = Path(directory) + environment["MOCK_DIRECTORY"] = fixture_dir.as_posix() + environment["PYRIT_AGENT_TEMP_DIRECTORY"] = fixture_dir.as_posix() + (fixture_dir / "image").write_text(PREVIOUS_IMAGE, encoding="utf-8") + (fixture_dir / "revision").write_text(PREVIOUS_REVISION, encoding="utf-8") + access = json.loads(environment["MOCK_ENVIRONMENT"])["publicNetworkAccess"] + (fixture_dir / "access").write_text(access, encoding="utf-8") + for log in ("az", "curl"): + (fixture_dir / f"{log}.log").touch() + jq_flags = "-b" if os.name == "nt" else "" + wrappers = ( + f'python3() {{ {shlex.quote(Path(sys.executable).as_posix())} "$@"; }}\n' + f'jq() {{ {shlex.quote(Path(JQ).as_posix())} {jq_flags} "$@"; }}\n' ) - ] - self.http.assert_called_once_with(f"https://{ACA_HOST}/api/health") - assert not any(call[0] in {"deployment", "network", "rest"} for call in self.calls) + result = subprocess.run( + [BASH, "--noprofile", "--norc", "-s"], + input=wrappers + HARNESS + f"\nsource {shlex.quote(DEPLOY_SCRIPT.as_posix())}\n", + capture_output=True, + text=True, + check=False, + env=environment, + timeout=60, + ) + self.az_calls, self.curl_calls = [ + [line.rstrip("\t").split("\t") for line in (fixture_dir / f"{log}.log").read_text().splitlines()] + for log in ("az", "curl") + ] + assert "Unexpected " not in result.stderr, result.stderr + return result - def test_private_mode_checks_front_door_without_changing_network(self) -> None: - self.values["properties.publicNetworkAccess"] = "Disabled" - self.arguments["slot"] = "prod" + def _assert_app_only_writes(self) -> None: + deployments = [call for call in self.az_calls if call[:3] == ["deployment", "group", "create"]] + assert len(deployments) == 1, self.az_calls + assert "deployInfra=false" in deployments[0] + assert "deployApp=true" in deployments[0] + assert deployments[0][deployments[0].index("--mode") + 1] == "Incremental" + assert "containerImage=" + IMAGE in deployments[0] + assert not any("rollback" in argument for call in self.az_calls for argument in call) + assert not any(call[:2] == ["containerapp", "update"] for call in self.az_calls) + assert not any(call[:2] == ["network", "private-endpoint-connection"] for call in self.az_calls) + assert not any(call[0] == "rest" for call in self.az_calls) - deploy_code.deploy_code(**self.arguments) + def test_app_only_reconciles_config_and_preserves_access_mode(self) -> None: + for access, front_door in (("Enabled", "0"), ("Enabled", "1"), ("Disabled", "1")): + with self.subTest(access=access, front_door=front_door): + self.fixtures["MOCK_ENVIRONMENT"]["publicNetworkAccess"] = access + result = self._run(MOCK_FRONT_DOOR_COUNT=front_door) + assert result.returncode == 0, result.stdout + result.stderr + self._assert_app_only_writes() + deployment = next(call for call in self.az_calls if call[:3] == ["deployment", "group", "create"]) + preview = next(call for call in self.az_calls if call[:3] == ["deployment", "group", "what-if"]) + for call in (preview, deployment): + assert call[call.index("--template-file") + 1].endswith("/infra/main.bicep") + assert f"enableFrontDoor={'true' if front_door == '1' else 'false'}" in call + assert f"disableContainerAppsPublicAccess={'true' if access == 'Disabled' else 'false'}" in call + assert "sqlDatabaseName=copyrit" in call + assert self.az_calls.index(preview) < self.az_calls.index(deployment) + expected_host = AFD_HOST if access == "Disabled" else ACA_HOST + assert self.curl_calls[0][-1] == f"https://{expected_host}/api/health" + assert [call[-1] for call in self.curl_calls] == ( + [f"https://{AFD_HOST}/api/health", f"https://{ACA_HOST}/api/health"] + if access == "Disabled" + else [f"https://{ACA_HOST}/api/health"] + ) + assert all("--location" not in call and "--insecure" not in call for call in self.curl_calls) + assert "Deployment healthy:" in result.stdout - self.http.assert_called_once_with(f"https://{AFD_HOST}/api/health") - rest_calls = [call for call in self.calls if call[0] == "rest"] - assert len(rest_calls) == 2 - assert all(call[1:3] == ("--method", "get") for call in rest_calls) - assert not any(call[0] in {"deployment", "network"} for call in self.calls) + def test_infrastructure_then_app_deploys_the_app_once(self) -> None: + infrastructure_preview = json.dumps( + { + "changes": [ + {"changeType": "Ignore", "resourceId": APP}, + {"changeType": "NoChange", "resourceId": APP + "/authConfigs/current"}, + { + "changeType": "Modify", + "resourceId": ENVIRONMENT, + "delta": [{"path": "properties.publicNetworkAccess"}], + }, + ] + } + ) + result = self._run( + PYRIT_DEPLOY_INFRA="true", PYRIT_CONTAINER_IMAGE="ignored:mutable", MOCK_WHAT_IF=infrastructure_preview + ) + assert result.returncode == 0, result.stdout + result.stderr + deployments = [call for call in self.az_calls if call[:3] == ["deployment", "group", "create"]] + assert len(deployments) == 1 + for call in [deployments[0], next(c for c in self.az_calls if c[:3] == ["deployment", "group", "what-if"])]: + assert call[call.index("--template-file") + 1].endswith("/infra/main.bicep") + assert "deployInfra=true" in call + assert "deployApp=false" in call + assert not any(argument.startswith("containerImage=") for argument in call) + assert call[call.index("--mode") + 1] == "Incremental" + assert "enableFrontDoorPrivateLink=true" in call + assert "disableContainerAppsPublicAccess=true" in call + assert any(call[:2] == ["network", "private-endpoint-connection"] for call in self.az_calls) + assert any(call[:3] == ["rest", "--method", "get"] for call in self.az_calls) + assert self.curl_calls[0][-1] == f"https://{AFD_HOST}/api/health" + assert f"app revision unchanged: {PREVIOUS_REVISION}" in result.stdout - def test_same_image_still_verified_without_update(self) -> None: - self.current_image = IMAGE + self.fixtures["MOCK_ENVIRONMENT"]["publicNetworkAccess"] = "Disabled" + result = self._run(MOCK_FRONT_DOOR_COUNT="1") + assert result.returncode == 0, result.stdout + result.stderr + self._assert_app_only_writes() + deployments += [call for call in self.az_calls if call[:3] == ["deployment", "group", "create"]] + assert [call for call in deployments if "deployApp=true" in call] == [deployments[1]] + assert f"Deployment healthy: {REVISION}" in result.stdout - deploy_code.deploy_code(**self.arguments) + def test_infrastructure_preview_rejects_app_and_child_writes(self) -> None: + for resource_id in (APP, APP.upper() + "/", APP + "/authConfigs/current"): + with self.subTest(resource_id=resource_id): + preview = {"changes": [{"changeType": "Modify", "resourceId": resource_id}]} + result = self._run(PYRIT_DEPLOY_INFRA="true", MOCK_WHAT_IF=json.dumps(preview)) + assert result.returncode != 0 + assert "Infrastructure-only preview must not write the Container App" in result.stdout + assert not any(call[:3] == ["deployment", "group", "create"] for call in self.az_calls) - assert not any(call[:2] == ("containerapp", "update") for call in self.calls) - self.http.assert_called_once() + def test_infrastructure_failures_roll_back_without_deploying_app(self) -> None: + for overrides, message in ( + ({"MOCK_REVISION": REVISION}, "changed the running app revision"), + ({"MOCK_HTTP_STATUS": "504"}, "Application endpoint did not return a healthy response"), + ): + with self.subTest(overrides=overrides): + result = self._run( + PYRIT_DEPLOY_INFRA="true", + MOCK_WHAT_IF=json.dumps({"changes": [{"changeType": "NoChange", "resourceId": APP}]}), + **overrides, + ) + assert result.returncode != 0 + assert message in result.stdout, result.stdout + result.stderr + assert "Public ACA origin rollback completed" in result.stdout + deployments = [call for call in self.az_calls if call[:3] == ["deployment", "group", "create"]] + assert len(deployments) == 3 + for call in deployments: + assert not any(argument.startswith("containerImage=") for argument in call) + if call[call.index("--template-file") + 1].endswith("/infra/main.bicep"): + assert "deployApp=false" in call + assert "deployInfra=true" in call + assert call[call.index("--mode") + 1] == "Incremental" - def test_invalid_inputs_fail_before_azure_calls(self) -> None: - invalid = { - "slot": "other", - "resource_group": "../wrong", - "app_name": "wrong/name", - "acr_resource_id": "/subscriptions/wrong", - "image": "another.azurecr.io/pyrit:latest", - } - for key, value in invalid.items(): - with self.subTest(key=key), self.assertRaises(ValueError): - deploy_code.deploy_code(**(self.arguments | {key: value})) - assert not self.calls - - def test_invalid_existing_topology_never_updates(self) -> None: - invalid = { - "id": "22222222-2222-2222-2222-222222222222", - "properties.managedEnvironmentId": ENVIRONMENT + "-other", - "properties.configuration.activeRevisionsMode": "Multiple", - "length(properties.template.containers)": "2", - "properties.template.containers[0].name": "other", - "properties.publicNetworkAccess": "unexpected", - "properties.configuration.ingress.fqdn": "attacker.example", - } - for query, value in invalid.items(): - with self.subTest(query=query), patch.dict(self.values, {query: value}), self.assertRaises(ValueError): - deploy_code.deploy_code(**self.arguments) - assert not any(call[:2] == ("containerapp", "update") for call in self.calls) - - def test_private_mode_requires_one_valid_front_door_endpoint(self) -> None: - self.values["properties.publicNetworkAccess"] = "Disabled" - query = "value[?properties.enabledState=='Enabled'].properties.hostName" - for value in ("", "None", f"{AFD_HOST}\nother.azurefd.net", "attacker.example"): - with self.subTest(value=value), patch.dict(self.values, {query: value}), self.assertRaises(ValueError): - deploy_code.deploy_code(**self.arguments) - assert not any(call[:2] == ("containerapp", "update") for call in self.calls) - - def test_unhealthy_revision_fails_without_rollback(self) -> None: - self.values["properties.healthState"] = "Unhealthy" - with patch.object(deploy_code.time, "sleep"), self.assertRaisesRegex(RuntimeError, "revision"): - deploy_code.deploy_code(**self.arguments) - assert self.current_image == IMAGE - assert len([call for call in self.calls if call[:2] == ("containerapp", "update")]) == 1 - self.http.assert_not_called() - assert not any(call[0] in {"deployment", "network", "rest"} for call in self.calls) - - def test_http_failure_does_not_fallback_or_roll_back(self) -> None: - self.values["properties.publicNetworkAccess"] = "Disabled" - self.http.side_effect = RuntimeError("Health check failed") - with self.assertRaisesRegex(RuntimeError, "Health check"): - deploy_code.deploy_code(**self.arguments) - self.http.assert_called_once_with(f"https://{AFD_HOST}/api/health") - assert self.current_image == IMAGE - assert len([call for call in self.calls if call[:2] == ("containerapp", "update")]) == 1 - - def test_current_ready_revision_must_be_the_verified_revision(self) -> None: - self.values["properties.latestReadyRevisionName"] = "copyrit-test--old" - with self.assertRaisesRegex(RuntimeError, "current ready revision"): - deploy_code.deploy_code(**self.arguments) - - def test_access_mode_change_is_not_reported_as_success(self) -> None: - self.http.side_effect = lambda _: self.values.update({"properties.publicNetworkAccess": "Disabled"}) - with self.assertRaisesRegex(RuntimeError, "access mode changed"): - deploy_code.deploy_code(**self.arguments) - - def test_main_reports_unresolved_input_without_azure_calls(self) -> None: - with patch.dict(os.environ, {"PYRIT_SLOT": "$(slot)"}): - assert deploy_code.main() == 1 - assert not self.calls - - -class TestHttpVerification(unittest.TestCase): - def test_http_success_does_not_follow_redirects(self) -> None: - response = subprocess.CompletedProcess(args=[], returncode=0, stdout="200", stderr="") - with patch.object(deploy_code.subprocess, "run", return_value=response) as run: - deploy_code._wait_for_http_health("https://example.azurefd.net/api/health") - arguments = run.call_args.args[0] - assert "--location" not in arguments - assert "--insecure" not in arguments - assert "--max-time" in arguments - - def test_http_errors_and_redirects_exhaust_the_bounded_budget(self) -> None: - for status, exit_code in (("302", 0), ("504", 0), ("200", 28)): - response = subprocess.CompletedProcess(args=[], returncode=exit_code, stdout=status, stderr="failure") - with ( - self.subTest(status=status, exit_code=exit_code), - patch.object(deploy_code.subprocess, "run", return_value=response), - patch.object(deploy_code.time, "monotonic", side_effect=[0, 290, 299, 301]), - patch.object(deploy_code.time, "sleep"), - self.assertRaisesRegex(RuntimeError, "health check failed"), - ): - deploy_code._wait_for_http_health("https://example.azurefd.net/api/health") + def test_invalid_inputs_or_topology_fail_before_deployment(self) -> None: + cases = [ + ({"PYRIT_SLOT": "$(slot)"}, "Required deployment value"), + ({"PYRIT_CONTAINER_IMAGE": "copyritacr.azurecr.io/pyrit:latest"}, "immutable registry digest"), + ({"PYRIT_CONTAINER_IMAGE": IMAGE.replace("copyritacr", "otheracr")}, "registry does not match"), + ({"PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX": "10.30.0.0/23"}, "Invalid network prefix"), + ({"MOCK_SUBSCRIPTION": "22222222-2222-2222-2222-222222222222"}, "subscription does not match"), + ({"MOCK_SUBNET": json.dumps(self.fixtures["MOCK_SUBNET"] | {"natId": NAT + "-other"})}, "topology"), + ({"MOCK_APP": json.dumps(self.fixtures["MOCK_APP"] | {"mode": "Multiple"})}, "single-revision"), + ({"MOCK_APP": json.dumps(self.fixtures["MOCK_APP"] | {"containers": []})}, "single-revision"), + ] + for overrides, message in cases: + with self.subTest(overrides=overrides): + result = self._run(**overrides) + assert result.returncode != 0 + assert message in result.stdout, result.stdout + result.stderr + assert not any(call[:2] == ["deployment", "group"] for call in self.az_calls) + assert not self.curl_calls + + def test_app_only_what_if_rejects_infrastructure_and_protected_changes(self) -> None: + cases = [ + (ENVIRONMENT, "Modify", "properties.publicNetworkAccess", "App-only preview"), + ( + f"{RESOURCE_GROUP}/providers/Microsoft.Cdn/profiles/copyrit-test-afd", + "Create", + "sku", + "App-only preview", + ), + (SUBNET, "Modify", "properties.addressPrefix", "protected-network change"), + ] + for resource_id, change_type, path, message in cases: + with self.subTest(resource_id=resource_id): + payload = { + "changes": [{"resourceId": resource_id, "changeType": change_type, "delta": [{"path": path}]}] + } + result = self._run(MOCK_WHAT_IF=json.dumps(payload)) + assert result.returncode != 0 + assert message in result.stdout, result.stdout + result.stderr + assert not any(call[:3] == ["deployment", "group", "create"] for call in self.az_calls) + assert not self.curl_calls + + def test_revision_and_current_ready_image_must_match(self) -> None: + cases = [ + ({"MOCK_REVISION_IMAGE": PREVIOUS_IMAGE}, "requested image", False), + ({"MOCK_HEALTH": "Unhealthy"}, "did not become healthy", False), + ({"MOCK_FINAL_READY": "copyrit-test--old"}, "current ready revision", True), + ({"MOCK_FINAL_LATEST": "copyrit-test--other"}, "current ready revision", True), + ({"MOCK_FINAL_IMAGE": PREVIOUS_IMAGE}, "current ready revision", True), + ({"MOCK_FINAL_ACCESS": "Disabled"}, "access mode changed", True), + ] + for overrides, message, probes_expected in cases: + with self.subTest(overrides=overrides): + result = self._run(**overrides) + assert result.returncode != 0 + assert message in result.stdout, result.stdout + result.stderr + assert bool(self.curl_calls) is probes_expected + self._assert_app_only_writes() + + def test_private_http_failure_never_falls_back_or_rolls_back(self) -> None: + self.fixtures["MOCK_ENVIRONMENT"]["publicNetworkAccess"] = "Disabled" + for status, exit_code in (("302", "0"), ("504", "0"), ("200", "28")): + with self.subTest(status=status, exit_code=exit_code): + result = self._run(MOCK_FRONT_DOOR_COUNT="1", MOCK_HTTP_STATUS=status, MOCK_HTTP_EXIT=exit_code) + assert result.returncode != 0 + assert "Application endpoint did not return a healthy response" in result.stdout + assert "Deployment healthy:" not in result.stdout + assert 1 <= len(self.curl_calls) <= 10 + assert all(call[-1] == f"https://{AFD_HOST}/api/health" for call in self.curl_calls) + self._assert_app_only_writes() + + def test_cli_failures_do_not_report_success_or_roll_back(self) -> None: + for command in ( + "resource show", + "deployment group what-if", + "deployment group create", + "containerapp revision show", + ): + with self.subTest(command=command): + result = self._run(MOCK_FAIL=command) + assert result.returncode != 0 + assert "Deployment healthy:" not in result.stdout + assert not self.curl_calls + assert not any("rollback" in argument for call in self.az_calls for argument in call) + assert not any(call[:2] == ["network", "private-endpoint-connection"] for call in self.az_calls) if __name__ == "__main__": diff --git a/tests/unit/infra/test_pipeline_guardrails.py b/tests/unit/infra/test_pipeline_guardrails.py index 0ddb9c740c..1aaeed4d69 100644 --- a/tests/unit/infra/test_pipeline_guardrails.py +++ b/tests/unit/infra/test_pipeline_guardrails.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. """Guard the independent application and infrastructure deployment contract.""" +import itertools import json import os import re @@ -16,7 +17,7 @@ REPO_ROOT = Path(__file__).resolve().parents[3] PIPELINE = REPO_ROOT / "gui-deploy.yml" -DEPLOY_SCRIPT = REPO_ROOT / "infra" / "pipelines" / "deploy_public_nat.sh" +DEPLOY_SCRIPT = REPO_ROOT / "infra" / "pipelines" / "deploy_gui.sh" INFRA_TEMPLATE = REPO_ROOT / "infra" / "pipelines" / "deploy-infra.yml" WHAT_IF_VALIDATOR = REPO_ROOT / "infra" / "pipelines" / "validate_what_if.py" EXAMPLE_PARAMETERS = REPO_ROOT / "infra" / "parameters.example.json" @@ -50,55 +51,143 @@ class TestPipelineGuardrails(unittest.TestCase): """Verify one preview-first test/prod deployment workflow.""" @classmethod - def setUpClass(cls): + def setUpClass(cls) -> None: cls.pipeline = PIPELINE.read_text(encoding="utf-8") cls.deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") cls.infra_template = INFRA_TEMPLATE.read_text(encoding="utf-8") + cls.pipeline_yaml = yaml.safe_load(cls.pipeline) + cls.infra_yaml = yaml.safe_load(cls.infra_template) - def test_pipeline_has_one_test_and_prod_workflow(self): + def test_pipeline_has_one_test_and_prod_workflow(self) -> None: assert "deploymentTarget" not in self.pipeline assert "applyReplacement" not in self.pipeline - assert "stage: Build" in self.pipeline - assert "stage: DeployTest" in self.pipeline - assert "stage: ApproveProd" in self.pipeline - assert "stage: DeployProd" in self.pipeline - assert "DeployReplacement" not in self.pipeline - assert self.pipeline.count("timeoutInMinutes: 120") == 2 - assert ( - self.pipeline.count('inlineScript: python3 "$(Build.SourcesDirectory)/infra/pipelines/deploy_code.py"') == 2 - ) - assert "deploy_public_nat.sh" not in self.pipeline - assert "scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh'" in self.infra_template + stages = self.pipeline_yaml["stages"] + assert [stage.get("stage") or stage["parameters"]["stageName"] for stage in stages] == [ + "ValidateInputs", + "Build", + "DeployTestInfra", + "DeployTest", + "ApproveProd", + "DeployProdInfra", + "DeployProd", + ] + deployment_stages = [stage for stage in stages if stage.get("stage") in {"DeployTest", "DeployProd"}] + deployment_stages += self.infra_yaml["stages"] + for stage in deployment_stages: + job = stage["jobs"][0] + assert "deployment" in job + assert job["timeoutInMinutes"] == 120 + steps = job["strategy"]["runOnce"]["deploy"]["steps"] + task = next(step for step in steps if step.get("task") == "AzureCLI@2") + assert task["inputs"] == { + "azureSubscription": "$(azureServiceConnection)", + "scriptType": "bash", + "scriptLocation": "scriptPath", + "scriptPath": "$(Build.SourcesDirectory)/infra/pipelines/deploy_gui.sh", + } + is_infrastructure = stage in self.infra_yaml["stages"] + assert task["env"]["PYRIT_DEPLOY_INFRA"] == ("true" if is_infrastructure else "false") + if is_infrastructure: + assert "PYRIT_CONTAINER_IMAGE" not in task["env"] + else: + assert task["env"]["PYRIT_CONTAINER_IMAGE"] == "$(immutableImage)" + assert "deploy_code.py" not in self.pipeline + assert "deploy_public_nat.sh" not in self.pipeline + self.infra_template def test_infrastructure_is_explicit_and_runs_before_code(self) -> None: - pipeline = yaml.safe_load(self.pipeline) + pipeline = self.pipeline_yaml parameters = {parameter["name"]: parameter for parameter in pipeline["parameters"]} assert parameters["deployInfra"]["default"] is False assert parameters["deployToProd"]["default"] is False stages = pipeline["stages"] - conditional = "${{ if eq(parameters.deployInfra, true) }}" - infrastructure = [stage[conditional][0] for stage in stages if conditional in stage] + infrastructure = [stage for stage in stages if "template" in stage] assert [stage["parameters"] for stage in infrastructure] == [ - {"stageName": "DeployTestInfra", "dependsOn": "Build", "slot": "test"}, - {"stageName": "DeployProdInfra", "dependsOn": "ApproveProd", "slot": "prod"}, + { + "stageName": "DeployTestInfra", + "dependsOn": "Build", + "slot": "test", + "deployInfra": "${{ parameters.deployInfra }}", + }, + { + "stageName": "DeployProdInfra", + "dependsOn": "ApproveProd", + "slot": "prod", + "deployInfra": "${{ parameters.deployInfra }}", + }, ] assert all(stage["template"] == "infra/pipelines/deploy-infra.yml" for stage in infrastructure) code_stages = {stage["stage"]: stage for stage in stages if stage.get("stage") in {"DeployTest", "DeployProd"}} - assert code_stages["DeployTest"]["dependsOn"] == ["Build", {conditional: ["DeployTestInfra"]}] - assert code_stages["DeployProd"]["dependsOn"] == [ - "ApproveProd", - "Build", - {conditional: ["DeployProdInfra"]}, - ] - template = yaml.safe_load(self.infra_template)["stages"][0] + assert code_stages["DeployTest"]["dependsOn"] == ["Build", "DeployTestInfra"] + assert code_stages["DeployProd"]["dependsOn"] == ["ApproveProd", "Build", "DeployProdInfra"] + template = self.infra_yaml["stages"][0] assert template["dependsOn"] == "${{ parameters.dependsOn }}" - assert "condition" not in template # Infrastructure must not run after a skipped approval. + assert " ".join(template["condition"].split()) == ( + "and(succeeded(), eq('${{ parameters.deployInfra }}', 'true'))" + ) assert "PYRIT_CONTAINER_IMAGE" not in self.infra_template assert "current_image=$(jq" in self.deploy_script - assert "retaining current image" in self.deploy_script - assert "PYRIT_CONTAINER_IMAGE" not in self.deploy_script - - def test_production_remains_opt_in_and_independently_approved(self): + assert "leaving the running application unchanged" in self.deploy_script + assert "requested_image=$current_image" in self.deploy_script + assert "requested_image=${PYRIT_CONTAINER_IMAGE:-}" in self.deploy_script + assert '"deployInfra=$PYRIT_DEPLOY_INFRA"' in self.deploy_script + assert '"deployApp=$deploy_app"' in self.deploy_script + assert ( + 'if [[ "$deploy_app" == "true" ]]; then\n parameters+=("containerImage=$immutable_image")\nfi' + in self.deploy_script + ) + assert "az containerapp update" not in self.deploy_script + + def test_app_stage_conditions_distinguish_failure_cancellation_and_intentional_skip(self) -> None: + success = {"Succeeded", "SucceededWithIssues"} + results = ["Succeeded", "SucceededWithIssues", "Failed", "Canceled", "Skipped"] + for stage in self.pipeline_yaml["stages"]: + if stage.get("stage") not in {"DeployTest", "DeployProd"}: + continue + infrastructure = stage["stage"] + "Infra" + condition = " ".join(stage["condition"].split()) + for deploy_infra, canceled, build, infra, approval in itertools.product( + (False, True), + (False, True), + results, + results, + results, + ): + with self.subTest( + stage=stage["stage"], + flag=deploy_infra, + canceled=canceled, + build=build, + infra=infra, + approval=approval, + ): + expression = condition.replace("not(canceled())", str(not canceled)) + expression = expression.replace( + "eq('${{ parameters.deployInfra }}', 'false')", str(not deploy_infra) + ) + expression = expression.replace( + f"eq(dependencies.{infrastructure}.result, 'Skipped')", + str(infra == "Skipped"), + ) + for dependency, outcome in (("Build", build), (infrastructure, infra), ("ApproveProd", approval)): + expression = expression.replace( + f"in(dependencies.{dependency}.result, 'Succeeded', 'SucceededWithIssues')", + str(outcome in success), + ) + expression = expression.replace("and(", "all_(").replace("or(", "any_(") + # Evaluate only these two known conditions, not a general Azure expression language. + assert re.fullmatch(r"(?:True|False|all_|any_|[(),\s])+", expression), expression + actual = eval( + expression, {"__builtins__": {}, "all_": lambda *v: all(v), "any_": lambda *v: any(v)} + ) + expected = ( + not canceled + and build in success + and (infra in success or (not deploy_infra and infra == "Skipped")) + and (stage["stage"] != "DeployProd" or approval in success) + ) + assert actual is expected + + def test_production_remains_opt_in_and_independently_approved(self) -> None: assert "job: ValidateProdConfiguration" in self.pipeline assert "copyrit-gui-prod must define prodApprovers" in self.pipeline assert "PROD_APPROVERS: $(prodApprovers)" in self.pipeline @@ -108,16 +197,18 @@ def test_production_remains_opt_in_and_independently_approved(self): assert "approvers: '$(prodApprovers)'" in self.pipeline assert "allowApproversToApproveTheirOwnRuns: false" in self.pipeline assert "dependsOn: ValidateProdConfiguration" in self.pipeline - approval_stage = self.pipeline[ - self.pipeline.index("stage: ApproveProd") : self.pipeline.index("stage: DeployProd") - ] - assert "- group: copyrit-gui-prod" in approval_stage + approval_stage = next(stage for stage in self.pipeline_yaml["stages"] if stage.get("stage") == "ApproveProd") + assert approval_stage["dependsOn"] == "DeployTest" + assert {"group": "copyrit-gui-prod"} in approval_stage["variables"] + assert " ".join(approval_stage["condition"].split()) == ( + "and(succeeded('DeployTest'), eq('${{ parameters.deployToProd }}', 'true'), " + "eq(variables['Build.SourceBranch'], 'refs/heads/main'))" + ) assert '"$BUILD_SOURCEBRANCH" != refs/heads/main' in self.pipeline assert "eq(variables['Build.SourceBranch'], 'refs/heads/main')" in self.pipeline assert "refs/heads/releases/" not in self.pipeline - assert "condition: and(succeeded(), succeeded('ApproveProd'))" in self.pipeline - def test_deploy_resolves_digest_and_previews_before_apply(self): + def test_deploy_resolves_digest_and_previews_before_apply(self) -> None: assert "name: BuildImage" in self.pipeline assert "variable=immutableImage;isOutput=true" in self.pipeline assert "stageDependencies.Build.BuildAndPush.outputs['BuildImage.immutableImage']" in self.pipeline @@ -133,10 +224,10 @@ def test_deploy_resolves_digest_and_previews_before_apply(self): assert "cross-resource-group write" in self.deploy_script assert "networkMode=" not in self.deploy_script assert "enablePrivateEndpoint=" not in self.deploy_script - assert '"enableFrontDoor=true"' in self.deploy_script - assert '"enableFrontDoorPrivateLink=true"' in self.deploy_script + assert '"enableFrontDoor=$enable_front_door"' in self.deploy_script + assert '"enableFrontDoorPrivateLink=$enable_private_link"' in self.deploy_script assert '"frontDoorPrivateLinkRequestMessage=$private_link_request_message"' in self.deploy_script - assert '"disableContainerAppsPublicAccess=true"' in self.deploy_script + assert '"disableContainerAppsPublicAccess=$disable_public_access"' in self.deploy_script def test_pipeline_passes_values_via_environment(self): deploy_yaml = self.infra_template @@ -182,13 +273,13 @@ def test_deploy_preserves_existing_network_and_tags(self): assert self.deploy_script.index("expected_egress_ip=") < self.deploy_script.index("az deployment group what-if") assert self.deploy_script.index("actual_pip_id=") > self.deploy_script.index("az deployment group create") - def test_data_plane_health_probe_respects_ingress_restrictions(self): + def test_data_plane_health_probe_respects_ingress_restrictions(self) -> None: assert "properties.outputs.frontDoorFqdn.value" in self.deploy_script assert '"https://$front_door_fqdn/api/health"' in self.deploy_script assert "direct_aca_health=$(curl" in self.deploy_script assert '"https://$app_fqdn/api/health"' in self.deploy_script assert '[[ "$direct_aca_health" == "200" ]]' in self.deploy_script - assert "Front Door did not route a healthy response" in self.deploy_script + assert "Application endpoint did not return a healthy response" in self.deploy_script assert "aca_private_endpoint_approval.bicep" in self.deploy_script assert "connection_suffix=${connection_name:0:8}" in self.deploy_script assert '"$deployment_name-private-link-approval-$connection_suffix"' in self.deploy_script @@ -214,26 +305,32 @@ def test_data_plane_health_probe_respects_ingress_restrictions(self): assert self.deploy_script.index(deletion_guard) < self.deploy_script.index( 'if az deployment group create \\\n --name "$deployment_name-rollback"' ) - assert "front_door_health_timeout_seconds=1800" in self.deploy_script - assert "front_door_health_deadline=$((SECONDS + front_door_health_timeout_seconds))" in self.deploy_script - assert "while ((SECONDS < front_door_health_deadline))" in self.deploy_script + assert "health_timeout_seconds=300" in self.deploy_script + assert '[[ "$PYRIT_DEPLOY_INFRA" == "true" ]] && health_timeout_seconds=1800' in self.deploy_script + assert "health_deadline=$((SECONDS + health_timeout_seconds))" in self.deploy_script + assert "while ((SECONDS < health_deadline))" in self.deploy_script assert "budget before request" in self.deploy_script assert "Front Door health attempt $attempt/60" not in self.deploy_script assert "Direct ACA public access remains reachable" in self.deploy_script - assert "ACA public access: disabled" in self.deploy_script + assert "ACA public access: $expected_public_access" in self.deploy_script + assert ( + 'if [[ "$PYRIT_DEPLOY_INFRA" == "true" ]]; then\n' + " trap rollback_public_origin EXIT\n" + " trap 'exit 143' TERM\n" + " trap 'exit 130' INT\n" + " cutover_in_progress=true\nfi" + ) in self.deploy_script def _run_cancellation_rollback(self, *, connection_count: int) -> tuple[subprocess.CompletedProcess[str], str]: assert BASH is not None lowercase_start = self.deploy_script.index("lowercase() {") lowercase_end = self.deploy_script.index("\n}\n", lowercase_start) + len("\n}\n") function_start = self.deploy_script.index("rollback_public_origin() {") - trap_start = self.deploy_script.index("trap rollback_public_origin EXIT", function_start) - trap_end = self.deploy_script.index("cutover_in_progress=true", trap_start) + function_end = self.deploy_script.index("\n}\n", function_start) + len("\n}\n") lowercase_function = self.deploy_script[lowercase_start:lowercase_end] - rollback_function = self.deploy_script[function_start:trap_start] - trap_setup = self.deploy_script[trap_start:trap_end] + rollback_function = self.deploy_script[function_start:function_end] - with tempfile.TemporaryDirectory() as directory: + with tempfile.TemporaryDirectory(prefix=".deployment-test-", dir=REPO_ROOT) as directory: call_log = Path(directory) / "az-calls.log" harness = f""" set -euo pipefail @@ -247,7 +344,7 @@ def _run_cancellation_rollback(self, *, connection_count: int) -> tuple[subproce normalized_expected_environment_id=$(lowercase "$expected_environment_id") deployment_name='pyrit-test-1' deployment_tags='{{}}' -rollback_parameters=('disableContainerAppsPublicAccess=false') +rollback_parameters=('deployInfra=true' 'deployApp=false' 'disableContainerAppsPublicAccess=false') az() {{ printf '%s\n' "$*" >> "$AZ_CALLS" @@ -266,19 +363,23 @@ def _run_cancellation_rollback(self, *, connection_count: int) -> tuple[subproce sleep() {{ :; }} {rollback_function} -{trap_setup} +trap rollback_public_origin EXIT +trap 'exit 143' TERM +trap 'exit 130' INT cutover_in_progress=true kill -TERM $$ """ environment = os.environ.copy() - environment["AZ_CALLS"] = str(call_log) + environment["AZ_CALLS"] = call_log.as_posix() + environment["MSYS2_ARG_CONV_EXCL"] = "*" result = subprocess.run( - [BASH, "-s"], + [BASH, "--noprofile", "--norc", "-s"], input=harness, capture_output=True, text=True, check=False, env=environment, + timeout=30, ) calls = call_log.read_text(encoding="utf-8") @@ -303,6 +404,9 @@ def test_cancellation_reenables_public_access_after_connection_deletion(self): assert "rest --method delete" in calls assert re.search(r"--name pyrit-test-1-rollback(?:\s|$)", calls) assert "disableContainerAppsPublicAccess=false" in calls + assert "deployApp=false" in calls + assert "containerImage=" not in calls + assert "--mode Incremental" in calls def test_manual_parameter_files_use_the_single_topology(self): example = json.loads(EXAMPLE_PARAMETERS.read_text(encoding="utf-8")) @@ -340,7 +444,7 @@ def _run_what_if_validator( *, expected_subnet_id: str = SUBNET_ID, ) -> subprocess.CompletedProcess[str]: - with tempfile.TemporaryDirectory() as directory: + with tempfile.TemporaryDirectory(prefix=".deployment-test-", dir=REPO_ROOT) as directory: what_if_file = Path(directory) / "what-if.json" what_if_file.write_text(json.dumps({"changes": changes}), encoding="utf-8") return subprocess.run(