diff --git a/gui-deploy.yml b/gui-deploy.yml index 92b618bb73..3325beea3b 100644 --- a/gui-deploy.yml +++ b/gui-deploy.yml @@ -1,9 +1,7 @@ # CI/CD pipeline for the CoPyRIT GUI. # -# Every deployment uses the single topology defined by infra/main.bicep: -# public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. -# Environment variable groups also provide adminGroupObjectId for configuration -# authorization and optional pyritConfigFileUri for blob-backed configuration. +# Code-only test deployment is the default. Infrastructure reconciliation is +# explicit and runs with the current image before deploying new code. trigger: branches: @@ -19,6 +17,10 @@ trigger: pr: none parameters: + - name: deployInfra + displayName: 'Deploy Infrastructure' + type: boolean + default: false - name: deployToProd displayName: 'Deploy to production' type: boolean @@ -123,9 +125,19 @@ stages: immutable_image="$PYRIT_ACR_LOGIN_SERVER/$PYRIT_IMAGE_NAME@$digest" echo "##vso[task.setvariable variable=immutableImage;isOutput=true]$immutable_image" + - ${{ if eq(parameters.deployInfra, true) }}: + - template: infra/pipelines/deploy-infra.yml + parameters: + stageName: DeployTestInfra + dependsOn: Build + slot: test + - stage: DeployTest - displayName: 'Deploy to Test' - dependsOn: Build + displayName: 'Deploy code to Test' + dependsOn: + - Build + - ${{ if eq(parameters.deployInfra, true) }}: + - DeployTestInfra variables: - group: copyrit-gui-common - group: copyrit-gui-test @@ -135,7 +147,7 @@ stages: vmImage: 'ubuntu-latest' jobs: - deployment: DeployToTest - displayName: 'Deploy test environment' + displayName: 'Deploy test application' timeoutInMinutes: 120 environment: 'copyrit-test' strategy: @@ -146,35 +158,18 @@ stages: fetchDepth: 1 - task: AzureCLI@2 - displayName: 'Preview, deploy, and verify test' + displayName: 'Deploy and verify test code without changing networking' env: PYRIT_SLOT: test - PYRIT_BUILD_ID: $(Build.BuildId) - PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) - PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) PYRIT_APP_NAME: $(deploymentAppName) PYRIT_CONTAINER_IMAGE: $(immutableImage) - PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) - PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) - PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) - PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) - PYRIT_ENTRA_TENANT_ID: $(entraTenantId) - PYRIT_ENTRA_CLIENT_ID: $(entraClientId) - PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) - PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId) - PYRIT_CONFIG_FILE_URI: $(pyritConfigFileUri) - PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) - PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) - PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) PYRIT_ACR_RESOURCE_ID: $(acrResourceId) - PYRIT_ENABLE_OTEL: $(enableOtel) - PYRIT_ENV_SECRET_NAME: $(envSecretName) inputs: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' - scriptLocation: 'scriptPath' - scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' + scriptLocation: 'inlineScript' + inlineScript: python3 "$(Build.SourcesDirectory)/infra/pipelines/deploy_code.py" - stage: ApproveProd displayName: 'Approve Production Deployment' @@ -213,16 +208,28 @@ stages: allowApproversToApproveTheirOwnRuns: false instructions: | Confirm test is healthy, Entra and backend group authorization work, - the static test egress IP is allow-listed, and production variables - describe the same public ACA plus fixed NAT topology. + and the static test egress IP is allow-listed. + Infrastructure deployment requested: ${{ parameters.deployInfra }}. + Code-only deployment preserves the existing public/private access mode. + If infrastructure is requested, approve the production Private Link + cutover and configuration changes before the new code is deployed. onTimeout: reject + - ${{ if eq(parameters.deployInfra, true) }}: + - template: infra/pipelines/deploy-infra.yml + parameters: + stageName: DeployProdInfra + dependsOn: ApproveProd + slot: prod + - stage: DeployProd - displayName: 'Deploy to Production' + displayName: 'Deploy code to Production' dependsOn: - ApproveProd - Build - condition: succeeded('ApproveProd') + - ${{ if eq(parameters.deployInfra, true) }}: + - DeployProdInfra + condition: and(succeeded(), succeeded('ApproveProd')) variables: - group: copyrit-gui-common - group: copyrit-gui-prod @@ -232,7 +239,7 @@ stages: vmImage: 'ubuntu-latest' jobs: - deployment: DeployToProd - displayName: 'Deploy production environment' + displayName: 'Deploy production application' timeoutInMinutes: 120 environment: 'copyrit-prod' strategy: @@ -243,32 +250,15 @@ stages: fetchDepth: 1 - task: AzureCLI@2 - displayName: 'Preview, deploy, and verify production' + displayName: 'Deploy and verify production code without changing networking' env: PYRIT_SLOT: prod - PYRIT_BUILD_ID: $(Build.BuildId) - PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) - PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) PYRIT_APP_NAME: $(deploymentAppName) PYRIT_CONTAINER_IMAGE: $(immutableImage) - PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) - PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) - PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) - PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) - PYRIT_ENTRA_TENANT_ID: $(entraTenantId) - PYRIT_ENTRA_CLIENT_ID: $(entraClientId) - PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) - PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId) - PYRIT_CONFIG_FILE_URI: $(pyritConfigFileUri) - PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) - PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) - PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) PYRIT_ACR_RESOURCE_ID: $(acrResourceId) - PYRIT_ENABLE_OTEL: $(enableOtel) - PYRIT_ENV_SECRET_NAME: $(envSecretName) inputs: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' - scriptLocation: 'scriptPath' - scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' + scriptLocation: 'inlineScript' + inlineScript: python3 "$(Build.SourcesDirectory)/infra/pipelines/deploy_code.py" diff --git a/infra/README.md b/infra/README.md index 85725e9578..abccff4ec0 100644 --- a/infra/README.md +++ b/infra/README.md @@ -72,7 +72,7 @@ flowchart TB app -.->|"Traces after agent setup"| appInsights ``` -The base topology is public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. `enableFrontDoor=true` adds Front Door Premium as the preferred managed HTTPS URL. By default, the ACA origin remains concurrently public and can bypass Front Door. `enableFrontDoorPrivateLink=true` instead connects Premium Front Door to the ACA environment through Private Link; setting `disableContainerAppsPublicAccess=true` then removes the direct public ACA path. Bicep rejects public-access shutdown unless both Front Door and its Private Link origin are enabled. The team ADO workflow uses this isolated-origin mode; community examples leave all three Front Door settings disabled. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door rather than the original client. Front Door changes inbound routing only: outbound connections from ACA continue to use the NAT Gateway's static IPv4. +The base topology is public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. `enableFrontDoor=true` adds Front Door Premium as the preferred managed HTTPS URL. By default, the ACA origin remains concurrently public and can bypass Front Door. `enableFrontDoorPrivateLink=true` instead connects Premium Front Door to the ACA environment through Private Link; setting `disableContainerAppsPublicAccess=true` then removes the direct public ACA path. Bicep rejects public-access shutdown unless both Front Door and its Private Link origin are enabled. The team ADO infrastructure stage uses this isolated-origin mode; code-only runs preserve the existing access mode. Community examples leave all three Front Door settings disabled. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door rather than the original client. Front Door changes inbound routing only: outbound connections from ACA continue to use the NAT Gateway's static IPv4. ## Development Workflow @@ -403,19 +403,40 @@ az deployment group show -g -n \ > This section documents the repository maintainers' internal pipeline. It depends on Microsoft team-owned ADO service connections, environments, and variable groups. It is not required or expected for community deployments; use direct Bicep or `deploy_instance.py` instead. -`gui-deploy.yml` is an **update-only** workflow for the pre-created test-v2 and prod-v2 stacks: +`gui-deploy.yml` is an **update-only** workflow for the pre-created test-v2 and prod-v2 stacks. Both parameters default to `false`: + +| `deployInfra` | `deployToProd` | Workflow | +| --- | --- | --- | +| `false` | `false` | Build, deploy code to test | +| `true` | `false` | Build, deploy test infrastructure with the current image, deploy code to test | +| `false` | `true` | Build, deploy code to test, production approval, deploy the same image to production | +| `true` | `true` | Build, test infrastructure, test code, production approval, production infrastructure, production code | + +Qualifying merges to `main` automatically deploy **code only to test**. Production remains opt-in: manually queue a commit merged to `main` with `deployToProd=true`. Approval rejects on timeout and the requester cannot self-approve. An infrastructure failure blocks the corresponding code stage; a code failure never invokes infrastructure rollback. + +#### Code-only deployment + +`infra/pipelines/deploy_code.py` updates only the existing `pyrit-gui` container image. It validates the subscription, registry digest, environment, and single-container/single-revision topology before writing. Application configuration, identity, networking, NAT, and Front Door are not redeployed. + +The script verifies the exact new revision and its existing access mode: direct ACA `/api/health` when public access is enabled, or Front Door `/api/health` when public access is disabled. It reports the mode explicitly and never falls back from a failed private path to public access. A public-mode success is **not** certification of Private Link readiness. The data-plane check has a five-minute budget after revision readiness. + +Code deployment does not downgrade or create a database; application startup still follows the image's normal migration behavior. Failures are reported without automatic image or database rollback because migrations may make the previous image incompatible. The previous image digest is logged for an explicit recovery decision. + +#### Optional infrastructure deployment + +Set `deployInfra=true` when changing Bicep-managed configuration or networking. Each infrastructure stage runs **before** the new code, using the image already deployed to that environment, not the Build output. Infrastructure rollback therefore cannot install a newly built application that has not passed code verification. + +`infra/pipelines/deploy_public_nat.sh` retains the existing infrastructure safeguards: 1. Build the source image and push a commit-SHA tag to ACR. -2. Capture the exact pushed digest and pass it across stages. +2. Capture the exact pushed digest for the code stages; infrastructure retains the currently deployed digest. 3. Require the existing app, environment, VNet, subnet, NAT, and reserved PIP; validate their IDs, prefixes, tags, SKU, allocation, and attachments. 4. Run a full ARM `what-if` through a fail-closed validator; reject malformed results, deletions, cross-resource-group writes, protected-network deltas other than the documented read-only NAT/PIP normalization, and core network, app, or Log Analytics workspace creates. The expected PIP protection lock may be created. 5. Preserve policy-managed PIP tags and deploy with Front Door Private Link, ACA public access disabled, and PIP protection enabled. 6. Validate the AFD origin targets the expected ACA environment, approve only active requests with the deterministic message, and require the ACA-side connection to report `Approved`. AFD can continue to display `Pending` after approval, so successful AFD health is the data-plane readiness signal. -7. Allow up to 30 minutes for Front Door propagation, then verify ACA public access is disabled, the digest-pinned revision and Front Door `/api/health` are healthy, direct ACA access is unavailable, and the PIP resource ID/address is unchanged. +7. Allow up to 30 minutes for Front Door propagation, then verify ACA public access is disabled, the retained-image revision and Front Door `/api/health` are healthy, direct ACA access is unavailable, and the PIP resource ID/address is unchanged. 8. If cutover validation fails, redeploy the prior public AFD origin and re-enable ACA public access; otherwise print the Front Door URL and static egress IPv4. -Qualifying merges to `main` automatically deploy test. Production deployment is independent of PyRIT package releases: manually queue a commit merged to `main` with `deployToProd=true`. The workflow deploys test first, then requires a timeout-rejecting manual approval whose requester cannot self-approve. - `copyrit-gui-common` supplies the shared image settings: | Variable | Purpose | @@ -441,7 +462,7 @@ Both `copyrit-gui-test` and `copyrit-gui-prod` supply: | `keyVaultResourceId`, `envSecretName` | Existing runtime configuration secret | | `acrResourceId`, `enableOtel` | Registry resource ID and observability setting | -The container image is not a library variable. The Build stage publishes the exact pushed digest as `immutableImage`, and both deployment stages consume that output. Do not add the legacy `image`, `resourceGroup`, `appName`, or `enablePrivateEndpoint` variables; the current workflow does not consume them. +The container image is not a library variable. The Build stage publishes the exact pushed digest as `immutableImage`, and both code stages consume that output. Infrastructure stages discover the existing image directly from ACA. Code-only runs consume only `deploymentResourceGroup`, `deploymentAppName`, and `acrResourceId` from each environment group; changes to the other configuration values require `deployInfra=true`. Do not add the legacy `image`, `resourceGroup`, `appName`, or `enablePrivateEndpoint` variables; the current workflow does not consume them. Pipeline definition 139 reads `gui-deploy.yml` from the GitHub commit being queued. Treat YAML and variable-group contract changes as one release: do not remove old keys before the commit that consumes the replacement keys reaches the target branch. Otherwise ADO leaves unresolved `$(name)` text in Bash, where it is interpreted as command substitution. @@ -451,7 +472,7 @@ The resource group, registry, image-pull authorization, managed identity, Key Va The internal workflow is update-only for networking: its app name and prefixes must resolve to the existing app/environment/VNet/subnet/NAT/PIP. It records the current PIP resource ID and address before preview, requires protected resources to remain unchanged except Azure read-only normalization, and verifies the same PIP/address after deployment. -The workflow also creates a `CanNotDelete` lock scoped to the reserved PIP. Its validated Front Door origin uses Private Link to the ACA environment, and the ACA public endpoint is disabled after deployment. +The optional infrastructure stage also creates a `CanNotDelete` lock scoped to the reserved PIP. Its validated Front Door origin uses Private Link to the ACA environment, and the ACA public endpoint is disabled after a successful infrastructure deployment. Code-only runs preserve the existing configuration, including an existing public-access fallback. ## Post-Deployment @@ -604,7 +625,7 @@ Supported Azure integrations, including OpenAI, Content Safety, and Speech, can ## Notes -- **Network topology**: Public ACA-managed HTTPS ingress with optional `allowedCidr` plus VNet-integrated fixed NAT egress is the base topology. Front Door Premium is an optional inbound layer. Private Link plus disabled ACA public access makes Front Door the only public application path. The team ADO workflow enables this isolated-origin mode. `allowedCidr` must be empty when Front Door is enabled; Bicep rejects the combination. +- **Network topology**: Public ACA-managed HTTPS ingress with optional `allowedCidr` plus VNet-integrated fixed NAT egress is the base topology. Front Door Premium is an optional inbound layer. Private Link plus disabled ACA public access makes Front Door the only public application path. The team ADO infrastructure stage enables this isolated-origin mode; default code-only runs preserve existing networking. `allowedCidr` must be empty when Front Door is enabled; Bicep rejects the combination. - **Ingress vs. egress**: Front Door affects inbound requests only. The reserved NAT public IP remains the source for ACA-originated outbound connections. - **NAT routing**: NAT Gateway supplies the outbound source IP only while the subnet's effective default route remains `Internet`. A UDR or propagated BGP `0.0.0.0/0` route to a firewall or gateway takes precedence; in that topology, allow-list the egress device's public IP instead. - **Network outputs**: `egressPublicIpAddress`, `natGatewayId`, `acaInfrastructureSubnetId`, and `vnetName` describe the created network. diff --git a/infra/pipelines/deploy-infra.yml b/infra/pipelines/deploy-infra.yml new file mode 100644 index 0000000000..143b2e9ac0 --- /dev/null +++ b/infra/pipelines/deploy-infra.yml @@ -0,0 +1,60 @@ +parameters: + - name: stageName + type: string + - name: dependsOn + type: string + - name: slot + type: string + values: + - test + - prod + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Deploy ${{ parameters.slot }} infrastructure' + dependsOn: ${{ parameters.dependsOn }} + variables: + - group: copyrit-gui-common + - group: copyrit-gui-${{ parameters.slot }} + pool: + vmImage: 'ubuntu-latest' + jobs: + - deployment: DeployInfrastructure + displayName: 'Preview, deploy, and verify infrastructure with the current image' + timeoutInMinutes: 120 + environment: 'copyrit-${{ parameters.slot }}' + strategy: + runOnce: + deploy: + steps: + - checkout: self + fetchDepth: 1 + - task: AzureCLI@2 + displayName: 'Preview, deploy, and verify ${{ parameters.slot }} infrastructure' + env: + PYRIT_SLOT: ${{ parameters.slot }} + PYRIT_BUILD_ID: $(Build.BuildId) + PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) + PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) + PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) + PYRIT_APP_NAME: $(deploymentAppName) + PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) + PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) + PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) + PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) + PYRIT_ENTRA_TENANT_ID: $(entraTenantId) + PYRIT_ENTRA_CLIENT_ID: $(entraClientId) + PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) + PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId) + PYRIT_CONFIG_FILE_URI: $(pyritConfigFileUri) + PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) + PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) + PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) + PYRIT_ACR_RESOURCE_ID: $(acrResourceId) + PYRIT_ENABLE_OTEL: $(enableOtel) + PYRIT_ENV_SECRET_NAME: $(envSecretName) + inputs: + azureSubscription: '$(azureServiceConnection)' + scriptType: 'bash' + scriptLocation: 'scriptPath' + scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' diff --git a/infra/pipelines/deploy_code.py b/infra/pipelines/deploy_code.py new file mode 100644 index 0000000000..d2803f623a --- /dev/null +++ b/infra/pipelines/deploy_code.py @@ -0,0 +1,192 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +"""Update only the existing CoPyRIT container image and verify its current access mode.""" + +import os +import re +import subprocess +import sys +import time + + +def _az(*arguments: str) -> str: + return subprocess.run( + ["az", *arguments, "--only-show-errors"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + +def _value(*arguments: str, query: str) -> str: + value = _az(*arguments, "--query", query, "--output", "tsv") + if not value or value in {"None", "null"} or "\n" in value or "\t" in value: + raise ValueError(f"Azure returned no single value for {query}") + return value + + +def _health_url(*, app_arguments: tuple[str, ...], environment_id: str, app_name: str) -> tuple[str, str]: + access = _value("containerapp", "env", "show", "--ids", environment_id, query="properties.publicNetworkAccess") + if access == "Enabled": + hostname = _value("containerapp", "show", *app_arguments, query="properties.configuration.ingress.fqdn") + suffix = ".azurecontainerapps.io" + elif access == "Disabled": + resource_group_id = environment_id.split("/providers/")[0] + hostname = _value( + "rest", + "--method", + "get", + "--url", + f"https://management.azure.com{resource_group_id}/providers/Microsoft.Cdn/profiles/" + f"{app_name}-afd/afdEndpoints?api-version=2024-09-01", + query="value[?properties.enabledState=='Enabled'].properties.hostName", + ) + suffix = ".azurefd.net" + else: + raise ValueError(f"Unsupported ACA public network access state: {access}") + if not re.fullmatch(r"[a-z0-9][a-z0-9.-]*", hostname) or not hostname.endswith(suffix): + raise ValueError("Azure returned an unexpected health-check hostname") + return f"https://{hostname}/api/health", access + + +def _wait_for_revision(*, app_arguments: tuple[str, ...], revision: str, image: str) -> None: + arguments = ("containerapp", "revision", "show", *app_arguments, "--revision", revision) + if _value(*arguments, query="properties.template.containers[0].image") != image: + raise ValueError("The deployed revision does not contain the requested image") + for attempt in range(1, 6): + health = _az(*arguments, "--query", "properties.healthState", "--output", "tsv") + print(f"Revision {revision} health attempt {attempt}/5: {health or ''}", flush=True) + if health == "Healthy": + return + if attempt < 5: + time.sleep(120) + raise RuntimeError("Deployed revision did not become healthy; networking was not changed") + + +def _wait_for_http_health(url: str) -> None: + deadline = time.monotonic() + 300 + while (remaining := deadline - time.monotonic()) > 0: + response = subprocess.run( + [ + "curl", + "--silent", + "--show-error", + "--output", + os.devnull, + "--write-out", + "%{http_code}", + "--max-time", + str(min(30, remaining)), + url, + ], + capture_output=True, + text=True, + check=False, + ) + status = response.stdout.strip() + print(f"Application health at {url}: {status or ''}", flush=True) + if response.returncode: + print(f"Health request failed: {response.stderr.strip()}", file=sys.stderr) + elif status == "200": + return + time.sleep(max(0, min(30, deadline - time.monotonic()))) + raise RuntimeError("Application health check failed; networking was not changed") + + +def deploy_code(*, slot: str, resource_group: str, app_name: str, acr_resource_id: str, image: str) -> None: + """Deploy an immutable image without applying templates or changing access settings.""" + if slot not in {"test", "prod"}: + raise ValueError("Invalid deployment slot") + if not re.fullmatch(r"[a-zA-Z0-9_.()-]{1,90}", resource_group) or resource_group.endswith("."): + raise ValueError("Invalid deployment resource group") + if not re.fullmatch(r"[a-z][a-z0-9-]{0,30}[a-z0-9]", app_name): + raise ValueError("Invalid container app name") + acr = re.fullmatch( + r"/subscriptions/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})" + r"/resourcegroups/[^/]+/providers/microsoft\.containerregistry/registries/([a-z0-9]{5,50})", + acr_resource_id.casefold(), + ) + if acr is None: + raise ValueError("ACR resource ID is not canonical") + subscription, registry = acr.groups() + if not re.fullmatch( + rf"{registry}\.azurecr\.io/[a-z0-9]+(?:[._-][a-z0-9]+)*" + r"(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*@sha256:[0-9a-fA-F]{64}", + image, + ): + raise ValueError("Image must be an immutable digest from the configured ACR") + if _value("account", "show", query="id").casefold() != subscription: + raise ValueError("Azure subscription does not match ACR") + + app_arguments = ("--resource-group", resource_group, "--name", app_name) + show = ("containerapp", "show", *app_arguments) + resource_group_id = f"/subscriptions/{subscription}/resourceGroups/{resource_group}" + environment_id = f"{resource_group_id}/providers/Microsoft.App/managedEnvironments/{app_name}-env" + if _value(*show, query="properties.managedEnvironmentId").casefold() != environment_id.casefold(): + raise ValueError("Existing app does not belong to the expected environment") + if _value(*show, query="properties.configuration.activeRevisionsMode") != "Single": + raise ValueError("Code-only deployment requires the existing single-revision topology") + if ( + _value(*show, query="length(properties.template.containers)") != "1" + or _value(*show, query="properties.template.containers[0].name") != "pyrit-gui" + ): + raise ValueError("Code-only deployment requires the existing pyrit-gui container") + url, access = _health_url(app_arguments=app_arguments, environment_id=environment_id, app_name=app_name) + previous_image = _value(*show, query="properties.template.containers[0].image") + print(f"Code-only {slot} deployment; ACA public access: {access}; verification URL: {url}", flush=True) + print(f"Previous image (not automatically restored): {previous_image}", flush=True) + if previous_image != image: + _az( + "containerapp", + "update", + *app_arguments, + "--container-name", + "pyrit-gui", + "--image", + image, + "--output", + "none", + ) + revision = _value(*show, query="properties.latestRevisionName") + _wait_for_revision(app_arguments=app_arguments, revision=revision, image=image) + _wait_for_http_health(url) + if ( + _value(*show, query="properties.latestRevisionName") != revision + or _value(*show, query="properties.latestReadyRevisionName") != revision + or _value(*show, query="properties.template.containers[0].image") != image + ): + raise RuntimeError("The verified image revision is not the current ready revision") + final_url, final_access = _health_url(app_arguments=app_arguments, environment_id=environment_id, app_name=app_name) + if (final_url, final_access) != (url, access): + raise RuntimeError("The application's access mode changed during code-only deployment") + print(f"Code deployment healthy: {revision}; networking unchanged; verified {url}", flush=True) + + +def main() -> int: + """Read pipeline inputs and fail without rolling back infrastructure.""" + inputs = { + "slot": "PYRIT_SLOT", + "resource_group": "PYRIT_DEPLOYMENT_RESOURCE_GROUP", + "app_name": "PYRIT_APP_NAME", + "acr_resource_id": "PYRIT_ACR_RESOURCE_ID", + "image": "PYRIT_CONTAINER_IMAGE", + } + try: + values = {} + for name, variable in inputs.items(): + value = os.environ.get(variable, "") + if not value or value.startswith("$("): + raise ValueError(f"Required deployment value is missing or unresolved: {variable}") + values[name] = value + deploy_code(**values) + except (ValueError, RuntimeError, OSError, subprocess.CalledProcessError) as error: + print(f"##vso[task.logissue type=error]Code deployment failed: {error}", file=sys.stderr) + if isinstance(error, subprocess.CalledProcessError) and error.stderr: + print(error.stderr, file=sys.stderr) + print("No automatic image, database, or infrastructure rollback was attempted.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/pipelines/deploy_public_nat.sh b/infra/pipelines/deploy_public_nat.sh index db577aaa3e..a286b6240d 100644 --- a/infra/pipelines/deploy_public_nat.sh +++ b/infra/pipelines/deploy_public_nat.sh @@ -15,7 +15,6 @@ required_variables=( PYRIT_AGENT_TEMP_DIRECTORY PYRIT_DEPLOYMENT_RESOURCE_GROUP PYRIT_APP_NAME - PYRIT_CONTAINER_IMAGE PYRIT_VNET_ADDRESS_PREFIX PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX PYRIT_MANAGED_IDENTITY_RESOURCE_ID @@ -195,7 +194,7 @@ normalized_expected_pip_id=$(lowercase "$expected_pip_id") existing_app=$(az containerapp show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME" \ - --query '{id:id,environmentId:properties.managedEnvironmentId,tags:tags}' -o json 2>/dev/null || true) + --query '{id:id,environmentId:properties.managedEnvironmentId,tags:tags,containers:properties.template.containers[].{name:name,image:image}}' -o json 2>/dev/null || true) existing_environment=$(az containerapp env show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-env" \ @@ -259,23 +258,30 @@ if [[ "$deployment_tags" == *'<'* || "$deployment_tags" == "null" \ exit 1 fi -if [[ ! "$PYRIT_CONTAINER_IMAGE" =~ ^([^/]+)/(.+)@(sha256:[0-9a-fA-F]{64})$ ]]; then - echo "##vso[task.logissue type=error]Built image must be an immutable registry digest" +if [[ "$(jq '.containers | length' <<< "$existing_app")" != "1" \ + || "$(jq -r '.containers[0].name' <<< "$existing_app")" != "pyrit-gui" ]]; then + echo "##vso[task.logissue type=error]Infrastructure deployment requires the existing pyrit-gui container" + exit 1 +fi +current_image=$(jq -r '.containers[0].image // empty' <<< "$existing_app") +if [[ ! "$current_image" =~ ^([^/]+)/(.+)@(sha256:[0-9a-fA-F]{64})$ ]]; then + echo "##vso[task.logissue type=error]Current image must be an immutable registry digest" exit 1 fi registry_server=${BASH_REMATCH[1]} repository=${BASH_REMATCH[2]} digest=${BASH_REMATCH[3]} if [[ "$registry_server" != "$acr_name.azurecr.io" ]]; then - echo "##vso[task.logissue type=error]Built image registry does not match ACR resource ID" + echo "##vso[task.logissue type=error]Current image registry does not match ACR resource ID" exit 1 fi repository_pattern='^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*$' if [[ ! "$repository" =~ $repository_pattern ]]; then - echo "##vso[task.logissue type=error]Built image repository is invalid" + echo "##vso[task.logissue type=error]Current image repository is invalid" exit 1 fi immutable_image="$registry_server/$repository@$digest" +echo "Infrastructure-only deployment; retaining current image: $immutable_image" private_link_request_message="Azure Front Door private access to $PYRIT_APP_NAME" parameters=( diff --git a/tests/unit/infra/test_code_deployment.py b/tests/unit/infra/test_code_deployment.py new file mode 100644 index 0000000000..283b6892b0 --- /dev/null +++ b/tests/unit/infra/test_code_deployment.py @@ -0,0 +1,204 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +"""Exercise code-only deployment with no Azure or network calls.""" + +import os +import subprocess +import unittest +from unittest.mock import patch + +from infra.pipelines import deploy_code + +SUBSCRIPTION = "11111111-1111-1111-1111-111111111111" +ENVIRONMENT = ( + f"/subscriptions/{SUBSCRIPTION}/resourceGroups/copyrit-test" + "/providers/Microsoft.App/managedEnvironments/copyrit-test-env" +) +IMAGE = f"copyritacr.azurecr.io/pyrit@sha256:{'a' * 64}" +PREVIOUS_IMAGE = f"copyritacr.azurecr.io/pyrit@sha256:{'b' * 64}" +ACA_HOST = "copyrit-test.example.westus2.azurecontainerapps.io" +AFD_HOST = "copyrit-test.example.azurefd.net" +REVISION = "copyrit-test--0000002" + + +class TestCodeDeployment(unittest.TestCase): + def setUp(self) -> None: + self.arguments = { + "slot": "test", + "resource_group": "copyrit-test", + "app_name": "copyrit-test", + "acr_resource_id": ( + f"/subscriptions/{SUBSCRIPTION}/resourceGroups/shared" + "/providers/Microsoft.ContainerRegistry/registries/copyritacr" + ), + "image": IMAGE, + } + self.values = { + "id": SUBSCRIPTION, + "properties.managedEnvironmentId": ENVIRONMENT, + "properties.configuration.activeRevisionsMode": "Single", + "length(properties.template.containers)": "1", + "properties.template.containers[0].name": "pyrit-gui", + "properties.configuration.ingress.fqdn": ACA_HOST, + "properties.publicNetworkAccess": "Enabled", + "properties.latestRevisionName": REVISION, + "properties.latestReadyRevisionName": REVISION, + "properties.healthState": "Healthy", + "value[?properties.enabledState=='Enabled'].properties.hostName": AFD_HOST, + } + self.current_image = PREVIOUS_IMAGE + self.calls: list[tuple[str, ...]] = [] + self.az_patch = patch.object(deploy_code, "_az", side_effect=self._az) + self.az_patch.start() + self.addCleanup(self.az_patch.stop) + self.http = patch.object(deploy_code, "_wait_for_http_health").start() + self.addCleanup(patch.stopall) + + def _az(self, *arguments: str) -> str: + self.calls.append(arguments) + if arguments[:2] == ("containerapp", "update"): + self.current_image = arguments[arguments.index("--image") + 1] + return "" + query = arguments[arguments.index("--query") + 1] + if query == "properties.template.containers[0].image": + return self.current_image + return self.values[query] + + def test_public_mode_updates_only_image_and_checks_aca(self) -> None: + deploy_code.deploy_code(**self.arguments) + + updates = [call for call in self.calls if call[:2] == ("containerapp", "update")] + assert updates == [ + ( + "containerapp", + "update", + "--resource-group", + "copyrit-test", + "--name", + "copyrit-test", + "--container-name", + "pyrit-gui", + "--image", + IMAGE, + "--output", + "none", + ) + ] + self.http.assert_called_once_with(f"https://{ACA_HOST}/api/health") + assert not any(call[0] in {"deployment", "network", "rest"} for call in self.calls) + + def test_private_mode_checks_front_door_without_changing_network(self) -> None: + self.values["properties.publicNetworkAccess"] = "Disabled" + self.arguments["slot"] = "prod" + + deploy_code.deploy_code(**self.arguments) + + self.http.assert_called_once_with(f"https://{AFD_HOST}/api/health") + rest_calls = [call for call in self.calls if call[0] == "rest"] + assert len(rest_calls) == 2 + assert all(call[1:3] == ("--method", "get") for call in rest_calls) + assert not any(call[0] in {"deployment", "network"} for call in self.calls) + + def test_same_image_still_verified_without_update(self) -> None: + self.current_image = IMAGE + + deploy_code.deploy_code(**self.arguments) + + assert not any(call[:2] == ("containerapp", "update") for call in self.calls) + self.http.assert_called_once() + + def test_invalid_inputs_fail_before_azure_calls(self) -> None: + invalid = { + "slot": "other", + "resource_group": "../wrong", + "app_name": "wrong/name", + "acr_resource_id": "/subscriptions/wrong", + "image": "another.azurecr.io/pyrit:latest", + } + for key, value in invalid.items(): + with self.subTest(key=key), self.assertRaises(ValueError): + deploy_code.deploy_code(**(self.arguments | {key: value})) + assert not self.calls + + def test_invalid_existing_topology_never_updates(self) -> None: + invalid = { + "id": "22222222-2222-2222-2222-222222222222", + "properties.managedEnvironmentId": ENVIRONMENT + "-other", + "properties.configuration.activeRevisionsMode": "Multiple", + "length(properties.template.containers)": "2", + "properties.template.containers[0].name": "other", + "properties.publicNetworkAccess": "unexpected", + "properties.configuration.ingress.fqdn": "attacker.example", + } + for query, value in invalid.items(): + with self.subTest(query=query), patch.dict(self.values, {query: value}), self.assertRaises(ValueError): + deploy_code.deploy_code(**self.arguments) + assert not any(call[:2] == ("containerapp", "update") for call in self.calls) + + def test_private_mode_requires_one_valid_front_door_endpoint(self) -> None: + self.values["properties.publicNetworkAccess"] = "Disabled" + query = "value[?properties.enabledState=='Enabled'].properties.hostName" + for value in ("", "None", f"{AFD_HOST}\nother.azurefd.net", "attacker.example"): + with self.subTest(value=value), patch.dict(self.values, {query: value}), self.assertRaises(ValueError): + deploy_code.deploy_code(**self.arguments) + assert not any(call[:2] == ("containerapp", "update") for call in self.calls) + + def test_unhealthy_revision_fails_without_rollback(self) -> None: + self.values["properties.healthState"] = "Unhealthy" + with patch.object(deploy_code.time, "sleep"), self.assertRaisesRegex(RuntimeError, "revision"): + deploy_code.deploy_code(**self.arguments) + assert self.current_image == IMAGE + assert len([call for call in self.calls if call[:2] == ("containerapp", "update")]) == 1 + self.http.assert_not_called() + assert not any(call[0] in {"deployment", "network", "rest"} for call in self.calls) + + def test_http_failure_does_not_fallback_or_roll_back(self) -> None: + self.values["properties.publicNetworkAccess"] = "Disabled" + self.http.side_effect = RuntimeError("Health check failed") + with self.assertRaisesRegex(RuntimeError, "Health check"): + deploy_code.deploy_code(**self.arguments) + self.http.assert_called_once_with(f"https://{AFD_HOST}/api/health") + assert self.current_image == IMAGE + assert len([call for call in self.calls if call[:2] == ("containerapp", "update")]) == 1 + + def test_current_ready_revision_must_be_the_verified_revision(self) -> None: + self.values["properties.latestReadyRevisionName"] = "copyrit-test--old" + with self.assertRaisesRegex(RuntimeError, "current ready revision"): + deploy_code.deploy_code(**self.arguments) + + def test_access_mode_change_is_not_reported_as_success(self) -> None: + self.http.side_effect = lambda _: self.values.update({"properties.publicNetworkAccess": "Disabled"}) + with self.assertRaisesRegex(RuntimeError, "access mode changed"): + deploy_code.deploy_code(**self.arguments) + + def test_main_reports_unresolved_input_without_azure_calls(self) -> None: + with patch.dict(os.environ, {"PYRIT_SLOT": "$(slot)"}): + assert deploy_code.main() == 1 + assert not self.calls + + +class TestHttpVerification(unittest.TestCase): + def test_http_success_does_not_follow_redirects(self) -> None: + response = subprocess.CompletedProcess(args=[], returncode=0, stdout="200", stderr="") + with patch.object(deploy_code.subprocess, "run", return_value=response) as run: + deploy_code._wait_for_http_health("https://example.azurefd.net/api/health") + arguments = run.call_args.args[0] + assert "--location" not in arguments + assert "--insecure" not in arguments + assert "--max-time" in arguments + + def test_http_errors_and_redirects_exhaust_the_bounded_budget(self) -> None: + for status, exit_code in (("302", 0), ("504", 0), ("200", 28)): + response = subprocess.CompletedProcess(args=[], returncode=exit_code, stdout=status, stderr="failure") + with ( + self.subTest(status=status, exit_code=exit_code), + patch.object(deploy_code.subprocess, "run", return_value=response), + patch.object(deploy_code.time, "monotonic", side_effect=[0, 290, 299, 301]), + patch.object(deploy_code.time, "sleep"), + self.assertRaisesRegex(RuntimeError, "health check failed"), + ): + deploy_code._wait_for_http_health("https://example.azurefd.net/api/health") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/infra/test_pipeline_guardrails.py b/tests/unit/infra/test_pipeline_guardrails.py index c124cae9c8..0ddb9c740c 100644 --- a/tests/unit/infra/test_pipeline_guardrails.py +++ b/tests/unit/infra/test_pipeline_guardrails.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Guard the single-topology Azure DevOps deployment contract.""" +"""Guard the independent application and infrastructure deployment contract.""" import json import os @@ -12,9 +12,12 @@ import unittest from pathlib import Path +import yaml + REPO_ROOT = Path(__file__).resolve().parents[3] PIPELINE = REPO_ROOT / "gui-deploy.yml" DEPLOY_SCRIPT = REPO_ROOT / "infra" / "pipelines" / "deploy_public_nat.sh" +INFRA_TEMPLATE = REPO_ROOT / "infra" / "pipelines" / "deploy-infra.yml" WHAT_IF_VALIDATOR = REPO_ROOT / "infra" / "pipelines" / "validate_what_if.py" EXAMPLE_PARAMETERS = REPO_ROOT / "infra" / "parameters.example.json" DEMO_PARAMETERS = REPO_ROOT / "infra" / "parameters.demo.json" @@ -50,6 +53,7 @@ class TestPipelineGuardrails(unittest.TestCase): def setUpClass(cls): cls.pipeline = PIPELINE.read_text(encoding="utf-8") cls.deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + cls.infra_template = INFRA_TEMPLATE.read_text(encoding="utf-8") def test_pipeline_has_one_test_and_prod_workflow(self): assert "deploymentTarget" not in self.pipeline @@ -60,7 +64,39 @@ def test_pipeline_has_one_test_and_prod_workflow(self): assert "stage: DeployProd" in self.pipeline assert "DeployReplacement" not in self.pipeline assert self.pipeline.count("timeoutInMinutes: 120") == 2 - assert self.pipeline.count("scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh'") == 2 + assert ( + self.pipeline.count('inlineScript: python3 "$(Build.SourcesDirectory)/infra/pipelines/deploy_code.py"') == 2 + ) + assert "deploy_public_nat.sh" not in self.pipeline + assert "scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh'" in self.infra_template + + def test_infrastructure_is_explicit_and_runs_before_code(self) -> None: + pipeline = yaml.safe_load(self.pipeline) + parameters = {parameter["name"]: parameter for parameter in pipeline["parameters"]} + assert parameters["deployInfra"]["default"] is False + assert parameters["deployToProd"]["default"] is False + stages = pipeline["stages"] + conditional = "${{ if eq(parameters.deployInfra, true) }}" + infrastructure = [stage[conditional][0] for stage in stages if conditional in stage] + assert [stage["parameters"] for stage in infrastructure] == [ + {"stageName": "DeployTestInfra", "dependsOn": "Build", "slot": "test"}, + {"stageName": "DeployProdInfra", "dependsOn": "ApproveProd", "slot": "prod"}, + ] + assert all(stage["template"] == "infra/pipelines/deploy-infra.yml" for stage in infrastructure) + code_stages = {stage["stage"]: stage for stage in stages if stage.get("stage") in {"DeployTest", "DeployProd"}} + assert code_stages["DeployTest"]["dependsOn"] == ["Build", {conditional: ["DeployTestInfra"]}] + assert code_stages["DeployProd"]["dependsOn"] == [ + "ApproveProd", + "Build", + {conditional: ["DeployProdInfra"]}, + ] + template = yaml.safe_load(self.infra_template)["stages"][0] + assert template["dependsOn"] == "${{ parameters.dependsOn }}" + assert "condition" not in template # Infrastructure must not run after a skipped approval. + assert "PYRIT_CONTAINER_IMAGE" not in self.infra_template + assert "current_image=$(jq" in self.deploy_script + assert "retaining current image" in self.deploy_script + assert "PYRIT_CONTAINER_IMAGE" not in self.deploy_script def test_production_remains_opt_in_and_independently_approved(self): assert "job: ValidateProdConfiguration" in self.pipeline @@ -79,7 +115,7 @@ def test_production_remains_opt_in_and_independently_approved(self): assert '"$BUILD_SOURCEBRANCH" != refs/heads/main' in self.pipeline assert "eq(variables['Build.SourceBranch'], 'refs/heads/main')" in self.pipeline assert "refs/heads/releases/" not in self.pipeline - assert "condition: succeeded('ApproveProd')" in self.pipeline + assert "condition: and(succeeded(), succeeded('ApproveProd'))" in self.pipeline def test_deploy_resolves_digest_and_previews_before_apply(self): assert "name: BuildImage" in self.pipeline @@ -103,9 +139,9 @@ def test_deploy_resolves_digest_and_previews_before_apply(self): assert '"disableContainerAppsPublicAccess=true"' in self.deploy_script def test_pipeline_passes_values_via_environment(self): - deploy_yaml = self.pipeline[self.pipeline.index("stage: DeployTest") :] + deploy_yaml = self.infra_template assert "PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup)" in deploy_yaml - assert "PYRIT_CONTAINER_IMAGE: $(immutableImage)" in deploy_yaml + assert "PYRIT_CONTAINER_IMAGE: $(immutableImage)" in self.pipeline assert "PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr)" in deploy_yaml assert "PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId)" in deploy_yaml assert "PYRIT_ADMIN_GROUP_OBJECT_ID: $(adminGroupObjectId)" in deploy_yaml